add ECUT_pos_html.py

yhangf · yhangf · commit 841168d8b331 · 2016-09-18T12:12:50.000+08:00
diff --git a/ECUT_pos_html.py b/ECUT_pos_html.py
@@ -0,0 +1,57 @@
+import requests
+import re
+from bs4 import BeautifulSoup as bs
+
+
+def crawl_all_main_url(page=10):
+    # 默认抓取官网前十页招聘信息的url
+    all_url_list = []
+    for _ in range(1, page+1):
+        url = 'http://zjc.ecit.edu.cn/jy/app/newslist.php?BigClassName=%D5%D0%C6%B8%D0%C5%CF%A2&Page={0}'.format(_)
+        page_html = requests.get(url).text
+        x_url_reg = re.compile('<a class="t_13px" href="(.*?)"')
+        x_url = re.findall(x_url_reg, page_html)
+        main_url = ['http://zjc.ecit.edu.cn/jy/app/{0}'.format(i) for i in x_url]
+        all_url_list.extend(main_url)
+    return all_url_list
+
+def get_title(son_url):
+    # 判断该网页是否为校园招聘
+    html = requests.get(son_url).content.decode('gbk')
+    explain_text_reg = re.compile('<h1 class="newstitle">(.*?)</h1>')
+    explain_text = re.findall(explain_text_reg, html)[0]
+    if ('时间' and '地点') in explain_text:
+        return True
+    else:
+        pass
+
+def save_html():
+    all_url_list = crawl_all_main_url()
+    for son_url in all_url_list:
+        if get_title(son_url):
+            text_html = requests.get(son_url).content.decode('gbk')
+            domain_url = 'http://zjc.ecit.edu.cn/jy'
+            img_url_reg = re.compile('border=0 src="\.\.(.*?)"')
+            child_url = re.findall(img_url_reg, text_html)
+            if child_url != []:
+                img_url = domain_url + child_url[0]
+                re_url = 'src="..{0}"'.format(child_url[0])
+                end_url = 'src="{0}"'.format(img_url)
+                end_html = text_html.replace(re_url, end_url)
+                soup = bs(end_html, 'lxml')
+                text_div = soup.find_all('div', id='main')[0]
+                with open('./{0}.html'.format(son_url[-11:]), 'wb') as file:
+                    text_html = '<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">U职网提供数据咨询服务   <html xmlns="http://www.w3.org/1999/xhtml">                            <head>                            <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />                            </head>                            {0}                            </body>'.format(text_div)
+                    file.write(text_html.encode('utf-8'))
+            else:
+                with open('./{0}.html'.format(son_url[-11:]), 'wb') as file:
+                    html = requests.get(son_url).content.decode('gbk')
+                    soup = bs(text_html, 'lxml')
+                    text_div = soup.find_all('div', id='main')[0]
+                    text_html = '<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">U职网提供数据咨询服务   <html xmlns="http://www.w3.org/1999/xhtml">                        <head>                        <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />                        </head>                        {0}                        </body>'.format(text_div)
+                    file.write(text_html.encode('utf-8'))
+        else:
+            continue
+
+if __name__ == '__main__':
+    save_html()
diff --git a/README.md b/README.md
@@ -13,3 +13,5 @@
 ##### 6.  student_img.py: 基于本学校官网的url漏洞，获取所有注册学生学籍证件照
 
 ##### 7.  JDSpider.py: 大批量抓取京东商品id和标签
+
+##### 8.  ECUT_pos_html.py: 抓取学校官网所有校园招聘信息，并保存为html格式，图片也会镶嵌在html中。
