update file

Author: yhangf

ECUT_pos_html.py

@@ -1,4 +1,4 @@
-# PythonCrawler: 用python编写的简单爬虫项目集合
+# PythonCrawler: 用python编写的爬虫项目集合
 ```
  (                                                                        
  )\ )          )    )               (                       (             
@@ -9,30 +9,90 @@
 |  _/| || ||  _|| ' \ / _ \| ' \))| (__ | '_|/ _` |\ V  V /| |/ -_) | '_|
 |_|   \_, | \__||_||_|\___/|_||_|  \___||_|  \__,_| \_/\_/ |_|\___| |_|   
       |__/  
-                                                        by yanghangfeng
+                                                ——————by yanghangfeng
 ```
 
 
-# 模块介绍
+# spiderFile模块简介
 
-##### 1.  [baiduImg.py](https://github.com/Fenghuapiao/PythonCrawler/blob/master/baiduImg.py): 抓取百度的‘高清摄影’图片
+##### 1.    [baidu_sy_img.py](https://github.com/Fenghuapiao/PythonCrawler/blob/master/baiduImg.py): 抓取百度的‘高清摄影’图片
 
-##### 2.  [baiduImg2.py](https://github.com/Fenghuapiao/PythonCrawler/blob/master/baiduImg2.py): 抓取百度图片‘唯美意境’模块
+##### 2.  [baidu_wm_img.py](https://github.com/Fenghuapiao/PythonCrawler/blob/master/baiduImg2.py): 抓取百度图片‘唯美意境’模块
 
-##### 3.  [GetPhotos2.py](https://github.com/Fenghuapiao/PythonCrawler/blob/master/GetPhotos2.py): 抓取百度贴吧某话题下的所有图片
+##### 3.  [get_photos.py](https://github.com/Fenghuapiao/PythonCrawler/blob/master/GetPhotos2.py): 抓取百度贴吧某话题下的所有图片
 
-##### 4.  [getWebAllImg.py](https://github.com/Fenghuapiao/PythonCrawler/blob/master/getWebAllImg.py): 抓取整个网站的图片
+##### 4.  [get_web_all_img.py](https://github.com/Fenghuapiao/PythonCrawler/blob/master/getWebAllImg.py): 抓取整个网站的图片
 
-##### 5.  [lagouPositionSpider.py](https://github.com/Fenghuapiao/PythonCrawler/blob/master/lagouPositionSpider.py): 任意输入关键字，一键抓取与关键字相关的职位招聘信息，并保存到本地文件
+##### 5.  [lagou_position_spider.py](https://github.com/Fenghuapiao/PythonCrawler/blob/master/lagouPositionSpider.py): 任意输入关键字，一键抓取与关键字相关的职位招聘信息，并保存到本地文件
 
 ##### 6.  [student_img.py](https://github.com/Fenghuapiao/PythonCrawler/blob/master/student_img.py): 基于本学校官网的url漏洞，获取所有注册学生学籍证件照
 
-##### 7.  [JDSpider.py](https://github.com/Fenghuapiao/PythonCrawler/blob/master/JDSpider.py): 大批量抓取京东商品id和标签
+##### 7.  [JD_spider.py](https://github.com/Fenghuapiao/PythonCrawler/blob/master/JDSpider.py): 大批量抓取京东商品id和标签
 
 ##### 8.  [ECUT_pos_html.py](https://github.com/Fenghuapiao/PythonCrawler/blob/master/ECUT_pos_html.py): 抓取学校官网所有校园招聘信息，并保存为html格式，图片也会镶嵌在html中。
 
 ##### 9.  [ECUT_get_grade.py](https://github.com/Fenghuapiao/PythonCrawler/blob/master/ECUT_get_grade.py): 模拟登陆学校官网，抓取成绩并计算平均学分绩
 
-##### 10. [githubHot.py](https://github.com/Fenghuapiao/PythonCrawler/blob/master/githubHot.py): 抓取github上面热门语言所对应的项目，并把项目简介和项目主页地址保存到本地文件。
+##### 10. [github_hot.py](https://github.com/Fenghuapiao/PythonCrawler/blob/master/githubHot.py): 抓取github上面热门语言所对应的项目，并把项目简介和项目主页地址保存到本地文件。
 
-##### 11.[pictureSpider.py](https://github.com/Fenghuapiao/PythonCrawler/blob/master/pictureSpider.py): 应一位知友的请求，抓取某网站上面所有的写真图片。
+##### 11.[xz_picture_spider.py](https://github.com/Fenghuapiao/PythonCrawler/blob/master/pictureSpider.py): 应一位知友的请求，抓取某网站上面所有的写真图片。
+---
+# spiderAPI模块简介
+#### 本模块提供一些网站的API爬虫接口，功能可能不是很全因此可塑性很大智慧的你如果有兴趣可以继续改进。
+##### 1.大众点评
+```python
+from spiderAPI.dianping import *
+
+'''
+citys = {
+    '北京': '2', '上海': '1', '广州': '4', '深圳': '7', '成都': '8', '重庆': '9', '杭州': '3', '南京': '5', '沈阳': '18', '苏州': '6', '天津': '10','武汉': '16', '西安': '17', '长沙': '344', '大连': '19', '济南': '22', '宁波': '11', '青岛': '21', '无锡': '13', '厦门': '15', '郑州': '160'
+}
+
+ranktype = {
+    '最佳餐厅': 'score', '人气餐厅': 'popscore', '口味最佳': 'score1', '环境最佳': 'score2', '服务最佳': 'score3'
+}
+'''
+
+result=bestRestaurant(cityId=1, rankType='popscore')#获取人气餐厅
+
+shoplist=dpindex(cityId=1, page=1)#商户风云榜
+
+restaurantlist=restaurantList('http://www.dianping.com/search/category/2/10/p2')#获取餐厅
+
+```
+
+##### 2.获取代理IP
+爬取http://proxy.ipcn.org，获取可用代理
+```python
+from spiderAPI.proxyip import get_enableips
+
+enableips=get_enableips()
+
+```
+
+##### 3.百度地图
+百度地图提供的API,对查询有一些限制，这里找出了web上查询的接口
+```python
+from spiderAPI.baidumap import *
+
+citys=citys()#获取城市列表
+result=search(keyword="美食", citycode="257", page=1)#获取搜索结果
+
+```
+
+##### 4.模拟登录github
+```python
+from spiderAPI.github import GitHub
+
+github = GitHub()
+github.login() # 这一步会提示你输入用户名和密码
+github.show_timeline() # 获取github主页时间线
+# 更多的功能有待你们自己去发掘
+```
+
+##### 5.拉勾网
+```python
+from spiderAPI.lagou import *
+
+lagou_spider(key='数据挖掘', page=1) # 获取关键字为数据挖掘的招聘信息
+```

README.md

@@ -0,0 +1,30 @@
+import requests
+import json
+
+headers = {
+    'Host': "map.baidu.com",
+    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+    "Accept-Encoding": "gzip, deflate",
+    "Accept-Language": "en-US,en;q=0.5",
+    "Connection": "keep-alive",
+    "User-Agent": "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:39.0) Gecko/20100101 Firefox/39.0"}
+
+
+def citys():
+    html = requests.get(
+        'http://map.baidu.com/?newmap=1&reqflag=pcmap&biz=1&from=webmap&da_par=baidu&pcevaname=pc4.1&qt=s&da_src=searchBox.button&wd=美食&c=1&src=0&wd2=&sug=0&l=5&b=(7002451.220000001,1994587.88;19470675.22,7343963.88)&from=webmap&biz_forward={%22scaler%22:1,%22styles%22:%22pl%22}&sug_forward=&tn=B_NORMAL_MAP&nn=0&u_loc=12736591.152491,3547888.166124&ie=utf-8&t=1459951988807', headers=headers).text
+    data = json.loads(html)
+    result = []
+    for item in data['more_city']:
+        for city in item['city']:
+            result.append(city)
+    for item in data['content']:
+        result.append(item)
+    return result
+
+
+def search(keyword, citycode, page):
+    html = requests.get('http://map.baidu.com/?newmap=1&reqflag=pcmap&biz=1&from=webmap&da_par=baidu&pcevaname=pc4.1&qt=con&from=webmap&c=' + str(citycode) + '&wd=' + keyword + '&wd2=&pn=' + str(
+        page) + '&nn=' + str(page * 10) + '&db=0&sug=0&addr=0&&da_src=pcmappg.poi.page&on_gel=1&src=7&gr=3&l=12&tn=B_NORMAL_MAP&u_loc=12736591.152491,3547888.166124&ie=utf-8', headers=headers).text
+    data = json.loads(html)['content']
+    return data

baiduImg.py

@@ -0,0 +1,87 @@
+import requests
+import json
+import os
+from bs4 import BeautifulSoup
+
+headers = {
+    'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:39.0) Gecko/20100101 Firefox/39.0',
+    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+    'Accept-Language': 'en-US,en;q=0.5',
+    'Accept-Encoding': 'gzip, deflate',
+    'Connection': 'keep-alive'}
+
+
+def bestRestaurant(cityId=1, rankType='popscore'):
+    html = requests.get('http://www.dianping.com/mylist/ajax/shoprank?cityId=%s&shopType=10&rankType=%s&categoryId=0' %
+                        (cityId, rankType), headers=headers).text
+    result = json.loads(html)['shopBeans']
+    return result
+
+
+def getCityId():
+    citys = {'北京': '2', '上海': '1', '广州': '4', '深圳': '7', '成都': '8', '重庆': '9', '杭州': '3', '南京': '5', '沈阳': '18', '苏州': '6', '天津': '10',
+             '武汉': '16', '西安': '17', '长沙': '344', '大连': '19', '济南': '22', '宁波': '11', '青岛': '21', '无锡': '13', '厦门': '15', '郑州': '160'}
+    return citys
+
+
+def getRankType():
+    RankType = {'最佳餐厅': 'score', '人气餐厅': 'popscore',
+                '口味最佳': 'score1', '环境最佳': 'score2', '服务最佳': 'score3'}
+    return RankType
+
+
+def dpindex(cityId=1, page=1):
+    url = 'http://dpindex.dianping.com/dpindex?region=&category=&type=rank&city=%s&p=%s' % (
+        cityId, page)
+    html = requests.get(url, headers=headers).text
+    table = BeautifulSoup(html, 'lxml').find(
+        'div', attrs={'class': 'idxmain-subcontainer'}).find_all('li')
+    result = []
+    for item in table:
+        shop = {}
+        shop['name'] = item.find('div', attrs={'class': 'field-name'}).get_text()
+        shop['url'] = item.find('a').get('href')
+        shop['num'] = item.find('div', attrs={'class': 'field-num'}).get_text()
+        shop['addr'] = item.find('div', attrs={'class': 'field-addr'}).get_text()
+        shop['index'] = item.find('div', attrs={'class': 'field-index'}).get_text()
+        result.append(shop)
+    return result
+
+
+def restaurantList(url):
+    html = requests.get(url, headers=headers, timeout=30).text.replace('\r', '').replace('\n', '')
+    table = BeautifulSoup(html, 'lxml').find('div', id='shop-all-list').find_all('li')
+    result = []
+    for item in table:
+        shop = {}
+        soup = item.find('div', attrs={'class': 'txt'})
+        tit = soup.find('div', attrs={'class': 'tit'})
+        comment = soup.find('div', attrs={'class': 'comment'})
+        tag_addr = soup.find('div', attrs={'class': 'tag-addr'})
+        shop['name'] = tit.find('a').get_text()
+        shop['star'] = comment.find('span').get('title')
+        shop['review-num'] = comment.find('a',
+                                          attrs={'class': 'review-num'}).get_text().replace('条点评', '')
+        shop['mean-price'] = comment.find('a', attrs={'class': 'mean-price'}).get_text()
+        shop['type'] = tag_addr.find('span', attrs={'class': 'tag'}).get_text()
+        shop['addr'] = tag_addr.find('span', attrs={'class': 'addr'}).get_text()
+        try:
+            comment_list = soup.find('span', attrs={'class': 'comment-list'}).find_all('span')
+        except:
+            comment_list = []
+        score = []
+        for i in comment_list:
+            score.append(i.get_text())
+        shop['score'] = score
+        tags = []
+        try:
+            for i in tit.find('div', attrs={'class': 'promo-icon'}).find_all('a'):
+                try:
+                    tags += i.get('class')
+                except:
+                    tags.append(i.get('class')[0])
+        except:
+            pass
+        shop['tags'] = tags
+        result.append(shop)
+    return result