first commit

Author: yhangf

Crawler_Exercise/GetPhotos2.py

@@ -0,0 +1,24 @@
+import requests
+from bs4 import BeautifulSoup
+
+url = 'http://tieba.baidu.com/p/4178314700'
+
+def GetHtml(url):
+    html = requests.get(url).text
+    return html
+
+def GetImg(html):
+    soup = BeautifulSoup(html, 'html.parser')
+    imglist = []
+    for photourl in soup.find_all('img'):
+        imglist.append(photourl.get('src'))
+    x = 0
+    for imgurl in imglist:
+        with open('E:/Pic/%s.jpg' % x, 'wb') as file:
+            file.write(requests.get(imgurl).content)
+            x += 1
+            
+if __name__ == '__main__':    
+    html = GetHtml(url)
+    GetImg(html)
+        

README.md renamed to Crawler_Exercise/README.md

@@ -0,0 +1,70 @@
+# -*- coding:utf-8 -*-
+import sys,os,pdb,re,time,random
+from spider import SpiderHTML
+from bs4 import BeautifulSoup
+
+class QiubaiSpider(SpiderHTML):
+    def __init__(self,contentType,pageStart=1, pageEnd=1):
+        #super.__init__(self)
+        self._contentType = contentType
+        self._pageStart = int(pageStart)
+        self._pageEnd = int(pageEnd)+1
+        self.__url = {'new':'http://www.qiushibaike.com/textnew/page/','hot':'http://www.qiushibaike.com/text/page/'}
+
+    def getJokes(self):
+        reqUrl = ''
+
+        if contentType in self.__url:
+            reqUrl = self.__url[self._contentType]
+        else:
+            reqUrl = self.__url['new']
+        for i in range(self._pageStart,self._pageEnd):
+            pageUrl = reqUrl+str(i)+'/'
+            jokes = self.getUrl(pageUrl)
+            jokes = jokes.find_all('div',id=re.compile('qiushi_tag_\d+'))
+            filepath = os.path.join('E:\\','qiubai','page_'+self._contentType+str(i))
+            info = '正在保存第{page}页的糗事到文件 {file}.txt'
+            print(info.format(page=i,file=filepath))
+            for joke in jokes:
+                jokeContent = str(joke.find('div',attrs={'class':'content'}))
+                jokeContent = re.sub('<div class="content">','',jokeContent)
+                jokeContent = re.sub('</div>','',jokeContent)
+                jokeContent = re.sub('<!--\d+-->','',jokeContent)
+                jokeContent = re.sub('<br>','\n',jokeContent)
+                jokeContent = re.sub('<br/>','\n',jokeContent)
+                try:
+                    author = joke.find(attrs={'class':'author clearfix'}).find('h2').string
+                    upvote = joke.find(attrs={'class':'stats'}).span.i.string
+                except AttributeError:
+                    pass
+
+                joke = '-----------------------------\r\n作者：{author}\r\n{joke}\r\n\r\n{upvote}人觉得很赞\r\n'.format(joke=jokeContent.strip(),author=author,upvote=upvote)
+                
+                self.saveText(filepath+'.txt',joke,'a')
+            if i%2 == 0:        #防止被封，间隔时间长一点
+                time.sleep(random.random()*3)
+
+if __name__ == '__main__':
+    contentType = 'new'
+    page = 5
+    paramsNum = len(sys.argv)
+
+    #输入想获取最新的糗百还是最热的糗百
+    #参数2,3为想要获取的页数
+    if paramsNum>=4:
+        contentType = sys.argv[1]
+        page = sys.argv[2]
+        pageEnd = sys.argv[3]
+    elif paramsNum>=3:
+        contentType = sys.argv[1]
+        page = sys.argv[2]
+        pageEnd = page
+    elif paramsNum == 2:
+        contentType = sys.argv[1]
+        page,pageEnd = 1,1
+    else:
+        contentType = 'new'
+        page,pageEnd = 1,1
+
+    qiubai = QiubaiSpider(contentType,page,pageEnd)
+    qiubai.getJokes()

Crawler_Exercise/WebSpider/__pycache__/spider.cpython-35.pyc

@@ -0,0 +1,33 @@
+# -*- coding: utf-8 -*-
+__author__ = 'yhf'
+import os,re,codecs,urllib
+from urllib import request
+from bs4 import BeautifulSoup
+
+class SpiderHTML(object):
+    #打开页面
+    def getUrl(self, url, coding='utf-8'):
+        req = request.Request(url)
+        req.add_header('User-Agent','Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.157 UBrowser/5.5.9703.2 Safari/537.36')
+        with request.urlopen(req) as response:
+            return BeautifulSoup(response.read().decode(coding),'html.parser')
+
+    #保存文本内容到本地
+    def saveText(self,filename,content,mode='w'):
+        self._checkPath(filename)
+        with codecs.open(filename, encoding='utf-8', mode=mode) as f:
+            f.write(content)
+            
+        
+    #保存图片
+    def saveImg(self, imgUrl, imgName):
+        data=urllib.request.urlopen(imgUrl).read()
+        self._checkPath(imgName)
+        with open(imgName,'wb') as f:
+            f.write(data)
+
+    #创建目录
+    def _checkPath(self, path):
+        dirname = os.path.dirname(path.strip())
+        if not os.path.exists(dirname):
+            os.makedirs(dirname)

Crawler_Exercise/WebSpider/qiubai.py

@@ -0,0 +1,81 @@
+from spider import SpiderHTML
+import re,os,sys,time,urllib,random,http
+'''
+抓取淘宝模特的靓图
+'''
+
+class TaobaommSpider(SpiderHTML):
+    #抓取起始页，结束页，每个妹子抓取的图片数量
+    def __init__(self,pageStart, pageEnd,limit_img):
+        self._pageStart = int(pageStart)
+        self._pageEnd = int(pageEnd)+1
+        self._limit = limit_img
+        self.__url = 'https://mm.taobao.com/json/request_top_list.htm?page='
+        self.__dir = r'E:\taobaomm'
+
+    def start(self):
+        for page in range(self._pageStart,self._pageEnd):
+            url = self.__url + str(page)
+            contents = self.getUrl(url,'gbk')
+            lists = contents.find_all('div',class_='personal-info')
+            for girl in lists:
+                info = girl.find('a',attrs={'class':'lady-name'})
+                avatar = girl.find('a',class_='lady-avatar')
+
+                girlinfo = {}
+                girlinfo['name'] = info.string  
+                girlinfo['age'] = info.find_next_sibling('em').strong.string  
+                girlinfo['city'] = info.find_next('span').string
+                girlinfo['url'] = 'https:'+avatar['href']
+                #去除掉缩小的图片
+                girlinfo['avatar'] = 'https:'+re.sub('_\d+x\d+\.\w+$','',avatar.img['src']) 
+                imgType = os.path.splitext(girlinfo['avatar'])[1]
+                logInfo = '找到一位MM：{name},{age}岁，她在{city}'.format(**girlinfo)
+                print(logInfo)
+                tmpDir = os.path.join(self.__dir,girlinfo['name']+'-'+girlinfo['age']+'-'+girlinfo['city'])
+                if(os.path.exists(tmpDir)):
+                    print('已经获得过信息，去找下一位')
+                    continue
+                #以名字命名，保存图片和基本信息
+                self.saveImg(girlinfo['avatar'],os.path.join(tmpDir,'avatar'+imgType))
+                print('正在进入她的个人中心获取私图')
+
+                gilrsCenter = self.getUrl(girlinfo['url'],'gbk')
+                imgs = gilrsCenter.find('div',class_='mm-aixiu-content').find_all('img')
+                i = 0
+                for img in imgs:
+                    i = i + 1
+                    if i % 5 == 0:
+                        print('正在获取第{i}张图'.format(i=i))
+                    try:
+                        imgurl = 'https:'+img['src']
+                        extend_name = os.path.splitext(img['src'])[1]
+                        if extend_name == '.gif':
+                            continue    #一般都是表情图，略过
+                        self.saveImg(imgurl,os.path.join(tmpDir,str(i)+extend_name))
+                    except urllib.error.HTTPError as e:
+                        pass
+                    except KeyError as e:
+                        pass
+                    except http.client.IncompleteRead:
+                        pass
+
+                    if i >= self._limit:
+                        pass    #若要限制每个模特抓图的张数，此处改为break
+                time.sleep(random.random()*2)
+
+
+if __name__ == '__main__':
+    page, limit, paramsNum= 1, 0, len(sys.argv)
+    if paramsNum>=4:
+        page, pageEnd, limit = sys.argv[1], sys.argv[2], int(sys.argv[3])
+    elif paramsNum == 2:
+        page = sys.argv[1]
+        pageEnd = page
+    else:
+        page,pageEnd = 1,1
+
+    if limit <5:
+        limit = 20
+    spider = TaobaommSpider(page,pageEnd,limit)
+    spider.start()

Crawler_Exercise/WebSpider/spider.py

@@ -0,0 +1,122 @@
+# -*- coding:utf-8 -*-
+
+from spider import SpiderHTML
+import sys,urllib,http,os,random,re,time
+__author__ = 'waiting'
+'''
+使用了第三方的类库 BeautifulSoup4，请自行安装
+需要目录下的spider.py文件
+运行环境：python3.4,windows7
+'''
+
+#收藏夹的地址
+url = 'https://www.zhihu.com/collection/69135664'  #page参数改为代码添加
+
+#本地存放的路径,不存在会自动创建
+store_path = 'E:\\zhihu\收藏夹\\攻不可破的大美妞阵线联盟'
+
+class zhihuCollectionSpider(SpiderHTML):
+    def __init__(self,pageStart, pageEnd, url):
+        self._url = url
+        self._pageStart = int(pageStart)
+        self._pageEnd = int(pageEnd)+1
+        self.downLimit = 0                        #低于此赞同的答案不收录
+
+    def start(self):
+        for page in range(self._pageStart,self._pageEnd):        #收藏夹的页数
+            url = self._url + '?page='+str(page)
+            content = self.getUrl(url)
+            questionList = content.find_all('div',class_='zm-item')
+            for question in questionList:                        #收藏夹的每个问题
+                Qtitle = question.find('h2',class_='zm-item-title')
+                if Qtitle is None:                                #被和谐了
+                    continue
+
+                questionStr = Qtitle.a.string
+                Qurl = 'https://www.zhihu.com'+Qtitle.a['href']    #问题题目
+                Qtitle = re.sub(r'[\\/:*?"<>]','#',Qtitle.a.string)            #windows文件/目录名不支持的特殊符号
+                print('-----正在获取问题:'+Qtitle+'-----')        #获取到问题的链接和标题，进入抓取
+                Qcontent = self.getUrl(Qurl)
+                answerList = Qcontent.find_all('div',class_='zm-item-answer  zm-item-expanded')
+                self._processAnswer(answerList,Qtitle)                        #处理问题的答案
+                time.sleep(5)
+
+
+    def _processAnswer(self,answerList,Qtitle):
+        j = 0            
+        for answer in answerList:
+            j = j + 1
+            
+            upvoted = int(answer.find('span',class_='count').string.replace('K','000'))     #获得此答案赞同数
+            if upvoted < 100:
+                continue
+            authorInfo = answer.find('div',class_='zm-item-answer-author-info')                #获取作者信息
+            author = {'introduction':'','link':''}
+            try:
+                author['name'] = authorInfo.find('a',class_='author-link').string             #获得作者的名字
+                author['introduction'] = str(authorInfo.find('span',class_='bio')['title']) #获得作者的简介
+            except AttributeError:
+                author['name'] = '匿名用户'+str(j)
+            except TypeError:                                                                  #简介为空的情况
+                pass
+    
+            try:
+                author['link'] = authorInfo.find('a',class_='author-link')['href']
+            except TypeError:                                                                  #匿名用户没有链接
+                pass
+    
+            file_name = os.path.join(store_path,Qtitle,'info',author['name']+'_info.txt')
+            if os.path.exists(file_name):                            #已经抓取过
+                continue
+    
+            self.saveText(file_name,'{introduction}\r\n{link}'.format(**author))            #保存作者的信息
+            print('正在获取用户`{name}`的答案'.format(**author))
+            answerContent = answer.find('div',class_='zm-editable-content clearfix')
+            if answerContent is None:                                #被举报的用户没有答案内容
+                continue
+    
+            imgs = answerContent.find_all('img')
+            if len(imgs) == 0:                                        #答案没有上图
+                pass
+            else:
+                self._getImgFromAnswer(imgs,Qtitle,**author)
+
+
+    #收录图片
+    def _getImgFromAnswer(self,imgs,Qtitle,**author):
+        i = 0
+        for img in imgs:
+            if 'inline-image' in img['class']:                    #不抓取知乎的小图
+                continue
+            i = i + 1
+            imgUrl = img['src']
+            extension = os.path.splitext(imgUrl)[1]
+            path_name = os.path.join(store_path,Qtitle,author['name']+'_'+str(i)+extension)
+            try:
+                self.saveImg(imgUrl,path_name)                    #捕获各种图片异常，流程不中断
+            except ValueError:                                    
+                pass
+            except urllib.error.HTTPError as e:
+                pass
+            except KeyError as e:
+                pass
+            except http.client.IncompleteRead:
+                pass
+    #收录文字
+    def _getTextFromAnswer(self):
+        pass
+
+#例：zhihu.py 1 5   获取1到5页的数据
+if __name__ == '__main__':
+    page, limit, paramsNum= 1, 0, len(sys.argv)
+    if paramsNum>=3:
+        page, pageEnd = sys.argv[1], sys.argv[2]
+    elif paramsNum == 2:
+        page = sys.argv[1]
+        pageEnd = page
+    else:
+        page,pageEnd = 1,1
+
+    spider = zhihuCollectionSpider(page,pageEnd,url)
+    spider.start()
+

Crawler_Exercise/WebSpider/taobaomm.py

@@ -0,0 +1,53 @@
+import requests
+import re
+
+url = 'http://image.baidu.com/search/index'
+headers = {
+                'User-Agent' : 'Mozilla/5.0 (Windows NT 10.0; WOW64; rv:44.0) Gecko/20100101 Firefox/44.0',
+                'Accept-Language' : 'zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3',
+                'Accept-Encoding' : 'gzip, deflate',
+                'Referer' : 'http://image.baidu.com/search/index?tn=baiduimage&ipn=r&ct=201326592&cl=2&fm=detail&lm=-1&st=-1&sf=2&fmq=&pv=&ic=0&nc=1&z=&se=&showtab=0&fb=0&width=&height=&face=0&istype=2&ie=utf-8&word=%E9%AB%98%E6%B8%85%E6%91%84%E5%BD%B1&oq=%E9%AB%98%E6%B8%85%E6%91%84%E5%BD%B1&rsp=-1',
+                'Cookie' : 'HOSUPPORT=1; UBI=fi_PncwhpxZ%7ETaMMzY0i9qXJ9ATcu3rvxFIc-a7KI9byBcYk%7EjBVmPGIbL3LTKKJ2D17mh5VfJ5yjlCncAb2yhPI5sZM51Qo7tpCemygM0VNUzuTBJwYF8OYmi3nsCCzbpo5U9tLSzkZfcQ1rxUcJSzaipThg__; HISTORY=fec845b215cd8e8be424cf320de232722d0050; PTOKEN=ff58b208cc3c16596889e0a20833991d; STOKEN=1b1f4b028b5a4415aa1dd9794ff061d312ad2a822d52418f3f1ffabbc0ac6142; SAVEUSERID=0868a2b4c9d166dc85e605f0dfd153; USERNAMETYPE=3; PSTM=1454309602; BAIDUID=E5493FD55CFE5424BA25B1996943B3B6:FG=1; BIDUPSID=B7D6D9EFA208B7B8C7CB6EF8F827BD4E; BDUSS=VSeFB6UXBmRWc3UEdFeXhKOFRvQm4ySmVmTkVEN2N0bldnM2o5RHdyaE54ZDlXQVFBQUFBJCQAAAAAAAAAAAEAAABzhCtU3Mbj5cfl0e8AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAE04uFZNOLhWZW; H_PS_PSSID=1447_18282_17946_18205_18559_17001_17073_15479_12166_18086_10634; BDRCVFR[feWj1Vr5u3D]=I67x6TjHwwYf0; BDRCVFR[X_XKQks0S63]=mk3SLVN4HKm; BDRCVFR[dG2JNJb_ajR]=mk3SLVN4HKm',
+               }
+def get_html(url, headers):
+    data = {
+              'cl' : '2',
+              'ct' : '201326592',
+              'face' : '0',
+              'fp' : 'result',
+              'gsm' : '200001e',
+              'ic' : '0',
+              'ie' : 'utf-8',
+              'ipn' : 'rj',
+              'istype' : '2',
+              'lm' : '-1',
+              'nc' : '1',
+              'oe' : 'utf-8',
+              'pn' : '30',
+              'queryword' : '高清摄影',
+              'rn' : '30',
+              'st' : '-1',
+              'tn' : 'resultjson_com',
+              'word' : '高清摄影'
+              }
+    
+    page = requests.get(url, data, headers = headers).text
+    return page
+
+def get_img(page, headers):
+#     img_url_list = []
+    reg = re.compile('http://.*?\.jpg')
+    imglist1 = re.findall(reg, page)
+    imglist2 = imglist1[0 : len(imglist1) : 3]
+#     [img_url_list.append(i) for i in imglist if not i in img_url_list]
+    x = 0
+    for imgurl in imglist2:
+        bin = requests.get(imgurl, headers = headers).content
+        with open('E:/Pic2/%s.jpg' % x, 'wb') as file:
+            file.write(bin)
+            x += 1
+
+if __name__ == '__main__':
+    page = get_html(url, headers)
+    get_img(page, headers)
+