refactor crawler

heart/test.py

@@ -0,0 +1,4 @@
+# -*- coding:utf-8 -*-
+import jieba
+
+print " ".join(jieba.cut(u"::	[AVI/329M][C0930-hitozuma1007] 坂井雪恵 Yukie Sakai  (308123)"))

pdf/crawler.py → pdf/crawler1.py

@@ -1,8 +1,9 @@
 # coding=utf-8
+import logging
 import os
 import re
 import time
-import logging
+
 import pdfkit
 import requests
 from bs4 import BeautifulSoup
@@ -21,6 +22,22 @@
 """
 
 
+class Crawler(object):
+    """
+    爬虫基类，所有爬虫都应该继承此类
+    """
+    name = None
+
+    def __init__(self, name=None, **kwargs):
+        if name is not None:
+            self.name = name
+        elif not getattr(self, 'name', None):
+            raise ValueError("%s must have a name" % type(self).__name__)
+
+    def parse(self):
+        raise NotImplementedError
+
+
 def parse_url_to_html(url, name):
     """
     解析URL，返回HTML内容
@@ -51,7 +68,8 @@ def func(m):
                 rtn = m.group(1) + "http://www.liaoxuefeng.com" + m.group(2) + m.group(3)
                 return rtn
             else:
-                return m.group(1)+m.group(2)+m.group(3)
+                return m.group(1) + m.group(2) + m.group(3)
+
         html = re.compile(pattern).sub(func, html)
         html = html_template.format(content=html)
         html = html.encode("utf-8")
@@ -120,4 +138,4 @@ def main():
 
 
 if __name__ == '__main__':
-    main()
+    Crawler().parse()

pdf/requirement.txt

@@ -0,0 +1,3 @@
+requests==2.12.4
+beautifulsoup4==4.5.3
+pdfkit==0.6.1