add heart

chashisu · Feb 15, 2017 · c985586 · c985586
1 parent e95494b
commit c985586
Show file tree

Hide file tree

Showing 33 changed files with 940 additions and 16,751 deletions.
diff --git a/agone-Heart.png b/agone-Heart.png
diff --git a/blog/__init__.py b/blog/__init__.py
diff --git a/crawler_jobbole.py → blog/crawler_blog.py b/crawler_jobbole.py → blog/crawler_blog.py
diff --git a/crawler_jobbole_async.py → blog/crawler_blog_async.py b/crawler_jobbole_async.py → blog/crawler_blog_async.py
diff --git a/heart/__init__.py b/heart/__init__.py
diff --git a/HeatherT-heart-vine-mask.jpg → heart/heart-mask.jpg b/HeatherT-heart-vine-mask.jpg → heart/heart-mask.jpg
diff --git a/heart/heart.jpg b/heart/heart.jpg
diff --git a/heart/heart.py b/heart/heart.py
@@ -0,0 +1,87 @@
+# -*- coding:utf-8 -*-
+import codecs
+import csv
+import re
+
+import jieba.analyse
+import matplotlib.pyplot as plt
+import requests
+from scipy.misc import imread
+from wordcloud import WordCloud
+
+__author__ = 'liuzhijun'
+
+cookies = {
+    "ALF": "xxxx",
+    "SCF": "xxxxxx.",
+    "SUBP": "xxxxx",
+    "SUB": "xxxx",
+    "SUHB": "xxx-", "xx": "xx", "_T_WM": "xxx",
+    "gsScrollPos": "", "H5_INDEX": "0_my", "H5_INDEX_TITLE": "xxx",
+    "M_WEIBOCN_PARAMS": "xxxx"
+}
+
+
+def fetch_weibo():
+    api = "http://m.weibo.cn/index/my?format=cards&page=%s"
+    for i in range(1, 102):
+        response = requests.get(url=api % i, cookies=cookies)
+        data = response.json()[0]
+        groups = data.get("card_group") or []
+        for group in groups:
+            text = group.get("mblog").get("text")
+            text = text.encode("utf-8")
+
+            def cleanring(content):
+                """
+                去掉无用字符
+                """
+                pattern = "<a .*?/a>|<i .*?/i>|转发微博|//:|Repost|，|？|。|、|分享图片"
+                content = re.sub(pattern, "", content)
+                return content
+
+            text = cleanring(text).strip()
+            if text:
+                yield text
+
+
+def write_csv(texts):
+    with codecs.open('./weibo.csv', 'w') as f:
+        writer = csv.DictWriter(f, fieldnames=["text"])
+        writer.writeheader()
+        for text in texts:
+            writer.writerow({"text": text})
+
+
+def read_csv():
+    with codecs.open('./weibo.csv', 'r') as f:
+        reader = csv.DictReader(f)
+        for row in reader:
+            yield row['text']
+
+
+def word_segment(texts):
+    jieba.analyse.set_stop_words("./stopwords.txt")
+    for text in texts:
+        tags = jieba.analyse.extract_tags(text, topK=20)
+        yield " ".join(tags)
+
+
+def generate_img(texts):
+    data = " ".join(text for text in texts)
+
+    mask_img = imread('./heart-mask.jpg', flatten=True)
+    wordcloud = WordCloud(
+        font_path='msyh.ttc',
+        background_color='white',
+        mask=mask_img
+    ).generate(data)
+    plt.imshow(wordcloud)
+    plt.axis('off')
+    plt.savefig('./heart.jpg', dpi=600)
+
+
+if __name__ == '__main__':
+    texts = fetch_weibo()
+    write_csv(texts)
+    generate_img(word_segment(read_csv()))
diff --git a/stop_word.txt → heart/stopwords.txt b/stop_word.txt → heart/stopwords.txt
diff --git a/heart/weibo.csv b/heart/weibo.csv
diff --git a/my_twitter_wordcloud_1.png b/my_twitter_wordcloud_1.png
diff --git a/my_twitter_wordcloud_10.png b/my_twitter_wordcloud_10.png
diff --git a/my_twitter_wordcloud_11.png b/my_twitter_wordcloud_11.png
diff --git a/my_twitter_wordcloud_12.png b/my_twitter_wordcloud_12.png
diff --git a/my_twitter_wordcloud_13.png b/my_twitter_wordcloud_13.png
diff --git a/my_twitter_wordcloud_2.png b/my_twitter_wordcloud_2.png
diff --git a/my_twitter_wordcloud_3.png b/my_twitter_wordcloud_3.png
diff --git a/my_twitter_wordcloud_4.png b/my_twitter_wordcloud_4.png
diff --git a/my_twitter_wordcloud_5.png b/my_twitter_wordcloud_5.png
diff --git a/pdf/__init__.py b/pdf/__init__.py
diff --git a/crawer-pdf.png → pdf/crawer-pdf.png b/crawer-pdf.png → pdf/crawer-pdf.png
diff --git a/crawler.py → pdf/crawler.py b/crawler.py → pdf/crawler.py
diff --git a/liaoxuefeng_Python3_tutorial.pdf → pdf/liaoxuefeng_Python3_tutorial.pdf b/liaoxuefeng_Python3_tutorial.pdf → pdf/liaoxuefeng_Python3_tutorial.pdf
diff --git a/test.html b/test.html
diff --git a/test2.html b/test2.html
diff --git a/test3.html b/test3.html
diff --git a/test4.html b/test4.html
diff --git a/twitter_mask.png b/twitter_mask.png
diff --git a/weibo.csv b/weibo.csv
diff --git a/weibo.py b/weibo.py