Update remove_words.py

encoding for Python 3 only.
davut1999 · Oct 11, 2018 · 546efef · 546efef
1 parent fb096ef
commit 546efef
Showing 1 changed file with 4 additions and 1 deletion.
diff --git a/remove_words.py b/remove_words.py
@@ -16,7 +16,10 @@
 dataset = '20ng'
 
 doc_content_list = []
-f = open('data/corpus/' + dataset + '.txt', 'r',encoding="utf8", errors='ignore')
+if sys.version_info > (3, 0):
+    f = open('data/corpus/' + dataset + '.txt', 'r', encoding = 'utf-8', errors='ignore')
+else:
+    f = open('data/corpus/' + dataset + '.txt', 'r')
 # f = open('data/wiki_long_abstracts_en_text.txt', 'r')
 for line in f.readlines():
     doc_content_list.append(line.strip())