Made the large corpus even larger

sshekh · May 25, 2017 · d09af8a · d09af8a
1 parent 8bf495e
commit d09af8a
Show file tree

Hide file tree

Showing 5 changed files with 391,869 additions and 96,815 deletions.
diff --git a/europarl_data.py b/europarl_data.py
@@ -2,6 +2,7 @@
 import nltk
 from unidecode import unidecode
 from collections import Counter
+import sys
 
 def filter_sentence(s):
     words = nltk.tokenize.word_tokenize(s)
@@ -52,4 +53,7 @@ def build(europath):
     for fname in os.listdir(europath):
         docs.append(create_doc(europath + '/' + fname, word_to_id))
 
-    write_all(id_to_word, docs)
+    write_all(id_to_word, docs)
+
+if __name__ == '__main__':
+    build(sys.argv[1])
diff --git a/fast-lda/europarl_fi/europarl_fi.dat b/fast-lda/europarl_fi/europarl_fi.dat