Better handling of newlines in data

chetan51 · Sep 12, 2013 · 55d1896 · 55d1896
1 parent 228ce62
commit 55d1896
Show file tree

Hide file tree

Showing 2 changed files with 5 additions and 2 deletions.
diff --git a/client/linguist.py b/client/linguist.py
@@ -81,8 +81,8 @@ def runLinguist(datapath):
         c = f.read(1)
         if not c: break
 
-        if ord(c) <= 31 and ord(c) >= 127: continue
-        if last_c == ' ' and c == last_c: continue
+        if ord(c) <= 31 and ord(c) >= 127 and ord(c) != 10: continue
+        if (last_c == ' ' or last_c == '\n') and c == last_c: continue
 
         last_c = c
 

diff --git a/tools/txt_to_csv.py b/tools/txt_to_csv.py
@@ -13,6 +13,7 @@
 def convert(datapath):
     data = file(datapath).read()
     data = re.sub(' +', ' ', data)
+    data = re.sub('\n+', '\n', data)
     N = len(data)
 
     print "letter"
@@ -23,6 +24,8 @@ def convert(datapath):
         c = data[i]
         if ord(c) > 31 and ord(c) < 127:
             print c
+        if ord(c) == 10:
+            print '|'
 
 if __name__ == "__main__":
     if len(sys.argv) > 1: