tokenize arrows

aajanki · Aug 23, 2023 · 2e0d63f · 2e0d63f
1 parent ded7d62
commit 2e0d63f
Showing 1 changed file with 3 additions and 2 deletions.
diff --git a/fi/fi.py b/fi/fi.py
@@ -21,17 +21,18 @@
 from spacy.lang.char_classes import LIST_HYPHENS, LIST_CURRENCY, CURRENCY, UNITS
 from spacy.lang.char_classes import ALPHA, ALPHA_LOWER, ALPHA_UPPER
 
-
 LIST_QUOTES = [x for x in LIST_QUOTES if x not in ['', ',']]
 LIST_QUOTES = LIST_QUOTES + ["‹", "›"]
 CONCAT_QUOTES = ''.join(LIST_QUOTES)
 
 LIST_PUNCT = LIST_PUNCT + ["•"]
 CONCAT_PUNCT = ''.join(LIST_PUNCT)
 
+LIST_ICONS = LIST_ICONS + ["←", "↑", "→", "↓"]
+
 _prefixes = (
     [
-        "§", "%", "=", "—", "–", r"\+(?![0-9])", "→",
+        "§", "%", "=", "—", "–", r"\+(?![0-9])",
         r"[/-](?=[{a}])".format(a=ALPHA)
     ]
     + LIST_PUNCT