specify correct encoding

defmacro-jam · Jan 5, 2021 · 74ce6da · 74ce6da
1 parent ae0e7fe
commit 74ce6da
Show file tree

Hide file tree

Showing 2 changed files with 3 additions and 3 deletions.
diff --git a/src/precomputed-tables.lisp b/src/precomputed-tables.lisp
@@ -9,7 +9,7 @@
 
 
 (defvar *unicode-data*
-  (with-open-file (in (uiop:merge-pathnames* *data-directory* "UnicodeData.txt") :external-format :ISO-8859-1)
+  (with-open-file (in (uiop:merge-pathnames* *data-directory* "UnicodeData.txt") :external-format :UTF-8)
     (loop for line = (read-line in nil nil)
        while line
        collect (cl-ppcre:split ";" line))))
@@ -41,7 +41,7 @@
 
 
 (defparameter *composition-exclusions-data* (make-hash-table))
-(with-open-file (in (uiop:merge-pathnames* *data-directory* "CompositionExclusions.txt" :external-format :ISO-8859-1))
+(with-open-file (in (uiop:merge-pathnames* *data-directory* "CompositionExclusions.txt" :external-format :UTF-8))
   (loop for line = (read-line in nil nil)
      while line
        when (and (plusp (length line))

diff --git a/src/uax-15.lisp b/src/uax-15.lisp
@@ -67,7 +67,7 @@
              (loop for code from (first range) to (or (second range) (first range))
                    for char = code
                collect (list char maybe?)))))
-    (with-open-file (in *derived-normalization-props-data-file* :external-format :ISO-8859-1)
+    (with-open-file (in *derived-normalization-props-data-file* :external-format :UTF-8)
       (loop for line = (read-line in nil nil)
             while line
         do