Better ($LANG-independent) fix for csv translation loading on py3.

MANIFEST.in

@@ -4,6 +4,7 @@ include tornado/ca-certificates.crt
 include tornado/test/README
 include tornado/test/test.crt
 include tornado/test/test.key
+include tornado/test/csv_translations/fr_FR.csv
 include tornado/test/static/robots.txt
 include tornado/test/templates/utf8.html
 global-exclude _auto2to3*

setup.py

@@ -45,8 +45,9 @@
     packages = ["tornado", "tornado.test", "tornado.platform"],
     package_data = {
         "tornado": ["ca-certificates.crt"],
+        # data files need to be listed both here and in MANIFEST.in
         "tornado.test": ["README", "test.crt", "test.key", "static/robots.txt",
-                         "templates/utf8.html"],
+                         "templates/utf8.html", "csv_translations/fr_FR.csv"],
         },
     ext_modules = extensions,
     author="Facebook",

tornado/locale.py

@@ -121,7 +121,16 @@ def load_translations(directory):
             logging.error("Unrecognized locale %r (path: %s)", locale,
                           os.path.join(directory, path))
             continue
-        f = open(os.path.join(directory, path), "r")
+        full_path = os.path.join(directory, path)
+        try:
+            # python 3: csv.reader requires a file open in text mode.
+            # Force utf8 to avoid dependence on $LANG environment variable.
+            f = open(full_path, "r", encoding="utf-8")
+        except TypeError:
+            # python 2: files return byte strings, which are decoded below.
+            # Once we drop python 2.5, this could use io.open instead
+            # on both 2 and 3.
+            f = open(full_path, "r")
         _translations[locale] = {}
         for i, row in enumerate(csv.reader(f)):
             if not row or len(row) < 2:
@@ -139,7 +148,7 @@ def load_translations(directory):
             _translations[locale].setdefault(plural, {})[english] = translation
         f.close()
     _supported_locales = frozenset(_translations.keys() + [_default_locale])
-    logging.info("Supported locales: %s", sorted(_supported_locales))
+    logging.debug("Supported locales: %s", sorted(_supported_locales))
 
 
 def load_gettext_translations(directory, domain):

tornado/test/csv_translations/fr_FR.csv

@@ -0,0 +1 @@
+"school","école"

tornado/test/locale_test.py

@@ -0,0 +1,24 @@
+from __future__ import absolute_import, division, with_statement
+
+import os
+import tornado.locale
+import unittest
+
+class TranslationLoaderTest(unittest.TestCase):
+    # TODO: less hacky way to get isolated tests
+    SAVE_VARS = ['_translations', '_supported_locales', '_use_gettext']
+
+    def setUp(self):
+        self.saved = {}
+        for var in TranslationLoaderTest.SAVE_VARS:
+            self.saved[var] = getattr(tornado.locale, var)
+
+    def tearDown(self):
+        for k, v in self.saved.items():
+            setattr(tornado.locale, k, v)
+
+    def test_csv(self):
+        tornado.locale.load_translations(
+            os.path.join(os.path.dirname(__file__), 'csv_translations'))
+        locale = tornado.locale.get("fr_FR")
+        self.assertEqual(locale.translate("school"), u"\u00e9cole")

tornado/test/runtests.py

@@ -17,6 +17,7 @@
     'tornado.test.import_test',
     'tornado.test.ioloop_test',
     'tornado.test.iostream_test',
+    'tornado.test.locale_test',
     'tornado.test.options_test',
     'tornado.test.process_test',
     'tornado.test.simple_httpclient_test',