declaring content coding

tornado/httpclient.py

@@ -512,19 +512,19 @@ def __init__(self, url, method="GET", headers={}, body=None,
                  max_redirects=5, user_agent=None, use_gzip=True,
                  network_interface=None, streaming_callback=None,
                  header_callback=None, prepare_curl_callback=None,
-                 allow_nonstandard_methods=False):
+                 allow_nonstandard_methods=False,encoding='utf-8'):
         if if_modified_since:
             timestamp = calendar.timegm(if_modified_since.utctimetuple())
             headers["If-Modified-Since"] = email.utils.formatdate(
                 timestamp, localtime=False, usegmt=True)
         if "Pragma" not in headers:
             headers["Pragma"] = ""
-        self.url = _utf8(url)
+        self.url = _encode(url,codec=encoding)
         self.method = method
         self.headers = headers
         self.body = body
-        self.auth_username = _utf8(auth_username)
-        self.auth_password = _utf8(auth_password)
+        self.auth_username = _encode(auth_username,codec=encoding)
+        self.auth_password = _encode(auth_password,codec=encoding)
         self.connect_timeout = connect_timeout
         self.request_timeout = request_timeout
         self.follow_redirects = follow_redirects
@@ -616,10 +616,10 @@ def _curl_create(max_simultaneous_connections=None):
     return curl
 
 
-def _curl_setup_request(curl, request, buffer, headers):
+def _curl_setup_request(curl, request, buffer, headers,codec='utf-8'):
     curl.setopt(pycurl.URL, request.url)
     curl.setopt(pycurl.HTTPHEADER,
-                [_utf8("%s: %s" % i) for i in request.headers.iteritems()])
+                [_encode("%s: %s" % i,codec=codec) for i in request.headers.iteritems()])
     if request.header_callback:
         curl.setopt(pycurl.HEADERFUNCTION, request.header_callback)
     else:
@@ -634,7 +634,7 @@ def _curl_setup_request(curl, request, buffer, headers):
     curl.setopt(pycurl.CONNECTTIMEOUT, int(request.connect_timeout))
     curl.setopt(pycurl.TIMEOUT, int(request.request_timeout))
     if request.user_agent:
-        curl.setopt(pycurl.USERAGENT, _utf8(request.user_agent))
+        curl.setopt(pycurl.USERAGENT, _encode(request.user_agent,codec=codec))
     else:
         curl.setopt(pycurl.USERAGENT, "Mozilla/5.0 (compatible; pycurl)")
     if request.network_interface:
@@ -718,10 +718,10 @@ def _curl_debug(debug_type, debug_msg):
         logging.debug('%s %r', debug_types[debug_type], debug_msg)
 
 
-def _utf8(value):
+def _encode(value,codec="utf-8"):
     if value is None:
         return value
     if isinstance(value, unicode):
-        return value.encode("utf-8")
+        return value.encode(codec)
     assert isinstance(value, str)
     return value

tornado/web.py

@@ -79,9 +79,10 @@ class RequestHandler(object):
     """
     SUPPORTED_METHODS = ("GET", "HEAD", "POST", "DELETE", "PUT")
 
-    def __init__(self, application, request, transforms=None):
+    def __init__(self, application, request, transforms=None,coding='utf-8'):
         self.application = application
         self.request = request
+        self.coding=coding
         self._headers_written = False
         self._finished = False
         self._auto_finish = True
@@ -167,7 +168,7 @@ def set_header(self, name, value):
         elif isinstance(value, int) or isinstance(value, long):
             value = str(value)
         else:
-            value = _utf8(value)
+            value = _encode(value,codec=self.coding)
             # If \n is allowed into the header, it is possible to inject
             # additional headers or split the request. Also cap length to
             # prevent obviously erroneous values.
@@ -205,7 +206,7 @@ def get_arguments(self, name, strip=True):
         values = self.request.arguments.get(name, [])
         # Get rid of any weird control chars
         values = [re.sub(r"[\x00-\x08\x0e-\x1f]", " ", x) for x in values]
-        values = [_unicode(x) for x in values]
+        values = [_unicode(x,codec=self.coding) for x in values]
         if strip:
             values = [x.strip() for x in values]
         return values
@@ -238,8 +239,8 @@ def set_cookie(self, name, value, domain=None, expires=None, path="/",
         See http://docs.python.org/library/cookie.html#morsel-objects
         for available attributes.
         """
-        name = _utf8(name)
-        value = _utf8(value)
+        name = _encode(name,codec=self.coding)
+        value = _encode(value,codec=self.coding)
         if re.search(r"[\x00-\x20]", name + value):
             # Don't let us accidentally inject bad stuff
             raise ValueError("Invalid cookie %r: %r" % (name, value))
@@ -331,7 +332,7 @@ def redirect(self, url, permanent=False):
             raise Exception("Cannot redirect after headers have been written")
         self.set_status(301 if permanent else 302)
         # Remove whitespace
-        url = re.sub(r"[\x00-\x20]+", "", _utf8(url))
+        url = re.sub(r"[\x00-\x20]+", "", _encode(url,codec=self.coding))
         self.set_header("Location", urlparse.urljoin(self.request.uri, url))
         self.finish()
 
@@ -347,7 +348,7 @@ def write(self, chunk):
         if isinstance(chunk, dict):
             chunk = escape.json_encode(chunk)
             self.set_header("Content-Type", "text/javascript; charset=UTF-8")
-        chunk = _utf8(chunk)
+        chunk = _encode(chunk,codec=self.coding)
         self._write_buffer.append(chunk)
 
     def render(self, template_name, **kwargs):
@@ -363,25 +364,25 @@ def render(self, template_name, **kwargs):
         html_bodies = []
         for module in getattr(self, "_active_modules", {}).itervalues():
             embed_part = module.embedded_javascript()
-            if embed_part: js_embed.append(_utf8(embed_part))
+            if embed_part: js_embed.append(_encode(embed_part,codec=self.coding))
             file_part = module.javascript_files()
             if file_part:
                 if isinstance(file_part, basestring):
                     js_files.append(file_part)
                 else:
                     js_files.extend(file_part)
             embed_part = module.embedded_css()
-            if embed_part: css_embed.append(_utf8(embed_part))
+            if embed_part: css_embed.append(_encode(embed_part,codec=self.coding))
             file_part = module.css_files()
             if file_part:
                 if isinstance(file_part, basestring):
                     css_files.append(file_part)
                 else:
                     css_files.extend(file_part)
             head_part = module.html_head()
-            if head_part: html_heads.append(_utf8(head_part))
+            if head_part: html_heads.append(_encode(head_part,codec=self.coding))
             body_part = module.html_body()
-            if body_part: html_bodies.append(_utf8(body_part))
+            if body_part: html_bodies.append(_encode(body_part,codec=self.coding))
         if js_files:
             # Maintain order of JavaScript files given by modules
             paths = []
@@ -1449,19 +1450,19 @@ def reverse(self, *args):
 
 url = URLSpec
 
-def _utf8(s):
+def _encode(s,codec='utf-8'):
     if isinstance(s, unicode):
-        return s.encode("utf-8")
+        return s.encode(codec)
     assert isinstance(s, str)
     return s
 
 
-def _unicode(s):
+def _unicode(s,codec='utf-8'):
     if isinstance(s, str):
         try:
-            return s.decode("utf-8")
+            return s.decode(codec)
         except UnicodeDecodeError:
-            raise HTTPError(400, "Non-utf8 argument")
+            raise HTTPError(400, "Non-%s argument" % codec)
     assert isinstance(s, unicode)
     return s