Convert all Python strings to unicode (oppia#7446)

* add unicode_literals

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* address comments

* fix lint

* fix lint

* fix

* fix tests

* fix lint

* fix

* fix

* add test

* fix

* increase coverage

* remove to ascii

* fix

* fix lint

* fix test

* address some comments

* address comments

* fix

* fix

* fix

* fix

* fix

* fix lint

* fix

* fix tes

* address comments

* fix lint

* fix

* address comments

* fix

* fix lint

.circleci/config.yml

@@ -71,6 +71,7 @@ jobs:
       - run: date +%F > date
       - restore_cache:
           <<: *restore_cache
+      - run: sudo pip install pyyaml
       - run:
           name: Run typescript tests
           command: |

appengine_config.py

@@ -16,6 +16,7 @@
 
 """Configuration for App Engine."""
 from __future__ import absolute_import  # pylint: disable=import-only-modules
+from __future__ import unicode_literals  # pylint: disable=import-only-modules
 
 import logging
 import os

constants.py

@@ -18,6 +18,7 @@
 
 """Loads constants for backend use."""
 from __future__ import absolute_import  # pylint: disable=import-only-modules
+from __future__ import unicode_literals  # pylint: disable=import-only-modules
 
 import json
 import os

constants_test.py

@@ -14,6 +14,7 @@
 
 """Tests for Constants object and cosntants.json file."""
 from __future__ import absolute_import  # pylint: disable=import-only-modules
+from __future__ import unicode_literals  # pylint: disable=import-only-modules
 
 import os
 

core/controllers/acl_decorators.py

@@ -16,6 +16,7 @@
 
 """Decorators to provide authorization across the site."""
 from __future__ import absolute_import  # pylint: disable=import-only-modules
+from __future__ import unicode_literals  # pylint: disable=import-only-modules
 
 import functools
 

core/controllers/acl_decorators_test.py

@@ -16,6 +16,7 @@
 
 """Tests for core.domain.acl_decorators."""
 from __future__ import absolute_import  # pylint: disable=import-only-modules
+from __future__ import unicode_literals  # pylint: disable=import-only-modules
 
 from core.controllers import acl_decorators
 from core.controllers import base

core/controllers/admin.py

@@ -14,6 +14,7 @@
 
 """Controllers for the admin view."""
 from __future__ import absolute_import  # pylint: disable=import-only-modules
+from __future__ import unicode_literals  # pylint: disable=import-only-modules
 
 import logging
 import random
@@ -210,7 +211,7 @@ def post(self):
                 recommendations_services.update_topic_similarities(data)
             self.render_json({})
         except Exception as e:
-            self.render_json({'error': python_utils.STR(e)})
+            self.render_json({'error': python_utils.UNICODE(e)})
             raise
 
     def _reload_exploration(self, exploration_id):

core/controllers/admin_test.py

@@ -14,6 +14,7 @@
 
 """Tests for the admin page."""
 from __future__ import absolute_import  # pylint: disable=import-only-modules
+from __future__ import unicode_literals  # pylint: disable=import-only-modules
 
 import logging
 

core/controllers/base.py

@@ -14,6 +14,7 @@
 
 """Base constants and handlers."""
 from __future__ import absolute_import  # pylint: disable=import-only-modules
+from __future__ import unicode_literals  # pylint: disable=import-only-modules
 
 import base64
 import datetime
@@ -57,14 +58,14 @@ def _clear_login_cookies(response_headers):
     # App Engine sets the ACSID cookie for http:// and the SACSID cookie
     # for https:// . We just unset both below.
     cookie = http.cookies.SimpleCookie()
-    for cookie_name in ['ACSID', 'SACSID']:
+    for cookie_name in [b'ACSID', b'SACSID']:
         cookie = http.cookies.SimpleCookie()
         cookie[cookie_name] = ''
         cookie[cookie_name]['expires'] = (
             datetime.datetime.utcnow() +
             datetime.timedelta(seconds=ONE_DAY_AGO_IN_SECS)
         ).strftime('%a, %d %b %Y %H:%M:%S GMT')
-        response_headers.add_header(*cookie.output().split(': ', 1))
+        response_headers.add_header(*cookie.output().split(b': ', 1))
 
 
 class LogoutPage(webapp2.RequestHandler):
@@ -200,7 +201,8 @@ def dispatch(self):
         # If the request is to the old demo server, redirect it permanently to
         # the new demo server.
         if self.request.uri.startswith('https://oppiaserver.appspot.com'):
-            self.redirect('https://oppiatestserver.appspot.com', permanent=True)
+            self.redirect(
+                b'https://oppiatestserver.appspot.com', permanent=True)
             return
 
         # In DEV_MODE, clearing cookies does not log out the user, so we
@@ -263,22 +265,23 @@ def render_json(self, values):
         Args:
             values: dict. The key-value pairs to encode in the JSON response.
         """
-        self.response.content_type = 'application/json; charset=utf-8'
-        self.response.headers['Content-Disposition'] = (
-            'attachment; filename="oppia-attachment.txt"')
-        self.response.headers['Strict-Transport-Security'] = (
-            'max-age=31536000; includeSubDomains')
-        self.response.headers['X-Content-Type-Options'] = 'nosniff'
-        self.response.headers['X-Xss-Protection'] = '1; mode=block'
+        self.response.content_type = b'application/json; charset=utf-8'
+        self.response.headers[b'Content-Disposition'] = (
+            b'attachment; filename="oppia-attachment.txt"')
+        self.response.headers[b'Strict-Transport-Security'] = (
+            b'max-age=31536000; includeSubDomains')
+        self.response.headers[b'X-Content-Type-Options'] = b'nosniff'
+        self.response.headers[b'X-Xss-Protection'] = b'1; mode=block'
 
         json_output = json.dumps(values, cls=utils.JSONEncoderForHTML)
         self.response.write('%s%s' % (feconf.XSSI_PREFIX, json_output))
 
     def render_downloadable_file(self, values, filename, content_type):
         """Prepares downloadable content to be sent to the client."""
-        self.response.headers['Content-Type'] = content_type
+        self.response.headers[b'Content-Type'] = python_utils.convert_to_bytes(
+            content_type)
         self.response.headers[
-            'Content-Disposition'] = python_utils.convert_to_bytes(
+            b'Content-Disposition'] = python_utils.convert_to_bytes(
                 'attachment; filename=%s' % filename)
         self.response.write(values)
 
@@ -326,15 +329,15 @@ def render_template(self, filepath, iframe_restriction='DENY'):
 
         self.response.cache_control.no_cache = True
         self.response.cache_control.must_revalidate = True
-        self.response.headers['Strict-Transport-Security'] = (
-            'max-age=31536000; includeSubDomains')
-        self.response.headers['X-Content-Type-Options'] = 'nosniff'
-        self.response.headers['X-Xss-Protection'] = '1; mode=block'
+        self.response.headers[b'Strict-Transport-Security'] = (
+            b'max-age=31536000; includeSubDomains')
+        self.response.headers[b'X-Content-Type-Options'] = b'nosniff'
+        self.response.headers[b'X-Xss-Protection'] = b'1; mode=block'
 
         if iframe_restriction is not None:
             if iframe_restriction in ['SAMEORIGIN', 'DENY']:
                 self.response.headers[
-                    'X-Frame-Options'] = python_utils.convert_to_bytes(
+                    b'X-Frame-Options'] = python_utils.convert_to_bytes(
                         iframe_restriction)
             else:
                 raise Exception(
@@ -427,7 +430,7 @@ def handle_exception(self, exception, unused_debug_mode):
                     current_user_services.create_login_url(self.request.uri))
             return
 
-        logging.info(''.join(traceback.format_exception(*sys.exc_info())))
+        logging.info(b''.join(traceback.format_exception(*sys.exc_info())))
 
         if isinstance(exception, self.PageNotFoundException):
             logging.warning('Invalid URL requested: %s', self.request.uri)

core/controllers/base_test.py

@@ -16,6 +16,7 @@
 
 """Tests for generic controller behavior."""
 from __future__ import absolute_import  # pylint: disable=import-only-modules
+from __future__ import unicode_literals  # pylint: disable=import-only-modules
 
 import datetime
 import importlib
@@ -393,7 +394,7 @@ def mock_create_handlers_map():
             import main  # pylint: disable-all
 
         headers_dict = {
-            'X-AppEngine-TaskName': 'taskname'
+            'X-AppEngine-TaskName': b'taskname'
         }
         self.assertEqual(len(main.MAPREDUCE_HANDLERS), 1)
         self.assertEqual(main.MAPREDUCE_HANDLERS[0][0], '/mock')

core/controllers/classifier.py

@@ -14,6 +14,7 @@
 
 """Controllers for communicating with the VM for training classifiers."""
 from __future__ import absolute_import  # pylint: disable=import-only-modules
+from __future__ import unicode_literals  # pylint: disable=import-only-modules
 
 import hashlib
 import hmac

core/controllers/classifier_test.py

@@ -16,6 +16,7 @@
 classifiers.
 """
 from __future__ import absolute_import  # pylint: disable=import-only-modules
+from __future__ import unicode_literals  # pylint: disable=import-only-modules
 
 import datetime
 import json
@@ -104,7 +105,7 @@ def setUp(self):
         self.payload['message'] = self.job_result_dict
         secret = feconf.DEFAULT_VM_SHARED_SECRET
         self.payload['signature'] = classifier.generate_signature(
-            secret, self.payload['message'])
+            python_utils.convert_to_bytes(secret), self.payload['message'])
 
     def test_trained_classifier_handler(self):
         # Normal end-to-end test.
@@ -268,17 +269,17 @@ def setUp(self):
         )
 
         self.expected_response = {
-            u'job_id': python_utils.STR(self.job_id, 'utf-8'),
+            u'job_id': self.job_id,
             u'training_data': self.training_data,
-            u'algorithm_id': python_utils.STR(self.algorithm_id, 'utf-8')
+            u'algorithm_id': self.algorithm_id
         }
 
         self.payload = {}
         self.payload['vm_id'] = feconf.DEFAULT_VM_ID
         secret = feconf.DEFAULT_VM_SHARED_SECRET
         self.payload['message'] = json.dumps({})
         self.payload['signature'] = classifier.generate_signature(
-            secret, self.payload['message'])
+            python_utils.convert_to_bytes(secret), self.payload['message'])
 
     def test_next_job_handler(self):
         json_response = self.post_json(

core/controllers/classroom.py

@@ -14,6 +14,7 @@
 
 """Controllers for the classroom page."""
 from __future__ import absolute_import  # pylint: disable=import-only-modules
+from __future__ import unicode_literals  # pylint: disable=import-only-modules
 
 from constants import constants
 from core.controllers import acl_decorators

core/controllers/classroom_test.py

@@ -14,6 +14,7 @@
 
 """Tests for the classroom page."""
 from __future__ import absolute_import  # pylint: disable=import-only-modules
+from __future__ import unicode_literals  # pylint: disable=import-only-modules
 
 from constants import constants
 from core.tests import test_utils

core/controllers/collection_editor.py

@@ -16,6 +16,7 @@
 
 """Controllers for the collections editor."""
 from __future__ import absolute_import  # pylint: disable=import-only-modules
+from __future__ import unicode_literals  # pylint: disable=import-only-modules
 
 import base64
 

core/controllers/collection_editor_test.py

@@ -14,6 +14,7 @@
 
 """Tests for the collection editor page."""
 from __future__ import absolute_import  # pylint: disable=import-only-modules
+from __future__ import unicode_literals  # pylint: disable=import-only-modules
 
 from core.domain import collection_domain
 from core.domain import collection_services

core/controllers/collection_viewer.py

@@ -14,6 +14,7 @@
 
 """Controllers for the Oppia collection learner view."""
 from __future__ import absolute_import  # pylint: disable=import-only-modules
+from __future__ import unicode_literals  # pylint: disable=import-only-modules
 
 from core.controllers import acl_decorators
 from core.controllers import base

core/controllers/collection_viewer_test.py

@@ -14,6 +14,7 @@
 
 """Tests for the page that allows learners to play through a collection."""
 from __future__ import absolute_import  # pylint: disable=import-only-modules
+from __future__ import unicode_literals  # pylint: disable=import-only-modules
 
 from core.domain import collection_services
 from core.domain import rights_manager

core/controllers/community_dashboard.py

@@ -14,6 +14,7 @@
 
 """Controllers for the community dashboard page."""
 from __future__ import absolute_import  # pylint: disable=import-only-modules
+from __future__ import unicode_literals  # pylint: disable=import-only-modules
 
 from constants import constants
 from core.controllers import acl_decorators

core/controllers/community_dashboard_test.py

@@ -14,6 +14,7 @@
 
 """Tests for the community dashboard controllers."""
 from __future__ import absolute_import  # pylint: disable=import-only-modules
+from __future__ import unicode_literals  # pylint: disable=import-only-modules
 
 from core.domain import exp_domain
 from core.domain import exp_services

core/controllers/concept_card_viewer.py

@@ -14,6 +14,7 @@
 
 """Controllers for the Oppia skill's concept card viewer."""
 from __future__ import absolute_import  # pylint: disable=import-only-modules
+from __future__ import unicode_literals  # pylint: disable=import-only-modules
 
 from constants import constants
 from core.controllers import acl_decorators

core/controllers/concept_card_viewer_test.py

@@ -14,6 +14,7 @@
 
 """Tests for the handler that returns concept card for a skill."""
 from __future__ import absolute_import  # pylint: disable=import-only-modules
+from __future__ import unicode_literals  # pylint: disable=import-only-modules
 
 from constants import constants
 from core.domain import skill_domain

core/controllers/creator_dashboard.py

@@ -16,6 +16,7 @@
 activities.
 """
 from __future__ import absolute_import  # pylint: disable=import-only-modules
+from __future__ import unicode_literals  # pylint: disable=import-only-modules
 
 import logging
 

core/controllers/creator_dashboard_test.py

@@ -14,6 +14,7 @@
 
 """Tests for the creator dashboard and the notifications dashboard."""
 from __future__ import absolute_import  # pylint: disable=import-only-modules
+from __future__ import unicode_literals  # pylint: disable=import-only-modules
 
 import datetime
 

core/controllers/cron.py

@@ -14,6 +14,7 @@
 
 """Controllers for the cron jobs."""
 from __future__ import absolute_import  # pylint: disable=import-only-modules
+from __future__ import unicode_literals  # pylint: disable=import-only-modules
 
 import logging
 

core/controllers/cron_test.py

@@ -14,6 +14,7 @@
 
 """Tests for the cron jobs."""
 from __future__ import absolute_import  # pylint: disable=import-only-modules
+from __future__ import unicode_literals  # pylint: disable=import-only-modules
 
 import logging
 

core/controllers/custom_landing_pages.py

@@ -16,6 +16,7 @@
 
 """Controllers for custom landing pages."""
 from __future__ import absolute_import  # pylint: disable=import-only-modules
+from __future__ import unicode_literals  # pylint: disable=import-only-modules
 
 from core.controllers import acl_decorators
 from core.controllers import base

core/controllers/custom_landing_pages_test.py

@@ -14,6 +14,7 @@
 
 """Tests for custom landing pages."""
 from __future__ import absolute_import  # pylint: disable=import-only-modules
+from __future__ import unicode_literals  # pylint: disable=import-only-modules
 
 from core.tests import test_utils
 import feconf

core/controllers/editor.py

@@ -16,6 +16,7 @@
 
 """Controllers for the editor view."""
 from __future__ import absolute_import  # pylint: disable=import-only-modules
+from __future__ import unicode_literals  # pylint: disable=import-only-modules
 
 import datetime
 import imghdr

core/controllers/editor_test.py

@@ -16,6 +16,7 @@
 
 """Tests for the exploration editor page."""
 from __future__ import absolute_import  # pylint: disable=import-only-modules
+from __future__ import unicode_literals  # pylint: disable=import-only-modules
 
 import datetime
 import logging

core/controllers/email_dashboard.py

@@ -14,6 +14,7 @@
 
 """Controller for user query related pages and handlers."""
 from __future__ import absolute_import  # pylint: disable=import-only-modules
+from __future__ import unicode_literals  # pylint: disable=import-only-modules
 
 from core.controllers import acl_decorators
 from core.controllers import base

core/controllers/email_dashboard_test.py

@@ -14,6 +14,7 @@
 
 """Tests for email dashboard handler."""
 from __future__ import absolute_import  # pylint: disable=import-only-modules
+from __future__ import unicode_literals  # pylint: disable=import-only-modules
 
 from core.domain import user_query_jobs_one_off
 from core.domain import user_query_services