Create a pip package for Oppia to support Beam jobs (oppia#13938)

* move jobs/ into core/ * update jobs packages * update jobs packages remnants * fix linter * fix imports * add setup.py * add encoding to open() * fix lint * move feconf * fix FECONF_PATH * fix test * move utils * move python_utils * fix linter test * fix python_utils test * remove temp_file.txt * fix linter test * move schema_utils * move constants * move android_validation_constants * Changes * Implement pip package Beam * Slight changes * Fix async * Remove auth * Update constants * Testing changes * Import all models * Changes * Include other changes * Fix linting * Fix tests * Fix backend shards * Fix tests * Fix backend tests * Fix tests and add docstrings * Fix coverage * Address comments * Address comments * Address comments Co-authored-by: Brian Rodriguez <[email protected]> Co-authored-by: Vojtěch Jelínek <[email protected]>
dummy-3-1 · Oct 1, 2021 · 0732ec3 · 0732ec3
1 parent 429612d
commit 0732ec3
Show file tree

Hide file tree

Showing 560 changed files with 1,534 additions and 1,568 deletions.
diff --git a/.coveragerc b/.coveragerc
@@ -8,6 +8,6 @@ omit =
     *core/tests/*
     *scripts/linters/test_files/*
     *__init__.py
-    # TODO(#7419): Remove python_utils from the list once the codebase is
+    # TODO(#13935): Remove python_utils from the list once the codebase is
     # run under Python 3.
-    python_utils.py
+    core/python_utils.py
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -45,7 +45,7 @@
 
 
 # Android team
-/android_validation_constants*.py @seanlip @BenHenning
+/core/android_validation_constants*.py @seanlip @BenHenning
 
 
 # Angular Migration team
@@ -479,19 +479,23 @@
 /core/templates/pages/topics-and-skills-dashboard-page/ @aks681 @seanlip
 
 
-# Infrastructure.
-/jobs/ @vojtechjelinek
-/jobs/transforms @vojtechjelinek
-/jobs/types @vojtechjelinek
+# Beam jobs
+/setup.py @vojtechjelinek @brianrodri
+/MANIFEST.in @vojtechjelinek @brianrodri
+/core/jobs/ @vojtechjelinek
+/core/jobs/transforms @vojtechjelinek
+/core/jobs/types @vojtechjelinek
 /core/templates/domain/jobs/ @vojtechjelinek
+/core/controllers/beam_jobs*.py @brianrodri
+
+# Infrastructure.
 /core/controllers/cron*.py @nithusha21
 /core/domain/cron_services*.py @nithusha21
 /main*.py @DubeySandeep @nithusha21
-/feconf.py @DubeySandeep @nithusha21 @seanlip
-/constants*.py @nithusha21
+/core/feconf.py @DubeySandeep @nithusha21 @seanlip
+/core/constants*.py @nithusha21
 /assets/constants.ts @nithusha21
 /core/controllers/tasks*.py @aks681
-/core/controllers/beam_jobs*.py @brianrodri
 /core/domain/beam_job*.py @vojtechjelinek
 /core/domain/email*.py @aks681
 /core/domain/image_service*.py @DubeySandeep
@@ -526,9 +530,9 @@
 /core/templates/domain/utilities/ @DubeySandeep
 /core/templates/Polyfills.ts @srijanreddy98
 /extensions/extensions.module.ts @vojtechjelinek
-/python_utils*.py @DubeySandeep
-/schema_utils*.py @DubeySandeep
-/utils*.py @aks681
+/core/python_utils*.py @DubeySandeep
+/core/schema_utils*.py @DubeySandeep
+/core/utils*.py @aks681
 
 
 # Python typing
@@ -557,7 +561,7 @@
 /core/controllers/release_coordinator*.py @nithusha21 @vojtechjelinek
 /core/templates/pages/release-coordinator-page/ @nithusha21 @vojtechjelinek
 /scripts/release_scripts/ @nithusha21 @vojtechjelinek
-/release_constants.json @nithusha21 @vojtechjelinek
+/assets/release_constants.json @nithusha21 @vojtechjelinek
 
 
 # Rich text editor team.

diff --git a/.gitignore b/.gitignore
@@ -3,6 +3,7 @@
 *.swp
 *.swo
 *.bak
+*.egg-info
 __pycache__/
 gae_runtime/*
 # Ignore the auto-generated protobuf python code.

diff --git a/MANIFEST.in b/MANIFEST.in
@@ -0,0 +1,9 @@
+# This file specifies which non-py files need to be included in the Oppia Python
+# package that is generated in order for Apache Beam jobs to be able to run
+# in production. Note that this package is generated
+# by running 'python setup.py sdist' in build.py.
+
+include requirements.txt
+include assets/constants.ts
+include assets/release_constants.json
+include assets/rich_text_components_definitions.ts
diff --git a/jobs/__init__.py → assets/__init__.py b/jobs/__init__.py → assets/__init__.py
diff --git a/release_constants.json → assets/release_constants.json b/release_constants.json → assets/release_constants.json
@@ -40,7 +40,7 @@
     "RELEASE_SUMMARY_FILEPATH": "../release_summary.md",
     "REMOTE_URL": "[email protected]:oppia/oppia.git",
     "REPO_SPECIFIC_CHANGE_MODULE_TARGET": "scripts.release_scripts.repo_specific_changes_fetcher",
-    "DEPLOYMENT_CONFIG_PATH": "feconf.py",
+    "DEPLOYMENT_CONFIG_PATH": "core/feconf.py",
     "SCHEMA_VERSIONS_TO_JOBS_MAPPING": {
         "CURRENT_COLLECTION_SCHEMA_VERSION": ["CollectionMigrationOneOffJob"],
         "CURRENT_STATE_SCHEMA_VERSION": [

diff --git a/android_validation_constants.py → core/android_validation_constants.py b/android_validation_constants.py → core/android_validation_constants.py
@@ -22,7 +22,7 @@
 
 import re
 
-from constants import constants
+from core.constants import constants
 
 
 # These are the valid interactions that are allowed in an exploration

diff --git a/android_validation_constants_test.py → core/android_validation_constants_test.py b/android_validation_constants_test.py → core/android_validation_constants_test.py
@@ -19,8 +19,8 @@
 from __future__ import absolute_import
 from __future__ import unicode_literals
 
-import android_validation_constants
-from constants import constants
+from core import android_validation_constants
+from core.constants import constants
 from core.tests import test_utils
 
 

diff --git a/constants.py → core/constants.py b/constants.py → core/constants.py
@@ -20,27 +20,26 @@
 from __future__ import unicode_literals
 
 import json
-import os
 import re
 
-import python_utils
+from core import python_utils
 
-from typing import Any, Dict, TextIO
+from typing import Any, Dict
 
 
 # Here we use Dict[str, Any] as return type because we need to parse and return
 # generic JSON objects.
-def parse_json_from_js(js_file: TextIO) -> Dict[str, Any]:
-    """Extracts JSON object from JS file.
+def parse_json_from_ts(ts_file_contents: str) -> Dict[str, Any]:
+    """Extracts JSON object from TS file.
 
     Args:
-        js_file: file. The js file containing JSON that needs to be parsed.
+        ts_file_contents: str. The contents of the TS file containing JSON that
+            needs to be parsed.
 
     Returns:
-        dict. The dict representation of JSON object in the js file.
+        dict. The dict representation of JSON object in the TS file.
     """
-    text = js_file.read()
-    text_without_comments = remove_comments(text)
+    text_without_comments = remove_comments(ts_file_contents)
     json_start = text_without_comments.find('{\n')
     # Add 1 to index returned because the '}' is part of the JSON object.
     json_end = text_without_comments.rfind('}') + 1
@@ -61,7 +60,7 @@ def remove_comments(text: str) -> str:
     return re.sub(r'  //.*\n', r'', text)
 
 
-class Constants(dict): # type: ignore[type-arg]
+class Constants(dict):  # type: ignore[type-arg]
     """Transforms dict to object, attributes can be accessed by dot notation."""
 
     # Here `value` has the type Any because it parses and stores the values of
@@ -76,8 +75,12 @@ def __getattr__(self, name: str) -> Any:
         return self[name]
 
 
-with python_utils.open_file(os.path.join('assets', 'constants.ts'), 'r') as f: # type: ignore[no-untyped-call]
-    constants = Constants(parse_json_from_js(f))  # pylint:disable=invalid-name
+constants = Constants(parse_json_from_ts(  # pylint:disable=invalid-name
+    python_utils.get_package_file_contents('assets', 'constants.ts')))
 
-with python_utils.open_file('release_constants.json', 'r') as f: # type: ignore[no-untyped-call]
-    release_constants = Constants(json.loads(f.read()))  # pylint:disable=invalid-name
+release_constants = Constants( # pylint:disable=invalid-name
+    json.loads(
+        python_utils.get_package_file_contents(
+            'assets', 'release_constants.json')
+    )
+)
diff --git a/constants_test.py → core/constants_test.py b/constants_test.py → core/constants_test.py
@@ -19,10 +19,10 @@
 
 import os
 
-import constants
+from core import constants
+from core import feconf
+from core import python_utils
 from core.tests import test_utils
-import feconf
-import python_utils
 
 
 class ConstantsTests(test_utils.GenericTestBase):
@@ -35,8 +35,9 @@ def test_constants_file_is_existing(self) -> None:
     def test_constants_file_contains_valid_json(self) -> None:
         """Test if the constants file is valid json file."""
         with python_utils.open_file( # type: ignore[no-untyped-call]
-            os.path.join('assets', 'constants.ts'), 'r') as f:
-            json = constants.parse_json_from_js(f)
+            os.path.join('assets', 'constants.ts'), 'r'
+        ) as f:
+            json = constants.parse_json_from_ts(f.read())
             self.assertTrue(isinstance(json, dict))
             self.assertEqual(json['TESTING_CONSTANT'], 'test')
 

diff --git a/core/controllers/access_validators.py b/core/controllers/access_validators.py
@@ -17,14 +17,13 @@
 from __future__ import absolute_import
 from __future__ import unicode_literals
 
-from constants import constants
+from core import feconf
+from core.constants import constants
 from core.controllers import acl_decorators
 from core.controllers import base
 from core.domain import classroom_services
 from core.domain import user_services
 
-import feconf
-
 from typing import Any, Dict # isort: skip
 
 

diff --git a/core/controllers/access_validators_test.py b/core/controllers/access_validators_test.py
@@ -17,10 +17,10 @@
 from __future__ import absolute_import
 from __future__ import unicode_literals
 
-from constants import constants
+from core import feconf
+from core.constants import constants
 from core.domain import config_services
 from core.tests import test_utils
-import feconf
 
 ACCESS_VALIDATION_HANDLER_PREFIX = feconf.ACCESS_VALIDATION_HANDLER_PREFIX
 

diff --git a/core/controllers/acl_decorators.py b/core/controllers/acl_decorators.py
@@ -23,8 +23,10 @@
 import logging
 import re
 
-import android_validation_constants
-from constants import constants
+from core import android_validation_constants
+from core import feconf
+from core import utils
+from core.constants import constants
 from core.controllers import base
 from core.domain import blog_services
 from core.domain import classifier_services
@@ -43,8 +45,6 @@
 from core.domain import topic_fetchers
 from core.domain import topic_services
 from core.domain import user_services
-import feconf
-import utils
 
 from typing import Any, Callable # isort: skip
 

diff --git a/core/controllers/acl_decorators_test.py b/core/controllers/acl_decorators_test.py
@@ -20,9 +20,11 @@
 from __future__ import unicode_literals
 
 import json
-import android_validation_constants
 
-from constants import constants
+from core import android_validation_constants
+from core import feconf
+from core import python_utils
+from core.constants import constants
 from core.controllers import acl_decorators
 from core.controllers import base
 from core.domain import app_feedback_report_domain
@@ -48,8 +50,6 @@
 from core.domain import topic_services
 from core.domain import user_services
 from core.tests import test_utils
-import feconf
-import python_utils
 
 import webapp2
 import webtest

diff --git a/core/controllers/admin.py b/core/controllers/admin.py
@@ -21,7 +21,10 @@
 import logging
 import random
 
-from constants import constants
+from core import feconf
+from core import python_utils
+from core import utils
+from core.constants import constants
 from core.controllers import acl_decorators
 from core.controllers import base
 from core.controllers import domain_objects_validator as validation_method
@@ -56,9 +59,6 @@
 from core.domain import topic_services
 from core.domain import user_services
 from core.domain import wipeout_service
-import feconf
-import python_utils
-import utils
 
 
 class AdminPage(base.BaseHandler):

diff --git a/core/controllers/admin_test.py b/core/controllers/admin_test.py
@@ -20,7 +20,10 @@
 import datetime
 import logging
 
-from constants import constants
+from core import feconf
+from core import python_utils
+from core import utils
+from core.constants import constants
 from core.domain import blog_services
 from core.domain import collection_services
 from core.domain import config_domain
@@ -49,9 +52,6 @@
 from core.platform import models
 from core.platform.auth import firebase_auth_services
 from core.tests import test_utils
-import feconf
-import python_utils
-import utils
 
 (
     audit_models, blog_models, exp_models, opportunity_models,

diff --git a/core/controllers/android_e2e_config.py b/core/controllers/android_e2e_config.py
@@ -19,7 +19,9 @@
 
 import os
 
-from constants import constants
+from core import feconf
+from core import python_utils
+from core.constants import constants
 from core.controllers import acl_decorators
 from core.controllers import base
 from core.domain import exp_domain
@@ -41,8 +43,6 @@
 from core.domain import topic_fetchers
 from core.domain import topic_services
 from core.domain import user_services
-import feconf
-import python_utils
 
 
 class InitializeAndroidTestDataHandler(base.BaseHandler):

diff --git a/core/controllers/android_e2e_config_test.py b/core/controllers/android_e2e_config_test.py
@@ -19,14 +19,14 @@
 
 import os
 
-from constants import constants
+from core import feconf
+from core.constants import constants
 from core.domain import exp_fetchers
 from core.domain import skill_fetchers
 from core.domain import story_fetchers
 from core.domain import topic_fetchers
 from core.domain import topic_services
 from core.tests import test_utils
-import feconf
 
 
 class AndroidConfigTest(test_utils.GenericTestBase):

diff --git a/core/controllers/base.py b/core/controllers/base.py
@@ -27,15 +27,15 @@
 import re
 import time
 
+from core import feconf
+from core import python_utils
+from core import utils
 from core.controllers import payload_validator
 from core.domain import auth_domain
 from core.domain import auth_services
 from core.domain import config_domain
 from core.domain import config_services
 from core.domain import user_services
-import feconf
-import python_utils
-import utils
 
 import webapp2