Release 2021.07.0 (epam#88)

Dockerfile

@@ -0,0 +1,21 @@
+FROM mcr.microsoft.com/azure-functions/python:3.0-python3.7-appservice
+
+ENV AzureWebJobsScriptRoot=/home/site/wwwroot \
+    AzureFunctionsJobHost__Logging__Console__IsEnabled=true
+
+COPY __app__/requirements.txt /
+RUN pip install -r /requirements.txt
+
+COPY osci/requirements.txt /home/site/wwwroot/osci/
+RUN pip install -r /home/site/wwwroot/osci/requirements.txt
+
+COPY __app__ /home/site/wwwroot
+COPY osci /home/site/wwwroot/osci
+COPY README.md /home/site/wwwroot
+COPY LICENSE /home/site/wwwroot
+COPY setup.py /home/site/wwwroot
+
+WORKDIR /home/site/
+RUN pip install -e wwwroot/
+
+EXPOSE 80

README.md

@@ -28,6 +28,15 @@ The table shows the OSCI ranking for GitHub activity in June 2021. The leading o
 
 ![GitHub OSCI Schematic Diagram](images/OSCI_Schematic_Architecture.png)
 
+## OSCI Versioning :newspaper:
+We decided to use special versioning `(<year>.<month>.<number of patch >)` e.g. `2021.05.0`. This will provide us with a
+clearer understanding of the relevance of the product.  
+Also, date of 
+[adding a new company](#how-can-i-add-a-company-which-is-missing-from-the-osci-ranking) is very important and versioning 
+releases depending on the date looks more logical.  
+This is supposed to be a monthly update of the release.
+
+
 ## How did we decide on the ranking logic?
 
 We realize that there are many approaches which could be used to develop this ranking. We experimented extensively with these before arriving at the logic now used.
@@ -77,7 +86,7 @@ In order to add a company to the OSCI ranking, do the following:
 
 1) Create a new pull request.
 
-1) Go to company domain match list ([company_domain_match_list.yaml](__app__/match_company/company_domain_match_list.yaml))
+1) Go to company domain match list ([company_domain_match_list.yaml](osci/preprocess/match_company/company_domain_match_list.yaml))
 
 1) Confirm that the company you wish to add is not listed.
 
@@ -99,7 +108,10 @@ In order to add a company to the OSCI ranking, do the following:
         - ^.*\.facebook\.com$
     ```
 
-We will review your pull request and if it matches our requirements, we will merge it. 
+We will review your pull request and if it matches our requirements, we will merge it.  
+It's important to add at **the start
+of the month** a new company, because the rating depends on previous data, i.e. data for the beginning of the month.
+Furthermore, this will lead to OSCI release consistency.
 
 # QuickStart
 ## Technical Note
@@ -120,21 +132,21 @@ The code published here on GitHub does not require the Azure cloud. You can repr
    ```
 
 ## Configuration
-Create a file `local.yml` (by default this file added to .gitignore) in the directory [`__app__/config/files`](__app__/config/files). 
-A sample file [`default.yml`](__app__/config/files/default.yml) is included, please don't change values in this file
+Create a file `local.yml` (by default this file added to .gitignore) in the directory [`osci/config/files`](osci/config/files). 
+A sample file [`default.yml`](osci/config/files/default.yml) is included, please don't change values in this file
 
 ## Sample run
 1) Run script to download data from archive (for example for 01 January 2020)
     ```shell script
-         python3 osci.py get-github-daily-push-events -d 2020-01-01
+         python3 osci-cli.py get-github-daily-push-events -d 2020-01-01
     ```
 1) Run script to add company field (matched by domain) (for example for 01 January 2020)
     ```shell script
-         python3 osci.py process-github-daily-push-events -d 2020-01-01
+         python3 osci-cli.py process-github-daily-push-events -d 2020-01-01
     ```
 1) Run script to add company field (matched by domain) (for example for 01 January 2020)
     ```shell script
-         python3 osci.py daily-osci-rankings -td 2020-01-02
+         python3 osci-cli.py daily-osci-rankings -td 2020-01-02
     ```
 
 # License

__app__/__init__.py

@@ -1 +1 @@
-__version__ = '1.0.0'
+__version__ = '2.0.0'

__app__/daily_active_repositories_list/function.json → __app__/actions/function.json

@@ -1,5 +1,5 @@
 {
-  "scriptFile": "__init__.py",
+  "scriptFile": "osci-azure.py",
   "bindings": [
     {
       "authLevel": "anonymous",
@@ -9,7 +9,8 @@
       "methods": [
         "get",
         "post"
-      ]
+      ],
+      "route": "actions/{action_name?}"
     },
     {
       "type": "http",

__app__/load_repositories/__init__.py → __app__/actions/osci-azure.py

@@ -14,33 +14,36 @@
 
    You should have received a copy of the GNU General Public License
    along with OSCI.  If not, see <http://www.gnu.org/licenses/>."""
-from .process import load_repositories
-from __app__.utils import get_req_param
+from typing import Mapping, Dict
+from osci.actions import Action, ListAction
+from http import HTTPStatus
 
-import datetime
-import logging
 import azure.functions as func
 import traceback as tb
+import logging
+import json
 
-DAY_FORMAT = "%Y-%m-%d"
-DEFAULT_DAY = (datetime.datetime.now() - datetime.timedelta(days=1)).strftime(DAY_FORMAT)
 log = logging.getLogger(__name__)
 
 
 def main(req: func.HttpRequest) -> func.HttpResponse:
+    """Handle http request and run azure function handlers"""
     logging.basicConfig(format='[%(asctime)s] [%(levelname)s] %(message)s', level=logging.DEBUG)
-    status_code = 200
+    handlers: Mapping[str, Action] = {class_var.name(): class_var for class_var in Action.__subclasses__()}
+    log.info(f"Find Actions subclasses: {handlers}")
+    success_message = {"output": "This HTTP triggered function executed."}
     try:
-        log.info(f"Http trigger. req.params: {req.params}")
-        day = datetime.datetime.strptime(
-            get_req_param(req, 'date', default=DEFAULT_DAY),
-            DAY_FORMAT
-        )
-        load_repositories(date=day)
-        return func.HttpResponse('{"output":"This HTTP triggered function executed."}',
-                                 status_code=status_code)
+        action_name = req.route_params.get('action_name', 'list')
+        log.info(f"Action name: `{action_name}`")
+        action = handlers.get(action_name)
+
+        if not action:
+            raise KeyError(f"Can't find required Action for `{action_name}` azure function")
+        params: Dict = req.params or req.get_json()
+        message = action().execute(**params) or success_message
+        return func.HttpResponse(json.dumps(message), status_code=HTTPStatus.OK)
     except Exception as ex:
         ex_message = (f'Exception {ex} \n'
                       f'{"".join(tb.format_exception(etype=type(ex), value=ex, tb=ex.__traceback__))}')
         log.error(ex_message)
-        return func.HttpResponse(ex_message, status_code=500)
+        return func.HttpResponse(ex_message, status_code=HTTPStatus.INTERNAL_SERVER_ERROR)

__app__/durable_functions_http_starter/__init__.py

@@ -11,11 +11,11 @@
 
 async def main(req: func.HttpRequest, starter: str) -> func.HttpResponse:
     client = DurableOrchestrationClient(starter)
-
+    logging.info(f"Request url: {req.url}")
     base_url = re.search("(?P<url>https?://[^\s]+api)", req.url).group("url")
     function_name = req.route_params["function"]
     url_params = '&'.join(f'{k}={v}' for k, v in req.params.items())
-    uri = f'{base_url}/{function_name}?{url_params}'
+    uri = f'{base_url}/actions/{function_name}?{url_params}'
 
     instance_id = await client.start_new(orchestration_function_name=req.route_params["orchestrator"],
                                          instance_id=None, client_input=uri)

__app__/durable_functions_http_starter/function.json

@@ -6,7 +6,7 @@
       "name": "req",
       "type": "httpTrigger",
       "direction": "in",
-      "route": "orchestrators/{orchestrator}/{function}",
+      "route": "orchestrators/{orchestrator}/actions/{function}",
       "methods": [
         "post",
         "get"