Merge branch 'release-2.2.0'

Author: nmanovic

.bandit

@@ -6,3 +6,4 @@
 # B406 : import_xml_sax
 # B410 : import_lxml
 skips: B101,B102,B320,B404,B406,B410
+exclude: **/tests/**,tests

.coveragerc

@@ -4,7 +4,8 @@ branch = true
 
 source =
     cvat/apps/
-    utils/cli/
+    cvat-sdk/
+    cvat-cli/
     utils/dataset_manifest
 
 omit =

.eslintrc.js

@@ -7,27 +7,27 @@ module.exports = {
     env: {
         node: true,
         browser: true,
-        es6: true,
+        es2020: true,
     },
     parserOptions: {
         sourceType: 'module',
-        ecmaVersion: 2018,
+        parser: '@typescript-eslint/parser',
     },
     ignorePatterns: [
         '.eslintrc.js',
         'lint-staged.config.js',
     ],
-    plugins: ['security', 'no-unsanitized', 'eslint-plugin-header', 'import'],
+    plugins: ['@typescript-eslint', 'security', 'no-unsanitized', 'eslint-plugin-header', 'import'],
     extends: [
         'eslint:recommended', 'plugin:security/recommended', 'plugin:no-unsanitized/DOM',
         'airbnb-base', 'plugin:import/errors', 'plugin:import/warnings',
-        'plugin:import/typescript',
+        'plugin:import/typescript', 'plugin:@typescript-eslint/recommended', 'airbnb-typescript/base',
     ],
     rules: {
-        'header/header': [2, 'line', [{
-            pattern: ' {1}Copyright \\(C\\) (?:20\\d{2}-)?2022 Intel Corporation',
-            template: ' Copyright (C) 2022 Intel Corporation'
-        }, '', ' SPDX-License-Identifier: MIT']],
+        // 'header/header': [2, 'line', [{
+        //     pattern: ' {1}Copyright \\(C\\) (?:20\\d{2}-)?2022 Intel Corporation',
+        //     template: ' Copyright (C) 2022 Intel Corporation'
+        // }, '', ' SPDX-License-Identifier: MIT']],
         'no-plusplus': 0,
         'no-continue': 0,
         'no-console': 0,
@@ -51,5 +51,22 @@ module.exports = {
         'security/detect-object-injection': 0, // the rule is relevant for user input data on the node.js environment
         'import/order': ['error', {'groups': ['builtin', 'external', 'internal']}],
         'import/prefer-default-export': 0, // works incorrect with interfaces
+
+        '@typescript-eslint/ban-ts-comment': 0,
+        '@typescript-eslint/no-explicit-any': 0,
+        '@typescript-eslint/indent': ['error', 4],
+        '@typescript-eslint/lines-between-class-members': 0,
+        '@typescript-eslint/explicit-function-return-type': ['warn', { allowExpressions: true }],
+        '@typescript-eslint/explicit-module-boundary-types': 'off',
+        '@typescript-eslint/ban-types': [
+            'error',
+            {
+                types: {
+                    '{}': false, // TODO: try to fix with Record<string, unknown>
+                    object: false, // TODO: try to fix with Record<string, unknown>
+                    Function: false, // TODO: try to fix somehow
+                },
+            },
+        ],
     },
 };

.github/ISSUE_TEMPLATE.md

@@ -1,11 +1,5 @@
-<!---
-Copyright (C) 2020-2021 Intel Corporation
-
-SPDX-License-Identifier: MIT
--->
-
 ### My actions before raising this issue
-- [ ] Read/searched [the docs](https://github.com/opencv/cvat/tree/master#documentation)
+- [ ] Read/searched [the docs](https://github.com/cvat-ai/cvat/tree/master#documentation)
 - [ ] Searched [past issues](/issues)
 
 <!--- Provide a general summary of the issue in the Title above -->
@@ -49,5 +43,3 @@ the bug in -->
     <summary>Logs from `cvat` container</summary>
     </details>
 
-### Next steps
-You may [join our Gitter](https://gitter.im/opencv-cvat/public) channel for community support.

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,13 +1,7 @@
-<!---
-Copyright (C) 2020-2022 Intel Corporation
-
-SPDX-License-Identifier: MIT
--->
-
-<!-- Raised an issue to propose your change (https://github.com/opencv/cvat/issues).
+<!-- Raised an issue to propose your change (https://github.com/cvat-ai/cvat/issues).
 It helps to avoid duplication of efforts from multiple independent contributors.
 Discuss your ideas with maintainers to be sure that changes will be approved and merged.
-Read the [CONTRIBUTION](https://github.com/opencv/cvat/blob/develop/CONTRIBUTING.md)
+Read the [CONTRIBUTION](https://github.com/cvat-ai/cvat/blob/develop/CONTRIBUTING.md)
 guide. -->
 
 <!-- Provide a general summary of your changes in the Title above -->
@@ -28,24 +22,18 @@ If an item isn't applicable by a reason then ~~explicitly strikethrough~~ the wh
 line. If you don't do that github will show an incorrect process for the pull request.
 If you're unsure about any of these, don't hesitate to ask. We're here to help! -->
 - [ ] I submit my changes into the `develop` branch
-- [ ] I have added a description of my changes into [CHANGELOG](https://github.com/opencv/cvat/blob/develop/CHANGELOG.md) file
+- [ ] I have added a description of my changes into [CHANGELOG](https://github.com/cvat-ai/cvat/blob/develop/CHANGELOG.md) file
 - [ ] I have updated the [documentation](
-  https://github.com/opencv/cvat/blob/develop/README.md#documentation) accordingly
+  https://github.com/cvat-ai/cvat/blob/develop/README.md#documentation) accordingly
 - [ ] I have added tests to cover my changes
 - [ ] I have linked related issues ([read github docs](
   https://help.github.com/en/github/managing-your-work-on-github/linking-a-pull-request-to-an-issue#linking-a-pull-request-to-an-issue-using-a-keyword))
-- [ ] I have increased versions of npm packages if it is necessary ([cvat-canvas](https://github.com/opencv/cvat/tree/develop/cvat-canvas#versioning),
-[cvat-core](https://github.com/opencv/cvat/tree/develop/cvat-core#versioning), [cvat-data](https://github.com/opencv/cvat/tree/develop/cvat-data#versioning) and [cvat-ui](https://github.com/opencv/cvat/tree/develop/cvat-ui#versioning))
+- [ ] I have increased versions of npm packages if it is necessary ([cvat-canvas](https://github.com/cvat-ai/cvat/tree/develop/cvat-canvas#versioning),
+[cvat-core](https://github.com/cvat-ai/cvat/tree/develop/cvat-core#versioning), [cvat-data](https://github.com/cvat-ai/cvat/tree/develop/cvat-data#versioning) and [cvat-ui](https://github.com/cvat-ai/cvat/tree/develop/cvat-ui#versioning))
 
 ### License
 
 - [ ] I submit _my code changes_ under the same [MIT License](
-  https://github.com/opencv/cvat/blob/develop/LICENSE) that covers the project.
+  https://github.com/cvat-ai/cvat/blob/develop/LICENSE) that covers the project.
   Feel free to contact the maintainers if that's a concern.
-- [ ] I have updated the license header for each file (see an example below)
-
-```python
-# Copyright (C) 2022 Intel Corporation
-#
-# SPDX-License-Identifier: MIT
-```
+  

.github/workflows/bandit.yml

@@ -1,7 +1,7 @@
-name: Linter
+name: Bandit
 on: pull_request
 jobs:
-  Bandit:
+  Linter:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
@@ -17,7 +17,8 @@ jobs:
           PR_FILES="$PR_FILES_AM $PR_FILES_RENAMED"
           for FILE in $PR_FILES; do
               EXTENSION="${FILE##*.}"
-              if [[ $EXTENSION == 'py' ]]; then
+              DIRECTORY="${FILE%%/*}"
+              if [[ "$EXTENSION" == 'py' && "$DIRECTORY" != 'cvat-sdk' ]]; then
                   CHANGED_FILES+=" $FILE"
               fi
           done
@@ -32,7 +33,7 @@ jobs:
 
             echo "Bandit version: "$(bandit --version | head -1)
             echo "The files will be checked: "$(echo $CHANGED_FILES)
-            bandit $CHANGED_FILES --exclude '**/tests/**' -a file --ini ./.bandit -f html -o ./bandit_report/bandit_checks.html
+            bandit -a file --ini .bandit -f html -o ./bandit_report/bandit_checks.html $CHANGED_FILES
             deactivate
           else
             echo "No files with the \"py\" extension found"

.github/workflows/black.yml

@@ -0,0 +1,82 @@
+name: Black
+on: pull_request
+jobs:
+  Linter:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - id: files
+        uses: jitterbit/get-changed-files@v1
+        continue-on-error: true
+
+      - name: Run checks
+        env:
+          PR_FILES_AM: ${{ steps.files.outputs.added_modified }}
+          PR_FILES_RENAMED: ${{ steps.files.outputs.renamed }}
+        run: |
+          # If different modules use different Black configs,
+          # we need to run Black for each python component group separately.
+          # Otherwise, they all will use the same config.
+          ENABLED_DIRS=("cvat-sdk" "cvat-cli" "tests/python/sdk" "tests/python/cli")
+
+          isValueIn () {
+            # Checks if a value is in an array
+            # https://stackoverflow.com/a/8574392
+            # args: value, array
+            local e match="$1"
+            shift
+            for e; do
+              [[ "$e" == "$match" ]] && return 0;
+            done
+            return 1
+          }
+
+          startswith () {
+            # Inspired by https://stackoverflow.com/a/2172367
+            # Checks if the first arg starts with the second one
+            local value="$1"
+            local beginning="$2"
+            return $([[ $value == ${beginning}* ]])
+          }
+
+          PR_FILES="$PR_FILES_AM $PR_FILES_RENAMED"
+          UPDATED_DIRS=""
+          for FILE in $PR_FILES; do
+              EXTENSION="${FILE##*.}"
+              DIRECTORY="$(dirname $FILE)"
+              if [[ "$EXTENSION" == "py" ]]; then
+                  for EDIR in ${ENABLED_DIRS[@]}; do
+                      if startswith "${DIRECTORY}/" "${EDIR}/" && ! isValueIn "${EDIR}" ${UPDATED_DIRS[@]};
+                      then
+                          UPDATED_DIRS+=" ${EDIR}"
+                      fi
+                  done
+              fi
+          done
+
+          if [[ ! -z $UPDATED_DIRS ]]; then
+            sudo apt-get --no-install-recommends install -y build-essential curl python3-dev python3-pip python3-venv
+            python3 -m venv .env
+            . .env/bin/activate
+            pip install -U pip wheel setuptools
+            pip install $(egrep "black.*" ./cvat-cli/requirements/development.txt)
+            mkdir -p black_report
+
+            echo "Black version: "$(black --version)
+            echo "The dirs will be checked: $UPDATED_DIRS"
+            EXIT_CODE=0
+            for DIR in $UPDATED_DIRS; do
+              black --check $DIR >> ./black_report/black_checks.txt || EXIT_CODE=$(($? | $EXIT_CODE)) || true
+            done
+            deactivate
+            exit $EXIT_CODE
+          else
+            echo "No files with the \"py\" extension found"
+          fi
+
+      - name: Upload artifacts
+        if: failure()
+        uses: actions/upload-artifact@v2
+        with:
+          name: black_report
+          path: black_report

.github/workflows/cache.yml

@@ -6,27 +6,74 @@ on:
 
 jobs:
   Caching_CVAT:
+    env:
+      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      REPO: ${{ github.repository }}
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - uses: actions/cache@v2
+
+      - name: Getting SHA with cache from the default branch
+        id: get-sha
+        run: |
+          DEFAULT_BRANCH=$(gh api /repos/$REPO | jq -r '.default_branch')
+          for sha in $(gh api "/repos/$REPO/commits?per_page=100&sha=${DEFAULT_BRANCH}" | jq -r '.[].sha');
+          do
+            RUN_status=$(gh api /repos/${REPO}/actions/workflows/cache.yml/runs | \
+              jq -r ".workflow_runs[]? | select((.head_sha == \"${sha}\") and (.conclusion == \"success\")) | .status")
+
+            if [[ ${RUN_status} == "completed" ]]; then
+              SHA=$sha
+              break
+            fi
+          done
+
+          echo Default branch is ${DEFAULT_BRANCH}
+          echo Workflow will try to get cache from commit: ${SHA}
+
+          echo ::set-output name=default_branch::${DEFAULT_BRANCH}
+          echo ::set-output name=sha::${SHA}
+
+      - uses: actions/cache@v3
+        id: server-cache-action
         with:
           path: /tmp/cvat_cache_server
           key: ${{ runner.os }}-build-server-${{ github.sha }}
           restore-keys: |
+            ${{ runner.os }}-build-server-${{ steps.get-sha.outputs.sha }}
             ${{ runner.os }}-build-server-
 
-      - uses: actions/cache@v2
+      - uses: actions/cache@v3
+        id: ui-cache-action
         with:
           path: /tmp/cvat_cache_ui
           key: ${{ runner.os }}-build-ui-${{ github.sha }}
           restore-keys: |
+            ${{ runner.os }}-build-ui-${{ steps.get-sha.outputs.sha }}
             ${{ runner.os }}-build-ui-
 
+      - uses: actions/cache@v3
+        id: elasticsearch-cache-action
+        with:
+          path: /tmp/cvat_cache_elasticsearch
+          key: ${{ runner.os }}-build-elasticsearch-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-build-elasticsearch-${{ steps.get-sha.outputs.sha }}
+            ${{ runner.os }}-build-elasticsearch-
+
+      - uses: actions/cache@v3
+        id: logstash-cache-action
+        with:
+          path: /tmp/cvat_cache_logstash
+          key: ${{ runner.os }}-build-logstash-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-build-logstash-${{ steps.get-sha.outputs.sha }}
+            ${{ runner.os }}-build-logstash-
+
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1.1.2
+        uses: docker/setup-buildx-action@v2
 
-      - name: Caching CVAT server
+      - name: Caching CVAT Server
         uses: docker/build-push-action@v2
         with:
           context: .
@@ -42,9 +89,34 @@ jobs:
           cache-from: type=local,src=/tmp/cvat_cache_ui
           cache-to: type=local,dest=/tmp/cvat_cache_ui-new
 
+      - name: Caching CVAT Elasticsearch
+        uses: docker/build-push-action@v2
+        with:
+          context: ./components/analytics/elasticsearch/
+          file: ./components/analytics/elasticsearch/Dockerfile
+          cache-from: type=local,src=/tmp/cvat_cache_elasticsearch
+          cache-to: type=local,dest=/tmp/cvat_cache_elasticsearch-new
+          build-args: ELK_VERSION=6.8.23
+
+      - name: Caching CVAT Logstash
+        uses: docker/build-push-action@v2
+        with:
+          context: ./components/analytics/logstash/
+          file: ./components/analytics/logstash/Dockerfile
+          cache-from: type=local,src=/tmp/cvat_cache_logstash
+          cache-to: type=local,dest=/tmp/cvat_cache_logstash-new
+          build-args: ELK_VERSION=6.8.23
+
       - name: Moving cache
         run: |
           rm -rf /tmp/cvat_cache_server
           mv /tmp/cvat_cache_server-new /tmp/cvat_cache_server
+
           rm -rf /tmp/cvat_cache_ui
           mv /tmp/cvat_cache_ui-new /tmp/cvat_cache_ui
+
+          rm -rf /tmp/cvat_cache_elasticsearch
+          mv /tmp/cvat_cache_elasticsearch-new /tmp/cvat_cache_elasticsearch
+
+          rm -rf /tmp/cvat_cache_logstash
+          mv /tmp/cvat_cache_logstash-new /tmp/cvat_cache_logstash

.github/workflows/cleanup.yml

@@ -0,0 +1,21 @@
+# Workflow deletes image artifacts that created by CI workflow
+name: Delete image artifacts
+on:
+  workflow_run:
+    workflows: [CI, Comment]
+    types:
+      - completed
+
+jobs:
+  cleanup:
+    runs-on: ubuntu-latest
+    env:
+      GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+    steps:
+      - name: Clean up
+        run: |
+          wri=${{ github.event.workflow_run.id }}
+          for ai in $(gh api /repos/${{ github.repository }}/actions/runs/$wri/artifacts | jq '.artifacts[] | select( .name | startswith("cvat"))  | .id');
+          do
+            gh api --method DELETE /repos/${{ github.repository }}/actions/artifacts/$ai
+          done