diff --git a/.coveragerc b/.coveragerc index 1190ab96c730..e2bc8347b10a 100644 --- a/.coveragerc +++ b/.coveragerc @@ -1,6 +1,5 @@ [run] branch = true -# relative_files = true # does not work? source = cvat/apps/ diff --git a/.github/codecov.yml b/.github/codecov.yml new file mode 100644 index 000000000000..31402461a072 --- /dev/null +++ b/.github/codecov.yml @@ -0,0 +1,21 @@ +comment: + layout: "header, diff, components" + +component_management: + individual_components: + - component_id: cvat-ui + name: cvat-ui + paths: + - cvat-canvas/** + - cvat-canvas3d/** + - cvat-core/** + - cvat-data/** + - cvat-ui/** + - component_id: cvat-server + name: cvat-server + paths: + - cvat/** + - cvat-cli/** + - cvat-sdk/** + - utils/** + diff --git a/.github/workflows/full.yml b/.github/workflows/full.yml index 8ff4d2fb2a3b..5faf726e3676 100644 --- a/.github/workflows/full.yml +++ b/.github/workflows/full.yml @@ -52,12 +52,19 @@ jobs: - name: CVAT server. Build and push uses: docker/build-push-action@v3 with: + build-args: | + "COVERAGE_PROCESS_START=.coveragerc" cache-from: type=local,src=/tmp/cvat_cache_server context: . file: Dockerfile tags: cvat/server outputs: type=docker,dest=/tmp/cvat_server/image.tar + - name: Instrumentation of the code then rebuilding the CVAT UI + run: | + yarn --frozen-lockfile + yarn run coverage + - name: CVAT UI. Build and push uses: docker/build-push-action@v3 with: @@ -154,11 +161,20 @@ jobs: - name: Running REST API and SDK tests id: run_tests + env: + COVERAGE_PROCESS_START: ".coveragerc" run: | - pip3 install --user '/tmp/cvat_sdk/[pytorch]' - pip3 install --user cvat-cli/ - pip3 install --user -r tests/python/requirements.txt - pytest tests/python/ -s -v + pip3 install -r cvat-sdk/gen/requirements.txt + ./cvat-sdk/gen/generate.sh + + pip3 install -r ./tests/python/requirements.txt + pip3 install -e ./cvat-sdk + pip3 install -e ./cvat-cli + + pytest tests/python/ --cov --cov-report xml + + - name: Upload coverage reports to Codecov with GitHub Action + uses: codecov/codecov-action@v3 - name: Creating a log file from cvat containers if: failure() && steps.run_tests.conclusion == 'failure' @@ -218,10 +234,13 @@ jobs: while [[ $(curl -s -o /dev/null -w "%{http_code}" localhost:8181/health?bundles) != "200" && max_tries -gt 0 ]]; do (( max_tries-- )); sleep 5; done docker compose -f docker-compose.yml -f docker-compose.dev.yml -f docker-compose.ci.yml run cvat_ci /bin/bash \ - -c 'python manage.py test cvat/apps -v 2' + -c 'coverage run -a manage.py test cvat/apps && coverage json && mv coverage.json ${CONTAINER_COVERAGE_DATA_DIR}' docker compose -f docker-compose.yml -f docker-compose.dev.yml -f docker-compose.ci.yml run cvat_ci /bin/bash \ - -c 'yarn --frozen-lockfile --ignore-scripts && yarn workspace cvat-core run test' + -c 'yarn --frozen-lockfile --ignore-scripts && yarn workspace cvat-core run test && mv cvat-core/reports/coverage/coverage-final.json ${CONTAINER_COVERAGE_DATA_DIR}' + + - name: Upload coverage reports to Codecov with GitHub Action + uses: codecov/codecov-action@v3 - name: Creating a log file from cvat containers if: failure() @@ -317,16 +336,17 @@ jobs: npx cypress run \ --headed \ --browser chrome \ - --env coverage=false \ --config-file cypress_canvas3d.config.js \ --spec 'cypress/e2e/${{ matrix.specs }}/**/*.js,cypress/e2e/remove_users_tasks_projects_organizations.js' else npx cypress run \ --browser chrome \ - --env coverage=false \ --spec 'cypress/e2e/${{ matrix.specs }}/**/*.js,cypress/e2e/remove_users_tasks_projects_organizations.js' fi + - name: Upload coverage reports to Codecov with GitHub Action + uses: codecov/codecov-action@v3 + - name: Creating a log file from "cvat" container logs if: failure() run: | diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 71d0412a8514..63a94f25440e 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -52,12 +52,19 @@ jobs: - name: CVAT server. Build and push uses: docker/build-push-action@v3 with: + build-args: | + "COVERAGE_PROCESS_START=.coveragerc" cache-from: type=local,src=/tmp/cvat_cache_server context: . file: Dockerfile tags: cvat/server outputs: type=docker,dest=/tmp/cvat_server/image.tar + - name: Instrumentation of the code then rebuilding the CVAT UI + run: | + yarn --frozen-lockfile + yarn run coverage + - name: CVAT UI. Build and push uses: docker/build-push-action@v3 with: @@ -130,12 +137,6 @@ jobs: name: cvat_ui path: /tmp/cvat_ui/ - - name: Download CVAT SDK package - uses: actions/download-artifact@v3 - with: - name: cvat_sdk - path: /tmp/cvat_sdk/ - - name: Load Docker images run: | docker load --input /tmp/cvat_server/image.tar @@ -144,7 +145,6 @@ jobs: docker tag cvat/ui:latest cvat/ui:${CVAT_VERSION} docker image ls -a - - name: Upload expected schema as an artifact if: failure() && steps.verify_schema.conclusion == 'failure' uses: actions/upload-artifact@v3.1.2 @@ -154,11 +154,20 @@ jobs: - name: Running REST API and SDK tests id: run_tests + env: + COVERAGE_PROCESS_START: ".coveragerc" run: | - pip3 install --user '/tmp/cvat_sdk/[pytorch]' - pip3 install --user cvat-cli/ - pip3 install --user -r tests/python/requirements.txt - pytest tests/python/ -s -v + pip3 install -r cvat-sdk/gen/requirements.txt + ./cvat-sdk/gen/generate.sh + + pip3 install -r ./tests/python/requirements.txt + pip3 install -e ./cvat-sdk + pip3 install -e ./cvat-cli + + pytest tests/python/ --cov --cov-report xml + + - name: Upload coverage reports to Codecov with GitHub Action + uses: codecov/codecov-action@v3 - name: Creating a log file from cvat containers if: failure() && steps.run_tests.conclusion == 'failure' @@ -216,10 +225,13 @@ jobs: while [[ $(curl -s -o /dev/null -w "%{http_code}" localhost:8181/health?bundles) != "200" && max_tries -gt 0 ]]; do (( max_tries-- )); sleep 5; done docker compose -f docker-compose.yml -f docker-compose.dev.yml -f docker-compose.ci.yml run cvat_ci /bin/bash \ - -c 'python manage.py test cvat/apps -v 2' + -c 'coverage run -a manage.py test cvat/apps && coverage json && mv coverage.json ${CONTAINER_COVERAGE_DATA_DIR}' docker compose -f docker-compose.yml -f docker-compose.dev.yml -f docker-compose.ci.yml run cvat_ci /bin/bash \ - -c 'yarn --frozen-lockfile --ignore-scripts && yarn workspace cvat-core run test' + -c 'yarn --frozen-lockfile --ignore-scripts && yarn workspace cvat-core run test && mv cvat-core/reports/coverage/coverage-final.json ${CONTAINER_COVERAGE_DATA_DIR}' + + - name: Upload coverage reports to Codecov with GitHub Action + uses: codecov/codecov-action@v3 - name: Creating a log file from cvat containers if: failure() @@ -313,16 +325,17 @@ jobs: npx cypress run \ --headed \ --browser chrome \ - --env coverage=false \ --config-file cypress_canvas3d.config.js \ --spec 'cypress/e2e/${{ matrix.specs }}/**/*.js,cypress/e2e/remove_users_tasks_projects_organizations.js' else npx cypress run \ --browser chrome \ - --env coverage=false \ --spec 'cypress/e2e/${{ matrix.specs }}/**/*.js,cypress/e2e/remove_users_tasks_projects_organizations.js' fi + - name: Upload coverage reports to Codecov with GitHub Action + uses: codecov/codecov-action@v3 + - name: Creating a log file from "cvat" container logs if: failure() run: | diff --git a/.github/workflows/schedule.yml b/.github/workflows/schedule.yml index 097c1b5cc034..ab7297b56011 100644 --- a/.github/workflows/schedule.yml +++ b/.github/workflows/schedule.yml @@ -88,6 +88,8 @@ jobs: - name: CVAT server. Build and push uses: docker/build-push-action@v3 with: + build-args: | + "COVERAGE_PROCESS_START=.coveragerc" cache-from: type=local,src=/tmp/cvat_cache_server context: . file: Dockerfile @@ -163,14 +165,17 @@ jobs: ./opa test cvat/apps/iam/rules - name: REST API and SDK tests + env: + COVERAGE_PROCESS_START: ".coveragerc" run: | - pip3 install --user -r cvat-sdk/gen/requirements.txt + pip3 install -r cvat-sdk/gen/requirements.txt ./cvat-sdk/gen/generate.sh - pip3 install --user 'cvat-sdk/[pytorch]' - pip3 install --user cvat-cli/ - pip3 install --user -r tests/python/requirements.txt - pytest tests/python/ + pip3 install -r ./tests/python/requirements.txt + pip3 install -e ./cvat-sdk + pip3 install -e ./cvat-cli + + pytest tests/python/ --cov --cov-report xml pytest tests/python/ --stop-services - name: Unit tests @@ -183,20 +188,15 @@ jobs: while [[ $(curl -s -o /dev/null -w "%{http_code}" localhost:8181/health?bundles) != "200" && max_tries -gt 0 ]]; do (( max_tries-- )); sleep 5; done docker compose -f docker-compose.yml -f docker-compose.dev.yml -f docker-compose.ci.yml run cvat_ci /bin/bash \ - -c 'coverage run -a manage.py test cvat/apps && mv .coverage ${CONTAINER_COVERAGE_DATA_DIR}' + -c 'coverage run -a manage.py test cvat/apps && coverage json && mv coverage.json ${CONTAINER_COVERAGE_DATA_DIR}' docker compose -f docker-compose.yml -f docker-compose.dev.yml -f docker-compose.ci.yml run cvat_ci /bin/bash \ - -c 'yarn --frozen-lockfile --ignore-scripts && yarn workspace cvat-core run test' + -c 'yarn --frozen-lockfile --ignore-scripts && yarn workspace cvat-core run test && mv cvat-core/reports/coverage/coverage-final.json ${CONTAINER_COVERAGE_DATA_DIR}' docker compose -f docker-compose.yml -f docker-compose.dev.yml -f docker-compose.ci.yml down -v - - name: Uploading code coverage results as an artifact - uses: actions/upload-artifact@v3.1.1 - with: - name: coverage_results - path: | - ${{ github.workspace }}/lcov.info - ${{ github.workspace }}/.coverage + - name: Upload coverage reports to Codecov with GitHub Action + uses: codecov/codecov-action@v3 e2e_testing: needs: build @@ -304,14 +304,15 @@ jobs: --browser chrome \ --config-file cypress_canvas3d.config.js \ --spec 'cypress/e2e/${{ matrix.specs }}/**/*.js,cypress/e2e/remove_users_tasks_projects_organizations.js' - mv ./.nyc_output/out.json ./.nyc_output/out_${{ matrix.specs }}.json else npx cypress run \ --browser chrome \ --spec 'cypress/e2e/${{ matrix.specs }}/**/*.js,cypress/e2e/remove_users_tasks_projects_organizations.js' - mv ./.nyc_output/out.json ./.nyc_output/out_${{ matrix.specs }}.json fi + - name: Upload coverage reports to Codecov with GitHub Action + uses: codecov/codecov-action@v3 + - name: Creating a log file from "cvat" container logs if: failure() run: | @@ -330,72 +331,3 @@ jobs: with: name: cvat_container_logs path: ${{ github.workspace }}/tests/cvat.log - - - name: Uploading code coverage results as an artifact - uses: actions/upload-artifact@v3.1.1 - with: - name: coverage_results - path: ${{ github.workspace }}/tests/.nyc_output - - coveralls: - runs-on: ubuntu-latest - needs: [unit_testing, e2e_testing] - steps: - - uses: actions/checkout@v3 - - - name: CVAT server. Extract metadata (tags, labels) for Docker - id: meta-server - uses: docker/metadata-action@master - with: - images: ${{ secrets.DOCKERHUB_CI_WORKSPACE }}/${{ env.SERVER_IMAGE_TEST_REPO }} - tags: - type=raw,value=nightly - - - name: Login to Docker Hub - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKERHUB_CI_USERNAME }} - password: ${{ secrets.DOCKERHUB_CI_TOKEN }} - - - name: Pull CVAT server image - run: | - docker pull ${{ steps.meta-server.outputs.tags }} - docker tag ${{ steps.meta-server.outputs.tags }} cvat/server:local - - - name: Downloading coverage results - uses: actions/download-artifact@v2 - with: - name: coverage_results - - - name: Combining coverage results - run: | - mkdir -p ./nyc_output_tmp - mv ./out_*.json ./nyc_output_tmp - mkdir -p ./.nyc_output - yarn --frozen-lockfile - npx nyc merge ./nyc_output_tmp ./.nyc_output/out.json - - - name: Sending results to Coveralls - env: - HOST_COVERAGE_DATA_DIR: ${{ github.workspace }} - CONTAINER_COVERAGE_DATA_DIR: "/coverage_data" - COVERALLS_SERVICE_NAME: github - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - npx nyc report --reporter=text-lcov >> ${HOST_COVERAGE_DATA_DIR}/lcov.info - - docker compose \ - -f docker-compose.yml \ - -f docker-compose.dev.yml \ - -f docker-compose.ci.yml \ - run cvat_ci /bin/bash -c 'cd ${CONTAINER_COVERAGE_DATA_DIR} && coveralls-lcov -v -n lcov.info > ${CONTAINER_COVERAGE_DATA_DIR}/coverage.json' - - docker compose \ - -f docker-compose.yml \ - -f docker-compose.dev.yml \ - -f docker-compose.ci.yml \ - run cvat_ci /bin/bash -c '\ - ln -s ${CONTAINER_COVERAGE_DATA_DIR}/.git . \ - && ln -s ${CONTAINER_COVERAGE_DATA_DIR}/.coverage . \ - && ln -s ${CONTAINER_COVERAGE_DATA_DIR}/coverage.json . \ - && coveralls --merge=coverage.json' diff --git a/Dockerfile b/Dockerfile index 974aab6eb888..855aa61de5ba 100644 --- a/Dockerfile +++ b/Dockerfile @@ -178,6 +178,11 @@ COPY --chown=${USER} wait-for-it.sh manage.py backend_entrypoint.sh ${HOME}/ COPY --chown=${USER} utils/ ${HOME}/utils COPY --chown=${USER} cvat/ ${HOME}/cvat +ARG COVERAGE_PROCESS_START +RUN if [ "${COVERAGE_PROCESS_START}" ]; then \ + echo "import coverage; coverage.process_startup()" > /opt/venv/lib/python3.10/site-packages/coverage_subprocess.pth; \ + fi + # RUN all commands below as 'django' user USER ${USER} WORKDIR ${HOME} diff --git a/Dockerfile.ci b/Dockerfile.ci index 1dd4b0c279c0..0438e80bdcc6 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -25,9 +25,7 @@ RUN apt-get update && \ COPY cvat/requirements/ /tmp/cvat/requirements/ COPY utils/dataset_manifest/requirements.txt /tmp/utils/dataset_manifest/requirements.txt -RUN DATUMARO_HEADLESS=1 python3 -m pip install --no-cache-dir -r /tmp/cvat/requirements/${DJANGO_CONFIGURATION}.txt && \ - python3 -m pip install --no-cache-dir coveralls -RUN gem install coveralls-lcov +RUN DATUMARO_HEADLESS=1 python3 -m pip install --no-cache-dir -r /tmp/cvat/requirements/${DJANGO_CONFIGURATION}.txt COPY cvat-core ${HOME}/cvat-core COPY cvat-data ${HOME}/cvat-data diff --git a/README.md b/README.md index b038ba67d35c..1b99a09ebe80 100644 --- a/README.md +++ b/README.md @@ -256,8 +256,8 @@ questions and get our support. [ci-url]: https://github.com/opencv/cvat/actions [gitter-img]: https://img.shields.io/gitter/room/opencv-cvat/public?style=flat [gitter-url]: https://gitter.im/opencv-cvat -[coverage-img]: https://coveralls.io/repos/github/cvat-ai/cvat/badge.svg?branch=develop -[coverage-url]: https://coveralls.io/github/cvat-ai/cvat?branch=develop +[coverage-img]: https://codecov.io/github/opencv/cvat/branch/develop/graph/badge.svg +[coverage-url]: https://codecov.io/github/opencv/cvat [doi-img]: https://zenodo.org/badge/139156354.svg [doi-url]: https://zenodo.org/badge/latestdoi/139156354 [discord-img]: https://img.shields.io/discord/1000789942802337834?label=discord diff --git a/cvat-core/package.json b/cvat-core/package.json index 8f20feefc61f..4cd658b3df60 100644 --- a/cvat-core/package.json +++ b/cvat-core/package.json @@ -6,7 +6,6 @@ "scripts": { "build": "webpack", "test": "jest --config=jest.config.js --coverage", - "coveralls": "cat ./reports/coverage/lcov.info | coveralls", "type-check": "tsc --noEmit", "type-check:watch": "yarn run type-check --watch" }, @@ -19,7 +18,6 @@ "> 2%" ], "devDependencies": { - "coveralls": "^3.0.5", "jest": "^29.5.0", "jest-config": "^29.5.0", "jest-environment-jsdom": "^29.5.0", diff --git a/cvat/requirements/production.in b/cvat/requirements/production.in index 530545b6a4c0..c349f996c947 100644 --- a/cvat/requirements/production.in +++ b/cvat/requirements/production.in @@ -1,2 +1,4 @@ -r base.in uvicorn[standard]==0.22.0 +coverage==7.2.3 # to collect the coverage for REST API tests. +# It will be enabled only if COVERAGE_PROCESS_START env is defined. diff --git a/cvat/requirements/production.txt b/cvat/requirements/production.txt index a142cfdc6af2..3899e8286174 100644 --- a/cvat/requirements/production.txt +++ b/cvat/requirements/production.txt @@ -26,5 +26,7 @@ watchfiles==0.19.0 # via uvicorn websockets==11.0.3 # via uvicorn +coverage==7.2.3 + # via -r cvat/requirements/production.in # The following packages are considered to be unsafe in a requirements file: diff --git a/cvat/requirements/testing.in b/cvat/requirements/testing.in index ff5ff5ca91dd..e2c70b149ea2 100644 --- a/cvat/requirements/testing.in +++ b/cvat/requirements/testing.in @@ -1,3 +1,3 @@ -r development.in -coveralls fakeredis==2.10.3 +coverage==7.2.3 diff --git a/cvat/requirements/testing.txt b/cvat/requirements/testing.txt index 6de652fc0d2a..a868cb1c6b0c 100644 --- a/cvat/requirements/testing.txt +++ b/cvat/requirements/testing.txt @@ -1,4 +1,4 @@ -# SHA1:8dec2a1cde7e68bd79ee31cdec270084f7e3266b +# SHA1:910e8edd8fcfdbe7c9a7278ba499bcfad1313c19 # # This file is autogenerated by pip-compile-multi # To update, run: @@ -8,15 +8,11 @@ -r development.txt --no-binary av -coverage==6.5.0 - # via coveralls -coveralls==3.3.1 - # via -r cvat/requirements/testing.in -docopt==0.6.2 - # via coveralls fakeredis==2.10.3 # via -r cvat/requirements/testing.in sortedcontainers==2.4.0 # via fakeredis +coverage==7.2.3 + # via -r cvat/requirements/testing.in # The following packages are considered to be unsafe in a requirements file: diff --git a/cvat/rqworker.py b/cvat/rqworker.py index 2d68a419f95b..78319ff2e841 100644 --- a/cvat/rqworker.py +++ b/cvat/rqworker.py @@ -1,8 +1,10 @@ # Copyright (C) 2018-2022 Intel Corporation -# Copyright (C) 2022 CVAT.ai Corporation +# Copyright (C) 2022-2023 CVAT.ai Corporation # # SPDX-License-Identifier: MIT +import os + from rq import Worker import cvat.utils.remote_debugger as debug @@ -62,3 +64,15 @@ def execute_job(self, *args, **kwargs): return super().execute_job(*args, **kwargs) DefaultWorker = RemoteDebugWorker + + +if os.environ.get("COVERAGE_PROCESS_START"): + import coverage + + def coverage_exit(): + cov = coverage.Coverage.current() + cov.stop() + cov.save() + os._exit(0) + + os._exit = coverage_exit diff --git a/docker-compose.ci.yml b/docker-compose.ci.yml index bc1628548c77..177d40011498 100644 --- a/docker-compose.ci.yml +++ b/docker-compose.ci.yml @@ -14,7 +14,6 @@ services: depends_on: - cvat_server environment: - COVERALLS_SERVICE_NAME: CONTAINER_COVERAGE_DATA_DIR: GITHUB_ACTIONS: GITHUB_TOKEN: diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 03a16b43b138..01d8d2f6bcb6 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -19,11 +19,13 @@ services: socks_proxy: CLAM_AV: CVAT_DEBUG_ENABLED: + COVERAGE_PROCESS_START: environment: CVAT_DEBUG_ENABLED: '${CVAT_DEBUG_ENABLED:-no}' CVAT_DEBUG_PORT: '9090' # If 'yes', wait for a debugger connection on startup CVAT_DEBUG_WAIT: '${CVAT_DEBUG_WAIT_CLIENT:-no}' + COVERAGE_PROCESS_START: ports: - '9090:9090' @@ -35,6 +37,7 @@ services: # NUMPROCS: 1 CVAT_DEBUG_ENABLED: '${CVAT_DEBUG_ENABLED:-no}' CVAT_DEBUG_PORT: '9092' + COVERAGE_PROCESS_START: ports: - '9092:9092' @@ -46,6 +49,7 @@ services: # NUMPROCS: 1 CVAT_DEBUG_ENABLED: '${CVAT_DEBUG_ENABLED:-no}' CVAT_DEBUG_PORT: '9093' + COVERAGE_PROCESS_START: ports: - '9093:9093' @@ -57,6 +61,7 @@ services: # NUMPROCS: 1 CVAT_DEBUG_ENABLED: '${CVAT_DEBUG_ENABLED:-no}' CVAT_DEBUG_PORT: '9091' + COVERAGE_PROCESS_START: ports: - '9091:9091' diff --git a/package.json b/package.json index 6cf2be82717c..14efb012c4ae 100644 --- a/package.json +++ b/package.json @@ -35,7 +35,6 @@ "babel-register": "^6.26.0", "bundle-declarations-webpack-plugin": "^3.1.0", "copy-webpack-plugin": "^11.0.0", - "coveralls": "^3.1.0", "css-loader": "^3.4.2", "eslint": "^7.11.0", "eslint-config-airbnb": "^18.0.1", @@ -89,9 +88,9 @@ "scripts": { "test": "echo \"Error: no test specified\" && exit 1", "coverage": "yarn run instrument && yarn run cp && yarn run rm", - "instrument": "nyc instrument cvat-ui cvat-ui_cov && nyc instrument cvat-canvas cvat-canvas_cov && nyc instrument cvat-data cvat-data_cov && nyc instrument cvat-core cvat-core_cov", - "cp": "cp -r cvat-ui_cov/* cvat-ui && cp -r cvat-canvas_cov/* cvat-canvas && cp -r cvat-data_cov/* cvat-data && cp -r cvat-core_cov/* cvat-core", - "rm": "rm -rf cvat-ui_cov cvat-canvas_cov cvat-data_cov cvat-core_cov", + "instrument": "nyc instrument cvat-ui cvat-ui_cov && nyc instrument cvat-canvas cvat-canvas_cov && nyc instrument cvat-canvas3d cvat-canvas3d_cov && nyc instrument cvat-data cvat-data_cov && nyc instrument cvat-core cvat-core_cov", + "cp": "cp -r cvat-ui_cov/* cvat-ui && cp -r cvat-canvas_cov/* cvat-canvas && cp -r cvat-canvas3d_cov/* cvat-canvas3d && cp -r cvat-data_cov/* cvat-data && cp -r cvat-core_cov/* cvat-core", + "rm": "rm -rf cvat-ui_cov cvat-canvas_cov cvat-canvas3d_cov cvat-data_cov cvat-core_cov", "prepare": "husky install && rm .husky/pre-commit; npx husky add .husky/pre-commit \"npx lint-staged\"", "precommit:cvat-tests": "cd tests && eslint --fix", "precommit:cvat-data": "cd cvat-data && eslint --fix", diff --git a/site/content/en/docs/contributing/running-tests.md b/site/content/en/docs/contributing/running-tests.md index 0363f11e08ee..ccebc46640e7 100644 --- a/site/content/en/docs/contributing/running-tests.md +++ b/site/content/en/docs/contributing/running-tests.md @@ -29,6 +29,12 @@ description: 'Instructions on how to run all existence tests.' yarn --frozen-lockfile ``` +If you want to get a code coverage report, instrument the code: +``` +yarn --frozen-lockfile +yarn run coverage +``` + **Running tests** ``` @@ -74,6 +80,11 @@ If you need to rebuild your CVAT images add `--rebuild` option: pytest ./tests/python --rebuild ``` +If you want to get a code coverage report, use special option for it: +``` +COVERAGE_PROCESS_START=.coveragerc pytest ./tests/python --rebuild --cov --cov-report xml +``` + **Debugging** Currently, this is only supported in deployments based on Docker Compose, @@ -146,6 +157,12 @@ Extra options: ``` python manage.py test --settings cvat.settings.testing cvat/apps -v 2 ``` + +If you want to get a code coverage report, run the next command: + ``` + coverage run manage.py test --settings cvat.settings.testing cvat/apps -v 2 + ``` + 1. JS tests ``` cd cvat-core diff --git a/tests/python/.coveragerc b/tests/python/.coveragerc new file mode 100644 index 000000000000..2e17038a15d0 --- /dev/null +++ b/tests/python/.coveragerc @@ -0,0 +1,40 @@ +[run] +branch = true +parallel = true +sigterm = true +concurrency=thread + +source = + cvat/apps/ + cvat-sdk/ + cvat-cli/ + utils/dataset_manifest + +omit = + cvat/settings/* + */tests/* + */test_* + */_test_* + */migrations/* + +[report] +exclude_lines = + # Have to re-enable the standard pragma + pragma: no cover + + # Don't complain about missing debug-only code: + def __repr__ + if\s+[\w\.()]+\.isEnabledFor\(log\.DEBUG\): + + # Don't complain if tests don't hit defensive assertion code: + raise AssertionError + raise NotImplementedError + + # Don't complain if non-runnable code isn't run: + if 0: + if __name__ == .__main__.: + +# don't fail on the code that can be found +ignore_errors = true + +skip_empty = true diff --git a/tests/python/requirements.txt b/tests/python/requirements.txt index 89af8413f6be..b41295b04a42 100644 --- a/tests/python/requirements.txt +++ b/tests/python/requirements.txt @@ -1,6 +1,7 @@ pytest==6.2.5 pytest-cases==3.6.13 pytest-timeout==2.1.0 +pytest-cov==4.0.0 requests==2.31.0 deepdiff==5.6.0 boto3==1.17.61 diff --git a/tests/python/rest_api/test_tasks.py b/tests/python/rest_api/test_tasks.py index 9300dafd2731..56e5478223fe 100644 --- a/tests/python/rest_api/test_tasks.py +++ b/tests/python/rest_api/test_tasks.py @@ -1780,7 +1780,7 @@ def _delete_annotations(self, task_id): (_, response) = api_client.tasks_api.destroy_annotations(id=task_id) assert response.status == HTTPStatus.NO_CONTENT - @pytest.mark.timeout(64) + @pytest.mark.timeout(70) @pytest.mark.parametrize("successful_upload", [True, False]) def test_can_import_annotations_after_previous_unclear_import( self, successful_upload: bool, tasks_with_shapes @@ -1815,7 +1815,7 @@ def test_can_import_annotations_after_previous_unclear_import( rq_id = json.loads(response.data)["rq_id"] assert rq_id else: - required_time = 54 + required_time = 60 uploader._tus_start_upload(url, query_params=params) uploader._upload_file_data_with_tus( url, filename, meta=params, logger=self.client.logger.debug @@ -1828,7 +1828,7 @@ def test_can_import_annotations_after_previous_unclear_import( task.import_annotations(self.format, filename) self._check_annotations(task_id) - @pytest.mark.timeout(64) + @pytest.mark.timeout(70) def test_check_import_cache_after_previous_interrupted_upload(self, tasks_with_shapes, request): task_id = tasks_with_shapes[0]["id"] with NamedTemporaryFile() as f: diff --git a/tests/python/shared/fixtures/init.py b/tests/python/shared/fixtures/init.py index 792fe36825f2..dd6d7aafdc43 100644 --- a/tests/python/shared/fixtures/init.py +++ b/tests/python/shared/fixtures/init.py @@ -1,9 +1,10 @@ -# Copyright (C) 2022 CVAT.ai Corporation +# Copyright (C) 2022-2023 CVAT.ai Corporation # # SPDX-License-Identifier: MIT import logging import os +from enum import Enum from http import HTTPStatus from pathlib import Path from subprocess import PIPE, CalledProcessError, run @@ -32,6 +33,23 @@ ] + CONTAINER_NAME_FILES +class Container(str, Enum): + DB = "cvat_db" + SERVER = "cvat_server" + WORKER_ANNOTATION = "cvat_worker_annotation" + WORKER_IMPORT = "cvat_worker_import" + WORKER_EXPORT = "cvat_worker_export" + WORKER_WEBHOOKS = "cvat_worker_webhooks" + UTILS = "cvat_utils" + + def __str__(self): + return self.value + + @classmethod + def covered(cls): + return [item.value for item in cls if item != cls.DB] + + def pytest_addoption(parser): group = parser.getgroup("CVAT REST API testing options") group._addoption( @@ -75,14 +93,13 @@ def pytest_addoption(parser): def _run(command, capture_output=True): _command = command.split() if isinstance(command, str) else command - try: stdout, stderr = "", "" if capture_output: proc = run(_command, check=True, stdout=PIPE, stderr=PIPE) # nosec stdout, stderr = proc.stdout.decode(), proc.stderr.decode() else: - proc = run(_command, check=True) # nosec + proc = run(_command) # nosec return stdout, stderr except CalledProcessError as exc: stderr = exc.stderr.decode() or exc.stdout.decode() if capture_output else "see above" @@ -120,6 +137,10 @@ def kube_cp(source, target): _run(f"kubectl cp {source} {target}") +def docker_exec(container, command, capture_output=True): + return _run(f"docker exec -u root {PREFIX}_{container}_1 {command}", capture_output) + + def docker_exec_cvat(command: Union[List[str], str]): base = f"docker exec {PREFIX}_cvat_server_1" _command = f"{base} {command}" if isinstance(command, str) else base.split() + command @@ -133,10 +154,6 @@ def kube_exec_cvat(command: Union[List[str], str]): return _run(_command) -def docker_exec_cvat_db(command): - _run(f"docker exec {PREFIX}_cvat_db_1 {command}") - - def kube_exec_cvat_db(command): pod_name = _kube_get_db_pod_name() _run(["kubectl", "exec", pod_name, "--"] + command) @@ -152,7 +169,9 @@ def kube_exec_clickhouse_db(command): def docker_restore_db(): - docker_exec_cvat_db("psql -U root -d postgres -v from=test_db -v to=cvat -f /tmp/restore.sql") + docker_exec( + Container.DB, "psql -U root -d postgres -v from=test_db -v to=cvat -f /tmp/restore.sql" + ) def kube_restore_db(): @@ -215,10 +234,17 @@ def create_compose_files(container_name_files): for service_name, service_config in dc_config["services"].items(): service_config.pop("container_name", None) - if service_name in ("cvat_server", "cvat_utils"): + if service_name in (Container.SERVER, Container.UTILS): service_env = service_config["environment"] service_env["DJANGO_SETTINGS_MODULE"] = "cvat.settings.testing_rest" + if service_name in Container.covered(): + service_env = service_config["environment"] + service_env["COVERAGE_PROCESS_START"] = ".coveragerc" + service_config["volumes"].append( + "./tests/python/.coveragerc:/home/django/.coveragerc" + ) + yaml.dump(dc_config, ndcf) @@ -252,7 +278,7 @@ def docker_restore_data_volumes(): CVAT_DB_DIR / "cvat_data.tar.bz2", f"{PREFIX}_cvat_server_1:/tmp/cvat_data.tar.bz2", ) - docker_exec_cvat("tar --strip 3 -xjf /tmp/cvat_data.tar.bz2 -C /home/django/data/") + docker_exec(Container.SERVER, "tar --strip 3 -xjf /tmp/cvat_data.tar.bz2 -C /home/django/data/") def kube_restore_data_volumes(): @@ -376,8 +402,10 @@ def local_start(start, stop, dumpdb, cleanup, rebuild, cvat_root_dir, cvat_db_di docker_cp(cvat_db_dir / "data.json", f"{PREFIX}_cvat_server_1:/tmp/data.json") wait_for_services() - docker_exec_cvat("python manage.py loaddata /tmp/data.json") - docker_exec_cvat_db("psql -U root -d postgres -v from=cvat -v to=test_db -f /tmp/restore.sql") + docker_exec(Container.SERVER, "python manage.py loaddata /tmp/data.json") + docker_exec( + Container.DB, "psql -U root -d postgres -v from=cvat -v to=test_db -f /tmp/restore.sql" + ) if start: pytest.exit("All necessary containers have been created and started.", returncode=0) @@ -408,18 +436,45 @@ def pytest_sessionstart(session: pytest.Session) -> None: def pytest_sessionfinish(session: pytest.Session, exitstatus: int) -> None: + session_finish(session) + + +def session_finish(session): if session.config.getoption("--collect-only"): return platform = session.config.getoption("--platform") if platform == "local": + if os.environ.get("COVERAGE_PROCESS_START"): + collect_code_coverage_from_containers() + docker_restore_db() - docker_exec_cvat_db("dropdb test_db") + docker_exec(Container.DB, "dropdb test_db") + + docker_exec(Container.DB, "dropdb --if-exists cvat") + docker_exec(Container.DB, "createdb cvat") + docker_exec(Container.SERVER, "python manage.py migrate") - docker_exec_cvat_db("dropdb --if-exists cvat") - docker_exec_cvat_db("createdb cvat") - docker_exec_cvat("python manage.py migrate") + +def collect_code_coverage_from_containers(): + for container in Container.covered(): + process_command = "python3" + + # find process with code coverage + pid, _ = docker_exec(container, f"pidof {process_command} -o 1") + + # stop process with code coverage + docker_exec(container, f"kill -15 {pid}") + sleep(3) + + # get code coverage report + docker_exec(container, "coverage combine", capture_output=False) + docker_exec(container, "coverage xml", capture_output=False) + docker_cp( + f"{PREFIX}_{container}_1:home/django/coverage.xml", + f"coverage_{container}.xml", + ) @pytest.fixture(scope="function") diff --git a/yarn.lock b/yarn.lock index 18636dc1c78d..85117e4f1f9e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3733,17 +3733,6 @@ cosmiconfig@^7.0.0: path-type "^4.0.0" yaml "^1.10.0" -coveralls@^3.0.5, coveralls@^3.1.0: - version "3.1.1" - resolved "https://registry.yarnpkg.com/coveralls/-/coveralls-3.1.1.tgz#f5d4431d8b5ae69c5079c8f8ca00d64ac77cf081" - integrity sha512-+dxnG2NHncSD1NrqbSM3dn/lE57O6Qf/koe9+I7c+wzkqRmEvcp0kgJdxKInzYzkICKkFMZsX3Vct3++tsF9ww== - dependencies: - js-yaml "^3.13.1" - lcov-parse "^1.0.0" - log-driver "^1.2.7" - minimist "^1.2.5" - request "^2.88.2" - cross-spawn@^7.0.0, cross-spawn@^7.0.2, cross-spawn@^7.0.3: version "7.0.3" resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-7.0.3.tgz#f73a85b9d5d41d045551c177e2882d4ac85728a6"