Skip to content

Commit 315c9cc

Browse files
author
Nikita Manovich
committed
Merge branch 'release-1.4.0'
2 parents c7033a7 + 3ca86bc commit 315c9cc

File tree

285 files changed

+35907
-5249
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

285 files changed

+35907
-5249
lines changed

‎.github/workflows/cache.yml

+50
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
name: Cache
2+
on:
3+
push:
4+
branches:
5+
- 'develop'
6+
7+
jobs:
8+
Caching_CVAT:
9+
runs-on: ubuntu-latest
10+
steps:
11+
- uses: actions/checkout@v2
12+
- uses: actions/cache@v2
13+
with:
14+
path: /tmp/cvat_cache_server
15+
key: ${{ runner.os }}-build-server-${{ github.sha }}
16+
restore-keys: |
17+
${{ runner.os }}-build-server-
18+
19+
- uses: actions/cache@v2
20+
with:
21+
path: /tmp/cvat_cache_ui
22+
key: ${{ runner.os }}-build-ui-${{ github.sha }}
23+
restore-keys: |
24+
${{ runner.os }}-build-ui-
25+
26+
- name: Set up Docker Buildx
27+
uses: docker/[email protected]
28+
29+
- name: Caching CVAT server
30+
uses: docker/build-push-action@v2
31+
with:
32+
context: .
33+
file: ./Dockerfile
34+
cache-from: type=local,src=/tmp/cvat_cache_server
35+
cache-to: type=local,dest=/tmp/cvat_cache_server-new
36+
37+
- name: Caching CVAT UI
38+
uses: docker/build-push-action@v2
39+
with:
40+
context: .
41+
file: ./Dockerfile.ui
42+
cache-from: type=local,src=/tmp/cvat_cache_ui
43+
cache-to: type=local,dest=/tmp/cvat_cache_ui-new
44+
45+
- name: Moving cache
46+
run: |
47+
rm -rf /tmp/cvat_cache_server
48+
mv /tmp/cvat_cache_server-new /tmp/cvat_cache_server
49+
rm -rf /tmp/cvat_cache_ui
50+
mv /tmp/cvat_cache_ui-new /tmp/cvat_cache_ui

‎.github/workflows/cancel.yml

+17
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
name: Cancelling Duplicates
2+
on:
3+
workflow_run:
4+
workflows: ['CI']
5+
types: ['requested']
6+
7+
jobs:
8+
cancel-duplicate-workflow-runs:
9+
name: "Cancel duplicate workflow runs"
10+
runs-on: ubuntu-latest
11+
steps:
12+
- uses: potiuk/cancel-workflow-runs@master
13+
name: "Cancel duplicate workflow runs"
14+
with:
15+
cancelMode: duplicates
16+
token: ${{ secrets.GITHUB_TOKEN }}
17+
sourceRunId: ${{ github.event.workflow_run.id }}

‎.github/workflows/hadolint.yml

+49
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
name: Linter
2+
on: pull_request
3+
jobs:
4+
HadoLint:
5+
runs-on: ubuntu-latest
6+
steps:
7+
- uses: actions/checkout@v2
8+
9+
- name: Run checks
10+
env:
11+
HADOLINT: "${{ github.workspace }}/hadolint"
12+
HADOLINT_VER: "2.1.0"
13+
VERIFICATION_LEVEL: "error"
14+
run: |
15+
URL="https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}/files"
16+
PR_FILES=$(curl -s -X GET -G $URL | jq -r '.[] | select(.status != "removed") | .filename')
17+
for file in $PR_FILES; do
18+
if [[ ${file} =~ 'Dockerfile' ]]; then
19+
changed_dockerfiles+=" ${file}"
20+
fi
21+
done
22+
23+
if [[ ! -z ${changed_dockerfiles} ]]; then
24+
curl -sL -o ${HADOLINT} "https://github.com/hadolint/hadolint/releases/download/v${HADOLINT_VER}/hadolint-Linux-x86_64" && chmod 700 ${HADOLINT}
25+
echo "HadoLint version: "`${HADOLINT} --version`
26+
echo "The files will be checked: "`echo ${changed_dockerfiles}`
27+
mkdir -p hadolint_report
28+
29+
${HADOLINT} --no-fail --format json ${changed_dockerfiles} > ./hadolint_report/hadolint_report.json
30+
get_verification_level=`cat ./hadolint_report/hadolint_report.json | jq -r '.[] | .level'`
31+
for line in ${get_verification_level}; do
32+
if [[ ${line} =~ ${VERIFICATION_LEVEL} ]]; then
33+
pip install json2html
34+
python ./tests/json_to_html.py ./hadolint_report/hadolint_report.json
35+
exit 1
36+
else
37+
exit 0
38+
fi
39+
done
40+
else
41+
echo "No files with the \"Dockerfile*\" name found"
42+
fi
43+
44+
- name: Upload artifacts
45+
if: failure()
46+
uses: actions/upload-artifact@v2
47+
with:
48+
name: hadolint_report
49+
path: hadolint_report

‎.github/workflows/main.yml

+211-23
Original file line numberDiff line numberDiff line change
@@ -8,50 +8,238 @@ on:
88
branches:
99
- '*'
1010
jobs:
11-
build:
12-
runs-on: ubuntu-latest
13-
steps:
11+
Unit_testing:
12+
runs-on: ubuntu-latest
13+
steps:
1414
- uses: actions/checkout@v2
15-
- uses: actions/setup-node@v2
15+
- name: Getting SHA from the default branch
16+
id: get-sha
17+
run: |
18+
URL_get_default_branch="https://api.github.com/repos/${{ github.repository }}"
19+
DEFAULT_BRANCH=$(curl -s -X GET -G ${URL_get_default_branch} | jq -r '.default_branch')
20+
URL_get_sha_default_branch="https://api.github.com/repos/${{ github.repository }}/git/ref/heads/${DEFAULT_BRANCH}"
21+
SHA=$(curl -s -X GET -G ${URL_get_sha_default_branch} | jq .object.sha | tr -d '"')
22+
echo ::set-output name=default_branch::${DEFAULT_BRANCH}
23+
echo ::set-output name=sha::${SHA}
24+
- name: Waiting a cache creation in the default branch
25+
run: |
26+
URL_runs="https://api.github.com/repos/${{ github.repository }}/actions/workflows/cache.yml/runs"
27+
SLEEP=45
28+
NUMBER_ATTEMPTS=10
29+
while [[ ${NUMBER_ATTEMPTS} -gt 0 ]]; do
30+
RUN_status=$(curl -s -X GET -G ${URL_runs} | jq -r '.workflow_runs[]? | select((.head_sha == "${{ steps.get-sha.outputs.sha }}") and (.event == "push") and (.name == "Cache") and (.head_branch == "${{ steps.get-sha.outputs.default_branch }}")) | .status')
31+
if [[ ${RUN_status} == "completed" ]]; then
32+
echo "The cache creation on the '${{ steps.get-sha.outputs.default_branch }}' branch has finished. Status: ${RUN_status}"
33+
break
34+
else
35+
echo "The creation of the cache is not yet complete."
36+
echo "There are still attempts to check the cache: ${NUMBER_ATTEMPTS}"
37+
echo "Status of caching in the '${{ steps.get-sha.outputs.default_branch }}' branch: ${RUN_status}"
38+
echo "sleep ${SLEEP}"
39+
sleep ${SLEEP}
40+
((NUMBER_ATTEMPTS--))
41+
fi
42+
done
43+
if [[ ${NUMBER_ATTEMPTS} -eq 0 ]]; then
44+
echo "Number of attempts expired!"
45+
echo "Probably the creation of the cache is not yet complete. Will continue working without the cache."
46+
fi
47+
- name: Getting CVAT server cache from the default branch
48+
uses: actions/cache@v2
1649
with:
17-
node-version: 12
18-
- name: Build CVAT
50+
path: /tmp/cvat_cache_server
51+
key: ${{ runner.os }}-build-server-${{ steps.get-sha.outputs.sha }}
52+
- name: Set up Docker Buildx
53+
uses: docker/[email protected]
54+
- name: Building CVAT server image
55+
uses: docker/build-push-action@v2
56+
with:
57+
context: .
58+
file: ./Dockerfile
59+
cache-from: type=local,src=/tmp/cvat_cache_server
60+
tags: openvino/cvat_server:latest
61+
load: true
62+
- name: Runing unit tests
1963
env:
2064
HOST_COVERAGE_DATA_DIR: ${{ github.workspace }}
21-
CONTAINER_COVERAGE_DATA_DIR: '/coverage_data'
22-
DJANGO_SU_NAME: 'admin'
23-
DJANGO_SU_EMAIL: '[email protected]'
24-
DJANGO_SU_PASSWORD: '12qwaszx'
65+
CONTAINER_COVERAGE_DATA_DIR: "/coverage_data"
2566
run: |
26-
docker-compose -f docker-compose.yml -f docker-compose.dev.yml -f docker-compose.ci.yml build
2767
docker-compose -f docker-compose.yml -f docker-compose.dev.yml -f docker-compose.ci.yml run cvat_ci /bin/bash -c 'coverage run -a manage.py test cvat/apps utils/cli && mv .coverage ${CONTAINER_COVERAGE_DATA_DIR}'
2868
docker-compose -f docker-compose.yml -f docker-compose.dev.yml -f docker-compose.ci.yml run cvat_ci /bin/bash -c 'cd cvat-data && npm ci && cd ../cvat-core && npm ci && npm run test && mv ./reports/coverage/lcov.info ${CONTAINER_COVERAGE_DATA_DIR} && chmod a+rwx ${CONTAINER_COVERAGE_DATA_DIR}/lcov.info'
29-
docker-compose up -d
30-
docker exec -i cvat /bin/bash -c "echo \"from django.contrib.auth.models import User; User.objects.create_superuser('${DJANGO_SU_NAME}', '${DJANGO_SU_EMAIL}', '${DJANGO_SU_PASSWORD}')\" | python3 ~/manage.py shell"
31-
- name: Code instrumentation
69+
- name: Uploading code coverage results as an artifact
70+
if: github.ref == 'refs/heads/develop'
71+
uses: actions/upload-artifact@v2
72+
with:
73+
name: coverage_results
74+
path: |
75+
${{ github.workspace }}/.coverage
76+
${{ github.workspace }}/lcov.info
77+
78+
E2E_testing:
79+
runs-on: ubuntu-latest
80+
strategy:
81+
fail-fast: false
82+
matrix:
83+
specs: ['actions_tasks', 'actions_tasks2', 'actions_tasks3', 'actions_objects', 'actions_objects2', 'actions_users', 'actions_projects', 'canvas3d_functionality', 'issues_prs', 'issues_prs2']
84+
steps:
85+
- uses: actions/checkout@v2
86+
- name: Getting SHA from the default branch
87+
id: get-sha
88+
run: |
89+
URL_get_default_branch="https://api.github.com/repos/${{ github.repository }}"
90+
DEFAULT_BRANCH=$(curl -s -X GET -G ${URL_get_default_branch} | jq -r '.default_branch')
91+
URL_get_sha_default_branch="https://api.github.com/repos/${{ github.repository }}/git/ref/heads/${DEFAULT_BRANCH}"
92+
SHA=$(curl -s -X GET -G ${URL_get_sha_default_branch} | jq .object.sha | tr -d '"')
93+
echo ::set-output name=default_branch::${DEFAULT_BRANCH}
94+
echo ::set-output name=sha::${SHA}
95+
- name: Waiting a cache creation in the default branch
96+
run: |
97+
URL_runs="https://api.github.com/repos/${{ github.repository }}/actions/workflows/cache.yml/runs"
98+
SLEEP=45
99+
NUMBER_ATTEMPTS=10
100+
while [[ ${NUMBER_ATTEMPTS} -gt 0 ]]; do
101+
RUN_status=$(curl -s -X GET -G ${URL_runs} | jq -r '.workflow_runs[]? | select((.head_sha == "${{ steps.get-sha.outputs.sha }}") and (.event == "push") and (.name == "Cache") and (.head_branch == "${{ steps.get-sha.outputs.default_branch }}")) | .status')
102+
if [[ ${RUN_status} == "completed" ]]; then
103+
echo "The cache creation on the '${{ steps.get-sha.outputs.default_branch }}' branch has finished. Status: ${RUN_status}"
104+
break
105+
else
106+
echo "The creation of the cache is not yet complete."
107+
echo "There are still attempts to check the cache: ${NUMBER_ATTEMPTS}"
108+
echo "Status of caching in the '${{ steps.get-sha.outputs.default_branch }}' branch: ${RUN_status}"
109+
echo "sleep ${SLEEP}"
110+
sleep ${SLEEP}
111+
((NUMBER_ATTEMPTS--))
112+
fi
113+
done
114+
if [[ ${NUMBER_ATTEMPTS} -eq 0 ]]; then
115+
echo "Number of attempts expired!"
116+
echo "Probably the creation of the cache is not yet complete. Will continue working without the cache."
117+
fi
118+
- name: Getting CVAT server cache from the default branch
119+
uses: actions/cache@v2
120+
with:
121+
path: /tmp/cvat_cache_server
122+
key: ${{ runner.os }}-build-server-${{ steps.get-sha.outputs.sha }}
123+
- name: Getting cache CVAT UI from the default branch
124+
uses: actions/cache@v2
125+
with:
126+
path: /tmp/cvat_cache_ui
127+
key: ${{ runner.os }}-build-ui-${{ steps.get-sha.outputs.sha }}
128+
- uses: actions/setup-node@v2
129+
with:
130+
node-version: 12
131+
- name: Set up Docker Buildx
132+
uses: docker/[email protected]
133+
- name: Building CVAT server image
134+
uses: docker/build-push-action@v2
135+
with:
136+
context: .
137+
file: ./Dockerfile
138+
cache-from: type=local,src=/tmp/cvat_cache_server
139+
tags: openvino/cvat_server:latest
140+
load: true
141+
- name: Building CVAT UI image
142+
uses: docker/build-push-action@v2
143+
with:
144+
context: .
145+
file: ./Dockerfile.ui
146+
cache-from: type=local,src=/tmp/cvat_cache_ui
147+
tags: openvino/cvat_ui:latest
148+
load: true
149+
- name: Instrumentation of the code then rebuilding the CVAT UI
150+
if: github.ref == 'refs/heads/develop'
32151
run: |
33152
npm ci
34153
npm run coverage
35-
docker-compose -f docker-compose.yml -f docker-compose.dev.yml up -d --build
36-
- name: End-to-end testing
154+
docker-compose -f docker-compose.yml -f docker-compose.dev.yml build cvat_ui
155+
- name: Running e2e tests
156+
env:
157+
DJANGO_SU_NAME: 'admin'
158+
DJANGO_SU_EMAIL: '[email protected]'
159+
DJANGO_SU_PASSWORD: '12qwaszx'
160+
API_ABOUT_PAGE: "localhost:8080/api/v1/server/about"
37161
run: |
162+
docker-compose -f docker-compose.yml -f docker-compose.dev.yml up -d
163+
/bin/bash -c 'while [[ "$(curl -s -o /dev/null -w ''%{http_code}'' ${API_ABOUT_PAGE})" != "401" ]]; do sleep 5; done'
164+
docker exec -i cvat /bin/bash -c "echo \"from django.contrib.auth.models import User; User.objects.create_superuser('${DJANGO_SU_NAME}', '${DJANGO_SU_EMAIL}', '${DJANGO_SU_PASSWORD}')\" | python3 ~/manage.py shell" docker exec -i cvat /bin/bash -c "echo \"from django.contrib.auth.models import User; User.objects.create_superuser('${DJANGO_SU_NAME}', '${DJANGO_SU_EMAIL}', '${DJANGO_SU_PASSWORD}')\" | python3 ~/manage.py shell"
38165
cd ./tests
39166
npm ci
40-
npx cypress run --headless --browser chrome
167+
if [[ ${{ github.ref }} == 'refs/heads/develop' ]]; then
168+
npx cypress run --headless --browser chrome --spec 'cypress/integration/${{ matrix.specs }}/**/*.js'
169+
mv ./.nyc_output/out.json ./.nyc_output/out_${{ matrix.specs }}.json
170+
else
171+
npx cypress run --headless --browser chrome --env coverage=false --spec 'cypress/integration/${{ matrix.specs }}/**/*.js'
172+
fi
173+
- name: Creating a log file from "cvat" container logs
174+
if: failure()
175+
run: |
176+
docker logs cvat > ${{ github.workspace }}/tests/cvat_${{ matrix.specs }}.log
41177
- name: Uploading cypress screenshots as an artifact
42178
if: failure()
43179
uses: actions/upload-artifact@v2
44180
with:
45-
name: cypress_screenshots
181+
name: cypress_screenshots_${{ matrix.specs }}
46182
path: ${{ github.workspace }}/tests/cypress/screenshots
47-
- name: Collect coverage data
183+
- name: Uploading "cvat" container logs as an artifact
184+
if: failure()
185+
uses: actions/upload-artifact@v2
186+
with:
187+
name: cvat_container_logs
188+
path: ${{ github.workspace }}/tests/cvat_${{ matrix.specs }}.log
189+
- name: Uploading code coverage results as an artifact
190+
if: github.ref == 'refs/heads/develop'
191+
uses: actions/upload-artifact@v2
192+
with:
193+
name: coverage_results
194+
path: ${{ github.workspace }}/tests/.nyc_output
195+
196+
Coveralls:
197+
if: github.ref == 'refs/heads/develop'
198+
runs-on: ubuntu-latest
199+
needs: [Unit_testing, E2E_testing]
200+
steps:
201+
- uses: actions/checkout@v2
202+
- name: Geting SHA from the default branch
203+
id: get-sha
204+
run: |
205+
URL_get_default_branch="https://api.github.com/repos/${{ github.repository }}"
206+
DEFAULT_BRANCH=$(curl -s -X GET -G ${URL_get_default_branch} | jq -r '.default_branch')
207+
URL_get_sha_default_branch="https://api.github.com/repos/${{ github.repository }}/git/ref/heads/${DEFAULT_BRANCH}"
208+
SHA=$(curl -s -X GET -G ${URL_get_sha_default_branch} | jq .object.sha | tr -d '"')
209+
echo ::set-output name=sha::${SHA}
210+
- name: Getting CVAT server cache from the default branch
211+
uses: actions/cache@v2
212+
with:
213+
path: /tmp/cvat_cache_server
214+
key: ${{ runner.os }}-build-server-${{ steps.get-sha.outputs.sha }}
215+
- name: Set up Docker Buildx
216+
uses: docker/[email protected]
217+
- name: Building CVAT server image
218+
uses: docker/build-push-action@v2
219+
with:
220+
context: .
221+
file: ./Dockerfile
222+
cache-from: type=local,src=/tmp/cvat_cache_server
223+
tags: openvino/cvat_server:latest
224+
load: true
225+
- name: Downloading coverage results
226+
uses: actions/download-artifact@v2
227+
with:
228+
name: coverage_results
229+
- name: Combining coverage results
230+
run: |
231+
mkdir -p ./nyc_output_tmp
232+
mv ./out_*.json ./nyc_output_tmp
233+
mkdir -p ./.nyc_output
234+
npm ci
235+
npx nyc merge ./nyc_output_tmp ./.nyc_output/out.json
236+
- name: Sending results to Coveralls
48237
env:
49238
HOST_COVERAGE_DATA_DIR: ${{ github.workspace }}
50239
CONTAINER_COVERAGE_DATA_DIR: "/coverage_data"
51240
COVERALLS_SERVICE_NAME: github
52241
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
53242
run: |
54-
mv ./tests/.nyc_output ./
55-
npx nyc report --reporter=text-lcov >> ${HOST_COVERAGE_DATA_DIR}/lcov.info
56-
docker-compose -f docker-compose.yml -f docker-compose.ci.yml run cvat_ci /bin/bash -c 'cd ${CONTAINER_COVERAGE_DATA_DIR} && coveralls-lcov -v -n lcov.info > ${CONTAINER_COVERAGE_DATA_DIR}/coverage.json'
57-
docker-compose -f docker-compose.yml -f docker-compose.ci.yml run cvat_ci /bin/bash -c 'ln -s ${CONTAINER_COVERAGE_DATA_DIR}/.git . && ln -s ${CONTAINER_COVERAGE_DATA_DIR}/.coverage . && ln -s ${CONTAINER_COVERAGE_DATA_DIR}/coverage.json . && coveralls --merge=coverage.json'
243+
npx nyc report --reporter=text-lcov >> ${HOST_COVERAGE_DATA_DIR}/lcov.info
244+
docker-compose -f docker-compose.yml -f docker-compose.dev.yml -f docker-compose.ci.yml run cvat_ci /bin/bash -c 'cd ${CONTAINER_COVERAGE_DATA_DIR} && coveralls-lcov -v -n lcov.info > ${CONTAINER_COVERAGE_DATA_DIR}/coverage.json'
245+
docker-compose -f docker-compose.yml -f docker-compose.dev.yml -f docker-compose.ci.yml run cvat_ci /bin/bash -c 'ln -s ${CONTAINER_COVERAGE_DATA_DIR}/.git . && ln -s ${CONTAINER_COVERAGE_DATA_DIR}/.coverage . && ln -s ${CONTAINER_COVERAGE_DATA_DIR}/coverage.json . && coveralls --merge=coverage.json'

0 commit comments

Comments
 (0)