diff --git a/Dockerfile b/Dockerfile index f96bedd4888b..a936ee5c9edd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -177,7 +177,7 @@ RUN if [ "${CVAT_DEBUG_ENABLED}" = 'yes' ]; then \ COPY cvat/nginx.conf /etc/nginx/nginx.conf COPY --chown=${USER} components /tmp/components COPY --chown=${USER} supervisord/ ${HOME}/supervisord -COPY --chown=${USER} wait-for-it.sh manage.py backend_entrypoint.sh ${HOME}/ +COPY --chown=${USER} wait-for-it.sh manage.py backend_entrypoint.sh wait_for_deps.sh ${HOME}/ COPY --chown=${USER} utils/ ${HOME}/utils COPY --chown=${USER} cvat/ ${HOME}/cvat COPY --chown=${USER} rqscheduler.py ${HOME} diff --git a/changelog.d/20231215_165536_roman_split_redis.md b/changelog.d/20231215_165536_roman_split_redis.md new file mode 100644 index 000000000000..e052e594d8e4 --- /dev/null +++ b/changelog.d/20231215_165536_roman_split_redis.md @@ -0,0 +1,4 @@ +### Changed + +- Job queues are now stored in a dedicated Redis instance + () diff --git a/cvat/settings/base.py b/cvat/settings/base.py index 3f454cb45413..a5e514260b4a 100644 --- a/cvat/settings/base.py +++ b/cvat/settings/base.py @@ -41,10 +41,6 @@ ALLOWED_HOSTS = os.environ.get('ALLOWED_HOSTS', 'localhost,127.0.0.1').split(',') INTERNAL_IPS = ['127.0.0.1'] -redis_host = os.getenv('CVAT_REDIS_HOST', 'localhost') -redis_port = os.getenv('CVAT_REDIS_PORT', 6379) -redis_password = os.getenv('CVAT_REDIS_PASSWORD', '') - def generate_secret_key(): """ Creates secret_key.py in such a way that multiple processes calling @@ -289,62 +285,49 @@ class CVAT_QUEUES(Enum): ANALYTICS_REPORTS = 'analytics_reports' CLEANING = 'cleaning' +redis_inmem_host = os.getenv('CVAT_REDIS_INMEM_HOST', 'localhost') +redis_inmem_port = os.getenv('CVAT_REDIS_INMEM_PORT', 6379) +redis_inmem_password = os.getenv('CVAT_REDIS_INMEM_PASSWORD', '') + +shared_queue_settings = { + 'HOST': redis_inmem_host, + 'PORT': redis_inmem_port, + 'DB': 0, + 'PASSWORD': urllib.parse.quote(redis_inmem_password), +} + RQ_QUEUES = { CVAT_QUEUES.IMPORT_DATA.value: { - 'HOST': redis_host, - 'PORT': redis_port, - 'DB': 0, + **shared_queue_settings, 'DEFAULT_TIMEOUT': '4h', - 'PASSWORD': urllib.parse.quote(redis_password), }, CVAT_QUEUES.EXPORT_DATA.value: { - 'HOST': redis_host, - 'PORT': redis_port, - 'DB': 0, + **shared_queue_settings, 'DEFAULT_TIMEOUT': '4h', - 'PASSWORD': urllib.parse.quote(redis_password), }, CVAT_QUEUES.AUTO_ANNOTATION.value: { - 'HOST': redis_host, - 'PORT': redis_port, - 'DB': 0, + **shared_queue_settings, 'DEFAULT_TIMEOUT': '24h', - 'PASSWORD': urllib.parse.quote(redis_password), }, CVAT_QUEUES.WEBHOOKS.value: { - 'HOST': redis_host, - 'PORT': redis_port, - 'DB': 0, + **shared_queue_settings, 'DEFAULT_TIMEOUT': '1h', - 'PASSWORD': urllib.parse.quote(redis_password), }, CVAT_QUEUES.NOTIFICATIONS.value: { - 'HOST': redis_host, - 'PORT': redis_port, - 'DB': 0, + **shared_queue_settings, 'DEFAULT_TIMEOUT': '1h', - 'PASSWORD': urllib.parse.quote(redis_password), }, CVAT_QUEUES.QUALITY_REPORTS.value: { - 'HOST': redis_host, - 'PORT': redis_port, - 'DB': 0, + **shared_queue_settings, 'DEFAULT_TIMEOUT': '1h', - 'PASSWORD': urllib.parse.quote(redis_password), }, CVAT_QUEUES.ANALYTICS_REPORTS.value: { - 'HOST': redis_host, - 'PORT': redis_port, - 'DB': 0, + **shared_queue_settings, 'DEFAULT_TIMEOUT': '1h', - 'PASSWORD': urllib.parse.quote(redis_password), }, CVAT_QUEUES.CLEANING.value: { - 'HOST': redis_host, - 'PORT': redis_port, - 'DB': 0, + **shared_queue_settings, 'DEFAULT_TIMEOUT': '1h', - 'PASSWORD': urllib.parse.quote(redis_password), }, } @@ -543,15 +526,22 @@ class CVAT_QUEUES(Enum): 'analytics_visibility': True, } +redis_ondisk_host = os.getenv('CVAT_REDIS_ONDISK_HOST', 'localhost') +# The default port is not Redis's default port (6379). +# This is so that a developer can run both in-mem and on-disk Redis on their machine +# without running into a port conflict. +redis_ondisk_port = os.getenv('CVAT_REDIS_ONDISK_PORT', 6479) +redis_ondisk_password = os.getenv('CVAT_REDIS_ONDISK_PASSWORD', '') + CACHES = { 'default': { 'BACKEND': 'django.core.cache.backends.locmem.LocMemCache', }, - 'media' : { + 'media': { 'BACKEND' : 'django.core.cache.backends.redis.RedisCache', - "LOCATION": f"redis://:{urllib.parse.quote(redis_password)}@{redis_host}:{redis_port}", + "LOCATION": f"redis://:{urllib.parse.quote(redis_ondisk_password)}@{redis_ondisk_host}:{redis_ondisk_port}", 'TIMEOUT' : 3600 * 24, # 1 day - } + } } USE_CACHE = True diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index ea53a91d7492..5e0daacb7938 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -105,6 +105,10 @@ services: ports: - '8181:8181' - cvat_redis: + cvat_redis_inmem: ports: - '6379:6379' + + cvat_redis_ondisk: + ports: + - '6479:6379' diff --git a/docker-compose.yml b/docker-compose.yml index 0bce183d4e48..f20e6db7ec89 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,12 +5,23 @@ x-backend-env: &backend-env CLICKHOUSE_HOST: clickhouse CVAT_POSTGRES_HOST: cvat_db - CVAT_REDIS_HOST: cvat_redis + CVAT_REDIS_INMEM_HOST: cvat_redis_inmem + CVAT_REDIS_INMEM_PORT: 6379 + CVAT_REDIS_ONDISK_HOST: cvat_redis_ondisk + CVAT_REDIS_ONDISK_PORT: 6379 DJANGO_LOG_SERVER_HOST: vector DJANGO_LOG_SERVER_PORT: 80 no_proxy: clickhouse,grafana,vector,nuclio,opa,${no_proxy:-} SMOKESCREEN_OPTS: ${SMOKESCREEN_OPTS:-} +x-backend-deps: &backend-deps + cvat_redis_inmem: + condition: service_started + cvat_redis_ondisk: + condition: service_started + cvat_db: + condition: service_started + services: cvat_db: container_name: cvat_db @@ -25,8 +36,22 @@ services: networks: - cvat - cvat_redis: - container_name: cvat_redis + cvat_redis_inmem: + container_name: cvat_redis_inmem + image: redis:7.2.3-alpine + restart: always + command: [ + "redis-server", + "--save", "60", "100", + "--appendonly", "yes", + ] + volumes: + - cvat_inmem_db:/data + networks: + - cvat + + cvat_redis_ondisk: + container_name: cvat_redis_ondisk image: eqalpha/keydb:x86_64_v6.3.2 restart: always command: [ @@ -46,9 +71,10 @@ services: image: cvat/server:${CVAT_VERSION:-dev} restart: always depends_on: - - cvat_redis - - cvat_db - - cvat_opa + <<: *backend-deps + cvat_opa: + condition: + service_started environment: <<: *backend-env DJANGO_MODWSGI_EXTRA_ARGS: '' @@ -79,13 +105,10 @@ services: container_name: cvat_utils image: cvat/server:${CVAT_VERSION:-dev} restart: always - depends_on: - - cvat_redis - - cvat_db - - cvat_opa + depends_on: *backend-deps environment: <<: *backend-env - CVAT_REDIS_PASSWORD: '' + CVAT_REDIS_INMEM_PASSWORD: '' NUMPROCS: 1 command: run utils volumes: @@ -99,9 +122,7 @@ services: container_name: cvat_worker_import image: cvat/server:${CVAT_VERSION:-dev} restart: always - depends_on: - - cvat_redis - - cvat_db + depends_on: *backend-deps environment: <<: *backend-env NUMPROCS: 2 @@ -117,9 +138,7 @@ services: container_name: cvat_worker_export image: cvat/server:${CVAT_VERSION:-dev} restart: always - depends_on: - - cvat_redis - - cvat_db + depends_on: *backend-deps environment: <<: *backend-env NUMPROCS: 2 @@ -135,10 +154,7 @@ services: container_name: cvat_worker_annotation image: cvat/server:${CVAT_VERSION:-dev} restart: always - depends_on: - - cvat_redis - - cvat_db - - cvat_opa + depends_on: *backend-deps environment: <<: *backend-env NUMPROCS: 1 @@ -154,10 +170,7 @@ services: container_name: cvat_worker_webhooks image: cvat/server:${CVAT_VERSION:-dev} restart: always - depends_on: - - cvat_redis - - cvat_db - - cvat_opa + depends_on: *backend-deps environment: <<: *backend-env NUMPROCS: 1 @@ -173,9 +186,7 @@ services: container_name: cvat_worker_quality_reports image: cvat/server:${CVAT_VERSION:-dev} restart: always - depends_on: - - cvat_redis - - cvat_db + depends_on: *backend-deps environment: <<: *backend-env NUMPROCS: 1 @@ -191,9 +202,7 @@ services: container_name: cvat_worker_analytics_reports image: cvat/server:${CVAT_VERSION:-dev} restart: always - depends_on: - - cvat_redis - - cvat_db + depends_on: *backend-deps environment: <<: *backend-env NUMPROCS: 2 @@ -371,6 +380,7 @@ volumes: cvat_data: cvat_keys: cvat_logs: + cvat_inmem_db: cvat_events_db: cvat_cache_db: diff --git a/helm-chart/Chart.yaml b/helm-chart/Chart.yaml index 6437fe70219b..4512b4cde983 100644 --- a/helm-chart/Chart.yaml +++ b/helm-chart/Chart.yaml @@ -56,6 +56,11 @@ dependencies: repository: https://helm.traefik.io/traefik condition: traefik.enabled + - name: redis + version: "18.5.*" + repository: https://charts.bitnami.com/bitnami + condition: redis.enabled + - name: keydb version: 0.48.0 repository: https://enapter.github.io/charts/ diff --git a/helm-chart/templates/_helpers.tpl b/helm-chart/templates/_helpers.tpl index 37dd9ecef1cc..ddfe1ea4b9b6 100644 --- a/helm-chart/templates/_helpers.tpl +++ b/helm-chart/templates/_helpers.tpl @@ -62,18 +62,36 @@ Create the name of the service account to use {{- end }} {{- define "cvat.sharedBackendEnv" }} +{{- if .Values.redis.enabled }} +- name: CVAT_REDIS_INMEM_HOST + value: "{{ .Release.Name }}-redis-master" +{{- else }} +- name: CVAT_REDIS_INMEM_HOST + value: "{{ .Values.redis.external.host }}" +{{- end }} +- name: CVAT_REDIS_INMEM_PORT + value: "6379" +- name: CVAT_REDIS_INMEM_PASSWORD + valueFrom: + secretKeyRef: + name: "{{ tpl (.Values.redis.secret.name) . }}" + key: password + {{- if .Values.keydb.enabled }} -- name: CVAT_REDIS_HOST +- name: CVAT_REDIS_ONDISK_HOST value: "{{ .Release.Name }}-keydb" {{- else }} -- name: CVAT_REDIS_HOST +- name: CVAT_REDIS_ONDISK_HOST value: "{{ .Values.keydb.external.host }}" {{- end }} -- name: CVAT_REDIS_PASSWORD +- name: CVAT_REDIS_ONDISK_PORT + value: "6379" +- name: CVAT_REDIS_ONDISK_PASSWORD valueFrom: secretKeyRef: name: "{{ tpl (.Values.keydb.secret.name) . }}" key: password + {{- if .Values.postgresql.enabled }} - name: CVAT_POSTGRES_HOST value: "{{ .Release.Name }}-postgresql" diff --git a/helm-chart/templates/cvat-redis-secret.yml b/helm-chart/templates/cvat-redis-secret.yml new file mode 100644 index 000000000000..6e1b5cff8700 --- /dev/null +++ b/helm-chart/templates/cvat-redis-secret.yml @@ -0,0 +1,12 @@ +{{- if .Values.redis.secret.create }} +apiVersion: v1 +kind: Secret +metadata: + name: "{{ tpl (.Values.redis.secret.name) . }}" + namespace: {{ .Release.Namespace }} + labels: + {{- include "cvat.labels" . | nindent 4 }} +type: generic +stringData: + password: {{ .Values.redis.secret.password | toString | toYaml }} +{{- end }} diff --git a/helm-chart/values.yaml b/helm-chart/values.yaml index eeaa2d56fe79..47b89098c6b9 100644 --- a/helm-chart/values.yaml +++ b/helm-chart/values.yaml @@ -243,6 +243,20 @@ postgresql: postgres_password: cvat_postgresql_postgres replication_password: cvat_postgresql_replica +# https://artifacthub.io/packages/helm/bitnami/redis +redis: + enabled: true + external: + host: 127.0.0.1 + architecture: standalone + auth: + existingSecret: "cvat-redis-secret" + existingSecretPasswordKey: password + secret: + create: true + name: cvat-redis-secret + password: cvat_redis + # TODO: persistence options # https://artifacthub.io/packages/helm/enapter/keydb keydb: diff --git a/site/content/en/docs/contributing/development-environment.md b/site/content/en/docs/contributing/development-environment.md index 198214d1ebd1..42bbcda4a41d 100644 --- a/site/content/en/docs/contributing/development-environment.md +++ b/site/content/en/docs/contributing/development-environment.md @@ -158,7 +158,7 @@ description: 'Installing a development environment for different operating syste ```bash docker compose -f docker-compose.yml -f docker-compose.dev.yml up -d --build \ - cvat_opa cvat_db cvat_redis cvat_server + cvat_opa cvat_db cvat_redis_inmem cvat_redis_ondisk cvat_server ``` Note: this runs an extra copy of the CVAT server in order to supply rules to OPA. diff --git a/supervisord/utils.conf b/supervisord/utils.conf index 3c0509205960..583a5a2ac27b 100644 --- a/supervisord/utils.conf +++ b/supervisord/utils.conf @@ -18,24 +18,25 @@ pidfile=/tmp/supervisord/supervisord.pid ; pidfile location childlogdir=%(ENV_HOME)s/logs/ ; where child log files will live [program:rqscheduler] -command=%(ENV_HOME)s/wait-for-it.sh %(ENV_CVAT_REDIS_HOST)s:6379 -t 0 -- bash -c \ - "python3 ~/rqscheduler.py --host %(ENV_CVAT_REDIS_HOST)s --password '%(ENV_CVAT_REDIS_PASSWORD)s' -i 30 --path '%(ENV_HOME)s'" +command=%(ENV_HOME)s/wait_for_deps.sh + python3 ~/rqscheduler.py + --host "%(ENV_CVAT_REDIS_INMEM_HOST)s" --port "%(ENV_CVAT_REDIS_INMEM_PORT)s" + --password "%(ENV_CVAT_REDIS_INMEM_PASSWORD)s" + -i 30 --path %(ENV_HOME)s environment=VECTOR_EVENT_HANDLER="SynchronousLogstashHandler" numprocs=1 [program:rqworker-notifications] -command=%(ENV_HOME)s/wait-for-it.sh %(ENV_CVAT_REDIS_HOST)s:6379 -t 0 -- bash -c " \ - exec python3 %(ENV_HOME)s/manage.py rqworker -v 3 notifications \ - --worker-class cvat.rqworker.DefaultWorker \ - " +command=%(ENV_HOME)s/wait_for_deps.sh + python3 %(ENV_HOME)s/manage.py rqworker -v 3 notifications + --worker-class cvat.rqworker.DefaultWorker environment=VECTOR_EVENT_HANDLER="SynchronousLogstashHandler" numprocs=1 [program:rqworker-cleaning] -command=%(ENV_HOME)s/wait-for-it.sh %(ENV_CVAT_REDIS_HOST)s:6379 -t 0 -- bash -c " \ - exec python3 %(ENV_HOME)s/manage.py rqworker -v 3 cleaning \ - --worker-class cvat.rqworker.DefaultWorker \ - " +command=%(ENV_HOME)s/wait_for_deps.sh + python3 %(ENV_HOME)s/manage.py rqworker -v 3 cleaning + --worker-class cvat.rqworker.DefaultWorker environment=VECTOR_EVENT_HANDLER="SynchronousLogstashHandler" numprocs=%(ENV_NUMPROCS)s process_name=%(program_name)s-%(process_num)d diff --git a/supervisord/worker.analytics_reports.conf b/supervisord/worker.analytics_reports.conf index 3d29ee46ec6f..613876310cf7 100644 --- a/supervisord/worker.analytics_reports.conf +++ b/supervisord/worker.analytics_reports.conf @@ -18,10 +18,9 @@ pidfile=/tmp/supervisord/supervisord.pid ; pidfile location childlogdir=%(ENV_HOME)s/logs/ ; where child log files will live [program:rqworker-analytics-reports] -command=%(ENV_HOME)s/wait-for-it.sh %(ENV_CVAT_REDIS_HOST)s:6379 -t 0 -- bash -c " \ - exec python3 %(ENV_HOME)s/manage.py rqworker -v 3 analytics_reports \ - --worker-class cvat.rqworker.DefaultWorker \ - " +command=%(ENV_HOME)s/wait_for_deps.sh + python3 %(ENV_HOME)s/manage.py rqworker -v 3 analytics_reports + --worker-class cvat.rqworker.DefaultWorker environment=VECTOR_EVENT_HANDLER="SynchronousLogstashHandler" numprocs=%(ENV_NUMPROCS)s process_name=%(program_name)s-%(process_num)d diff --git a/supervisord/worker.annotation.conf b/supervisord/worker.annotation.conf index 32adcbb757b6..9343d613d6e0 100644 --- a/supervisord/worker.annotation.conf +++ b/supervisord/worker.annotation.conf @@ -18,10 +18,9 @@ pidfile=/tmp/supervisord/supervisord.pid ; pidfile location childlogdir=%(ENV_HOME)s/logs/ ; where child log files will live [program:rqworker-annotation] -command=%(ENV_HOME)s/wait-for-it.sh %(ENV_CVAT_REDIS_HOST)s:6379 -t 0 -- bash -c " \ - exec python3 %(ENV_HOME)s/manage.py rqworker -v 3 annotation \ - --worker-class cvat.rqworker.DefaultWorker \ - " +command=%(ENV_HOME)s/wait_for_deps.sh + python3 %(ENV_HOME)s/manage.py rqworker -v 3 annotation + --worker-class cvat.rqworker.DefaultWorker environment=VECTOR_EVENT_HANDLER="SynchronousLogstashHandler" numprocs=%(ENV_NUMPROCS)s process_name=%(program_name)s-%(process_num)d diff --git a/supervisord/worker.export.conf b/supervisord/worker.export.conf index 1e8dd3e8b14d..816c9182bc42 100644 --- a/supervisord/worker.export.conf +++ b/supervisord/worker.export.conf @@ -18,10 +18,9 @@ pidfile=/tmp/supervisord/supervisord.pid ; pidfile location childlogdir=%(ENV_HOME)s/logs/ ; where child log files will live [program:rqworker-export] -command=%(ENV_HOME)s/wait-for-it.sh %(ENV_CVAT_REDIS_HOST)s:6379 -t 0 -- bash -c " \ - exec python3 %(ENV_HOME)s/manage.py rqworker -v 3 export \ - --worker-class cvat.rqworker.DefaultWorker \ - " +command=%(ENV_HOME)s/wait_for_deps.sh + python3 %(ENV_HOME)s/manage.py rqworker -v 3 export + --worker-class cvat.rqworker.DefaultWorker environment=VECTOR_EVENT_HANDLER="SynchronousLogstashHandler" numprocs=%(ENV_NUMPROCS)s process_name=%(program_name)s-%(process_num)d diff --git a/supervisord/worker.import.conf b/supervisord/worker.import.conf index 22dca6772c77..4af0cdc9764d 100644 --- a/supervisord/worker.import.conf +++ b/supervisord/worker.import.conf @@ -18,10 +18,9 @@ pidfile=/tmp/supervisord/supervisord.pid ; pidfile location childlogdir=%(ENV_HOME)s/logs/ ; where child log files will live [program:rqworker-import] -command=%(ENV_HOME)s/wait-for-it.sh %(ENV_CVAT_REDIS_HOST)s:6379 -t 0 -- bash -c " \ - exec python3 %(ENV_HOME)s/manage.py rqworker -v 3 import \ - --worker-class cvat.rqworker.DefaultWorker \ - " +command=%(ENV_HOME)s/wait_for_deps.sh + python3 %(ENV_HOME)s/manage.py rqworker -v 3 import + --worker-class cvat.rqworker.DefaultWorker environment=VECTOR_EVENT_HANDLER="SynchronousLogstashHandler" numprocs=%(ENV_NUMPROCS)s process_name=%(program_name)s-%(process_num)d diff --git a/supervisord/worker.quality_reports.conf b/supervisord/worker.quality_reports.conf index 155afd4461c5..b537940d2bb4 100644 --- a/supervisord/worker.quality_reports.conf +++ b/supervisord/worker.quality_reports.conf @@ -18,10 +18,9 @@ pidfile=/tmp/supervisord/supervisord.pid ; pidfile location childlogdir=%(ENV_HOME)s/logs/ ; where child log files will live [program:rqworker-quality-reports] -command=%(ENV_HOME)s/wait-for-it.sh %(ENV_CVAT_REDIS_HOST)s:6379 -t 0 -- bash -c " \ - exec python3 %(ENV_HOME)s/manage.py rqworker -v 3 quality_reports \ - --worker-class cvat.rqworker.DefaultWorker \ - " +command=%(ENV_HOME)s/wait_for_deps.sh + python3 %(ENV_HOME)s/manage.py rqworker -v 3 quality_reports + --worker-class cvat.rqworker.DefaultWorker environment=VECTOR_EVENT_HANDLER="SynchronousLogstashHandler" numprocs=%(ENV_NUMPROCS)s process_name=%(program_name)s-%(process_num)d diff --git a/supervisord/worker.webhooks.conf b/supervisord/worker.webhooks.conf index b818f05082a0..632c9d6832a7 100644 --- a/supervisord/worker.webhooks.conf +++ b/supervisord/worker.webhooks.conf @@ -18,10 +18,9 @@ pidfile=/tmp/supervisord/supervisord.pid ; pidfile location childlogdir=%(ENV_HOME)s/logs/ ; where child log files will live [program:rqworker-webhooks] -command=%(ENV_HOME)s/wait-for-it.sh %(ENV_CVAT_REDIS_HOST)s:6379 -t 0 -- bash -c \ - "exec python3 %(ENV_HOME)s/manage.py rqworker -v 3 webhooks \ - --worker-class cvat.rqworker.DefaultWorker \ - " +command=%(ENV_HOME)s/wait_for_deps.sh + python3 %(ENV_HOME)s/manage.py rqworker -v 3 webhooks + --worker-class cvat.rqworker.DefaultWorker environment=VECTOR_EVENT_HANDLER="SynchronousLogstashHandler" numprocs=%(ENV_NUMPROCS)s process_name=%(program_name)s-%(process_num)d diff --git a/tests/python/rest_api/test_queues.py b/tests/python/rest_api/test_queues.py index 80c15e3bc0cf..a1729cf6f253 100644 --- a/tests/python/rest_api/test_queues.py +++ b/tests/python/rest_api/test_queues.py @@ -19,7 +19,7 @@ @pytest.mark.usefixtures("restore_db_per_function") @pytest.mark.usefixtures("restore_cvat_data") -@pytest.mark.usefixtures("restore_redis_db_per_function") +@pytest.mark.usefixtures("restore_redis_inmem_per_function") class TestRQQueueWorking: _USER_1 = "admin1" _USER_2 = "admin2" diff --git a/tests/python/rest_api/test_tasks.py b/tests/python/rest_api/test_tasks.py index 0cdb94e2c02e..241852f2d7d4 100644 --- a/tests/python/rest_api/test_tasks.py +++ b/tests/python/rest_api/test_tasks.py @@ -748,7 +748,7 @@ def test_export_dataset_after_deleting_related_cloud_storage(self, admin_user, t @pytest.mark.usefixtures("restore_db_per_function") @pytest.mark.usefixtures("restore_cvat_data") -@pytest.mark.usefixtures("restore_redis_db_per_function") +@pytest.mark.usefixtures("restore_redis_ondisk_per_function") class TestPostTaskData: _USERNAME = "admin1" @@ -1889,7 +1889,7 @@ def test_admin_can_add_skeleton(self, tasks, admin_user): @pytest.mark.usefixtures("restore_db_per_function") @pytest.mark.usefixtures("restore_cvat_data") -@pytest.mark.usefixtures("restore_redis_db_per_function") +@pytest.mark.usefixtures("restore_redis_ondisk_per_function") class TestWorkWithTask: _USERNAME = "admin1" diff --git a/tests/python/shared/fixtures/init.py b/tests/python/shared/fixtures/init.py index 2fa0376081ad..ab828d601cbb 100644 --- a/tests/python/shared/fixtures/init.py +++ b/tests/python/shared/fixtures/init.py @@ -130,7 +130,11 @@ def _kube_get_clichouse_pod_name(): return _kube_get_pod_name("app.kubernetes.io/name=clickhouse") -def _kube_get_redis_pod_name(): +def _kube_get_redis_inmem_pod_name(): + return _kube_get_pod_name("app.kubernetes.io/name=redis") + + +def _kube_get_redis_ondisk_pod_name(): return _kube_get_pod_name("app.kubernetes.io/name=keydb") @@ -173,12 +177,21 @@ def kube_exec_clickhouse_db(command): _run(["kubectl", "exec", pod_name, "--"] + command) -def docker_exec_redis_db(command): - _run(["docker", "exec", f"{PREFIX}_cvat_redis_1"] + command) +def docker_exec_redis_inmem(command): + _run(["docker", "exec", f"{PREFIX}_cvat_redis_inmem_1"] + command) + + +def kube_exec_redis_inmem(command): + pod_name = _kube_get_redis_inmem_pod_name() + _run(["kubectl", "exec", pod_name, "--"] + command) + + +def docker_exec_redis_ondisk(command): + _run(["docker", "exec", f"{PREFIX}_cvat_redis_ondisk_1"] + command) -def kube_exec_redis_db(command): - pod_name = _kube_get_redis_pod_name() +def kube_exec_redis_ondisk(command): + pod_name = _kube_get_redis_ondisk_pod_name() _run(["kubectl", "exec", pod_name, "--"] + command) @@ -218,24 +231,20 @@ def kube_restore_clickhouse_db(): ) -def docker_restore_redis_db(): - docker_exec_redis_db( - [ - "/bin/sh", - "-c", - "keydb-cli flushall", - ] - ) +def docker_restore_redis_inmem(): + docker_exec_redis_inmem(["redis-cli", "flushall"]) -def kube_restore_redis_db(): - kube_exec_redis_db( - [ - "/bin/sh", - "-c", - "keydb-cli flushall", - ] - ) +def kube_restore_redis_inmem(): + kube_exec_redis_inmem(["redis-cli", "flushall"]) + + +def docker_restore_redis_ondisk(): + docker_exec_redis_ondisk(["keydb-cli", "flushall"]) + + +def kube_restore_redis_ondisk(): + kube_exec_redis_ondisk(["keydb-cli", "flushall"]) def running_containers(): @@ -570,12 +579,21 @@ def restore_clickhouse_db_per_class(request): @pytest.fixture(scope="function") -def restore_redis_db_per_function(request): +def restore_redis_inmem_per_function(request): # Note that autouse fixtures are executed first within their scope, so be aware of the order # Pre-test DB setups (eg. with class-declared autouse setup() method) may be cleaned. # https://docs.pytest.org/en/stable/reference/fixtures.html#autouse-fixtures-are-executed-first-within-their-scope platform = request.config.getoption("--platform") if platform == "local": - docker_restore_redis_db() + docker_restore_redis_inmem() + else: + kube_restore_redis_inmem() + + +@pytest.fixture(scope="function") +def restore_redis_ondisk_per_function(request): + platform = request.config.getoption("--platform") + if platform == "local": + docker_restore_redis_ondisk() else: - kube_restore_redis_db() + kube_restore_redis_ondisk() diff --git a/wait_for_deps.sh b/wait_for_deps.sh new file mode 100755 index 000000000000..c78950cf96c4 --- /dev/null +++ b/wait_for_deps.sh @@ -0,0 +1,18 @@ +#!/bin/sh + +# Copyright (C) 2023 CVAT.ai Corporation +# +# SPDX-License-Identifier: MIT + +# This is a wrapper script for running backend services. It waits for services +# the backend depends on to start before executing the backend itself. + +# Ideally, the check that all DB migrations have completed should also be here, +# but it's too resource-intensive to execute for every worker we might be running +# in a container. Instead, it's in backend_entrypoint.sh. + +~/wait-for-it.sh "${CVAT_POSTGRES_HOST}:${CVAT_POSTGRES_PORT:-5432}" -t 0 +~/wait-for-it.sh "${CVAT_REDIS_INMEM_HOST}:${CVAT_REDIS_INMEM_PORT}" -t 0 +~/wait-for-it.sh "${CVAT_REDIS_ONDISK_HOST}:${CVAT_REDIS_ONDISK_PORT}" -t 0 + +exec "$@"