Replace keydb with kvrocks (cvat-ai#7339)

Resolved cvat-ai#7345
winpkay · Jan 18, 2024 · 7a1a4b1 · 7a1a4b1
1 parent d25b863
commit 7a1a4b1
Show file tree

Hide file tree

Showing 12 changed files with 211 additions and 59 deletions.
diff --git a/changelog.d/20240115_172625_andrey_switch_to_kvrocks.md b/changelog.d/20240115_172625_andrey_switch_to_kvrocks.md
@@ -0,0 +1,4 @@
+### Changed
+
+- KeyDB used as data cache replaced by Kvrocks
+  (<https://github.com/opencv/cvat/pull/7339>)
diff --git a/cvat/settings/base.py b/cvat/settings/base.py
@@ -521,9 +521,9 @@ class CVAT_QUEUES(Enum):
 
 redis_ondisk_host = os.getenv('CVAT_REDIS_ONDISK_HOST', 'localhost')
 # The default port is not Redis's default port (6379).
-# This is so that a developer can run both in-mem and on-disk Redis on their machine
+# This is so that a developer can run both in-mem Redis and on-disk Kvrocks on their machine
 # without running into a port conflict.
-redis_ondisk_port = os.getenv('CVAT_REDIS_ONDISK_PORT', 6479)
+redis_ondisk_port = os.getenv('CVAT_REDIS_ONDISK_PORT', 6666)
 redis_ondisk_password = os.getenv('CVAT_REDIS_ONDISK_PASSWORD', '')
 
 CACHES = {

diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
@@ -111,4 +111,4 @@ services:
 
   cvat_redis_ondisk:
     ports:
-      - '6479:6379'
+      - '6666:6666'
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -8,7 +8,7 @@ x-backend-env: &backend-env
   CVAT_REDIS_INMEM_HOST: cvat_redis_inmem
   CVAT_REDIS_INMEM_PORT: 6379
   CVAT_REDIS_ONDISK_HOST: cvat_redis_ondisk
-  CVAT_REDIS_ONDISK_PORT: 6379
+  CVAT_REDIS_ONDISK_PORT: 6666
   DJANGO_LOG_SERVER_HOST: vector
   DJANGO_LOG_SERVER_PORT: 80
   no_proxy: clickhouse,grafana,vector,nuclio,opa,${no_proxy:-}
@@ -52,17 +52,13 @@ services:
 
   cvat_redis_ondisk:
     container_name: cvat_redis_ondisk
-    image: eqalpha/keydb:x86_64_v6.3.2
+    image: apache/kvrocks:2.7.0
     restart: always
     command: [
-      "keydb-server",
-      "/etc/keydb/keydb.conf",
-      "--storage-provider", "flash", "/data/flash",
-      "--maxmemory", "5G",
-      "--maxmemory-policy", "allkeys-lfu",
+      "--dir", "/var/lib/kvrocks/data"
     ]
     volumes:
-      - cvat_cache_db:/data
+      - cvat_cache_db:/var/lib/kvrocks/data
     networks:
       - cvat
 

diff --git a/helm-chart/Chart.yaml b/helm-chart/Chart.yaml
@@ -16,7 +16,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.11.2
+version: 0.12.0
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to
@@ -60,8 +60,3 @@ dependencies:
     version: "18.5.*"
     repository: https://charts.bitnami.com/bitnami
     condition: redis.enabled
-
-  - name: keydb
-    version: 0.48.0
-    repository: https://enapter.github.io/charts/
-    condition: keydb.enabled
diff --git a/helm-chart/templates/_helpers.tpl b/helm-chart/templates/_helpers.tpl
@@ -77,19 +77,19 @@ Create the name of the service account to use
       name: "{{ tpl (.Values.redis.secret.name) . }}"
       key: password
 
-{{- if .Values.keydb.enabled }}
+{{- if .Values.cvat.kvrocks.enabled }}
 - name: CVAT_REDIS_ONDISK_HOST
-  value: "{{ .Release.Name }}-keydb"
+  value: "{{ .Release.Name }}-kvrocks"
 {{- else }}
 - name: CVAT_REDIS_ONDISK_HOST
-  value: "{{ .Values.keydb.external.host }}"
+  value: "{{ .Values.cvat.kvrocks.external.host }}"
 {{- end }}
 - name: CVAT_REDIS_ONDISK_PORT
-  value: "6379"
+  value: "6666"
 - name: CVAT_REDIS_ONDISK_PASSWORD
   valueFrom:
     secretKeyRef:
-      name: "{{ tpl (.Values.keydb.secret.name) . }}"
+      name: "{{ tpl (.Values.cvat.kvrocks.secret.name) . }}"
       key: password
 
 {{- if .Values.postgresql.enabled }}

diff --git a/helm-chart/templates/cvat-keydb-secret.yml b/helm-chart/templates/cvat-keydb-secret.yml
diff --git a/helm-chart/templates/cvat_kvrocks/secret.yml b/helm-chart/templates/cvat_kvrocks/secret.yml
@@ -0,0 +1,12 @@
+{{- if and .Values.cvat.kvrocks.enabled .Values.cvat.kvrocks.secret.create }}
+apiVersion: v1
+kind: Secret
+metadata:
+  name: "{{ tpl (.Values.cvat.kvrocks.secret.name) . }}"
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "cvat.labels" . | nindent 4 }}
+type: generic
+stringData:
+  password: {{ .Values.cvat.kvrocks.secret.password | toString | quote }}
+{{- end }}
diff --git a/helm-chart/templates/cvat_kvrocks/service.yml b/helm-chart/templates/cvat_kvrocks/service.yml
@@ -0,0 +1,22 @@
+{{- if .Values.cvat.kvrocks.enabled }}
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ .Release.Name }}-kvrocks
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "cvat.labels" . | nindent 4 }}
+    app: cvat-app
+    tier: kvrocks
+spec:
+  selector:
+    app: cvat-app
+    tier: kvrocks
+    {{- include "cvat.labels" . | nindent 4 }}
+  type: ClusterIP
+  ports:
+  - port: 6666
+    targetPort: 6666
+    protocol: TCP
+    name: http
+{{- end }}
diff --git a/helm-chart/templates/cvat_kvrocks/statefulset.yml b/helm-chart/templates/cvat_kvrocks/statefulset.yml
@@ -0,0 +1,105 @@
+{{- if .Values.cvat.kvrocks.enabled }}
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: {{ .Release.Name }}-kvrocks
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "cvat.labels" . | nindent 4 }}
+    app: cvat-app
+    tier: kvrocks
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      {{- include "cvat.labels" . | nindent 6 }}
+      app: cvat-app
+      tier: kvrocks
+  template:
+    metadata:
+      labels:
+        {{- include "cvat.labels" . | nindent 8 }}
+        app: cvat-app
+        tier: kvrocks
+        {{- with .Values.cvat.kvrocks.labels }}
+        {{- toYaml . | nindent 8 }}
+        {{- end }}
+      {{- with .Values.cvat.kvrocks.annotations }}
+      annotations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+    spec:
+      containers:
+        - name: cvat-kvrocks-app-container
+          image: {{ .Values.cvat.kvrocks.image }}:{{ .Values.cvat.kvrocks.tag }}
+          imagePullPolicy: {{ .Values.cvat.kvrocks.imagePullPolicy }}
+          args:
+            - --dir
+            - /var/lib/kvrocks/data
+            - --requirepass
+            - "$(CVAT_REDIS_ONDISK_PASSWORD)"
+          {{- with .Values.cvat.kvrocks.resources }}
+          resources:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          ports:
+          - containerPort: 6666
+          env:
+          - name: CVAT_REDIS_ONDISK_PASSWORD
+            valueFrom:
+              secretKeyRef:
+                {{- if .Values.cvat.kvrocks.secret.create }}
+                name: "{{ tpl (.Values.cvat.kvrocks.secret.name) . }}"
+                {{- else }}
+                name: "{{ tpl (.Values.cvat.kvrocks.existingSecret) . }}"
+                {{- end }}
+                key: password
+          {{- with .Values.cvat.kvrocks.additionalEnv }}
+          {{- toYaml . | nindent 10 }}
+          {{- end }}
+          volumeMounts:
+          - name: {{ .Release.Name }}-kvrocks-data
+            mountPath: /var/lib/kvrocks/data
+          {{- with .Values.cvat.kvrocks.additionalVolumeMounts }}
+          {{- toYaml . | nindent 10 }}
+          {{- end }}
+      {{- with .Values.cvat.kvrocks.additionalVolumes }}
+      volumes:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.cvat.kvrocks.affinity }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.cvat.kvrocks.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.imagePullSecrets }}
+      imagePullSecrets:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+  {{- if .Values.cvat.kvrocks.defaultStorage.enabled }}
+  volumeClaimTemplates:
+  - metadata:
+      name: {{ .Release.Name }}-kvrocks-data
+      namespace: {{ .Release.Namespace }}
+      labels:
+        {{- include "cvat.labels" . | nindent 8 }}
+        app: cvat-app
+        tier: kvroocks
+    spec:
+      accessModes:
+      {{- if .Values.cvat.kvrocks.defaultStorage.accessModes }}
+      {{ .Values.cvat.kvrocks.defaultStorage.accessModes | toYaml | nindent 6 }}
+      {{- else }}
+      - ReadWriteOnce
+      {{- end }}
+      {{-  if .Values.cvat.kvrocks.defaultStorage.storageClassName }}
+      storageClassName: {{ .Values.cvat.kvrocks.defaultStorage.storageClassName }}
+      {{- end }}
+      resources:
+        requests:
+          storage: {{ .Values.cvat.kvrocks.defaultStorage.size }}
+  {{- end}}
+{{- end }}
diff --git a/helm-chart/values.yaml b/helm-chart/values.yaml
@@ -181,7 +181,7 @@ cvat:
     replicas: 1
     image: openpolicyagent/opa
     tag: 0.45.0-rootless
-    imagepullploicy: IfNotPresent
+    imagePullPolicy: IfNotPresent
     labels: {}
     #  test: test
     annotations: {}
@@ -222,6 +222,55 @@ cvat:
           protocol: TCP
           name: http
 
+  kvrocks:
+    enabled: true
+    external:
+      host: kvrocks-external.localdomain
+    existingSecret: "cvat-kvrocks-secret"
+    secret:
+      create: true
+      name: cvat-kvrocks-secret
+      password: cvat_kvrocks
+    image: apache/kvrocks
+    tag: 2.7.0
+    imagePullPolicy: IfNotPresent
+    labels: {}
+    #  test: test
+    annotations: {}
+    # test.io/test: test
+    resources: {}
+    affinity: {}
+    tolerations: []
+    nodeAffinity: {}
+    #   requiredDuringSchedulingIgnoredDuringExecution:
+    #     nodeSelectorTerms:
+    #     - matchExpressions:
+    #       - key: kubernetes.io/e2e-az-name
+    #         operator: In
+    #         values:
+    #         - e2e-az1
+    #         - e2e-az2
+    additionalEnv: []
+    # Example:
+    # - name: TEST
+    #   value: "test"
+    additionalVolumes: []
+    # Example(assumes that pvc was already created):
+    # - name: tmp
+    #   persistentVolumeClaim:
+    #       claimName: tmp
+    additionalVolumeMounts: []
+    # Example:
+    # -   mountPath: /tmp
+    #     name: tmp
+    #     subPath: test
+    defaultStorage:
+      enabled: true
+#     storageClassName: default
+#     accessModes:
+#       - ReadWriteOnce
+      size: 100Gi
+
 postgresql:
   #See https://github.com/bitnami/charts/blob/master/bitnami/postgresql/ for more info
   enabled: true # false for external db
@@ -258,27 +307,6 @@ redis:
     password: cvat_redis
   # TODO: persistence options
 
-# https://artifacthub.io/packages/helm/enapter/keydb
-keydb:
-  enabled: true
-  external:
-    host: 127.0.0.1
-  existingSecret: "cvat-keydb-secret"
-  secret:
-    create: true
-    name: cvat-keydb-secret
-    password: cvat_keydb
-  multiMaster: false
-  activeReplicas: false
-  nodes: 1
-  configExtraArgs:
-  - storage-provider: ["flash", "/data/flash"]
-  - maxmemory: "5G"
-  - maxmemory-policy: "allkeys-lfu"
-  resources:
-    requests:
-      memory: "7G"
-
 nuclio:
   enabled: false
 # See https://github.com/nuclio/nuclio/blob/master/hack/k8s/helm/nuclio/values.yaml for more info

diff --git a/tests/python/shared/fixtures/init.py b/tests/python/shared/fixtures/init.py
@@ -135,7 +135,7 @@ def _kube_get_redis_inmem_pod_name():
 
 
 def _kube_get_redis_ondisk_pod_name():
-    return _kube_get_pod_name("app.kubernetes.io/name=keydb")
+    return _kube_get_pod_name("app.kubernetes.io/name=cvat,tier=kvrocks")
 
 
 def docker_cp(source, target):
@@ -240,11 +240,13 @@ def kube_restore_redis_inmem():
 
 
 def docker_restore_redis_ondisk():
-    docker_exec_redis_ondisk(["keydb-cli", "flushall"])
+    docker_exec_redis_ondisk(["redis-cli", "-p", "6666", "flushall"])
 
 
 def kube_restore_redis_ondisk():
-    kube_exec_redis_ondisk(["keydb-cli", "flushall"])
+    kube_exec_redis_ondisk(
+        ["redis-cli", "-p", "6666", "-a", "${CVAT_REDIS_ONDISK_PASSWORD}", "flushall"]
+    )
 
 
 def running_containers():