feat: support gzip compression for data stored in redis (argoproj#10190)

* feat: support gzip compression for data stored in redis Signed-off-by: Alexander Matyushentsev <[email protected]> * allow using different redis compression algorithms Signed-off-by: Alexander Matyushentsev <[email protected]> Signed-off-by: Alexander Matyushentsev <[email protected]>
yaron · Aug 15, 2022 · 9b4f468 · 9b4f468
1 parent e25f21e
commit 9b4f468
Show file tree

Hide file tree

Showing 19 changed files with 227 additions and 20 deletions.
diff --git a/cmd/argocd/commands/admin/cluster.go b/cmd/argocd/commands/admin/cluster.go
@@ -84,7 +84,7 @@ func loadClusters(ctx context.Context, kubeClient *kubernetes.Clientset, appClie
 			return nil, err
 		}
 		client := redis.NewClient(&redis.Options{Addr: fmt.Sprintf("localhost:%d", port)})
-		cache = appstatecache.NewCache(cacheutil.NewCache(cacheutil.NewRedisCache(client, time.Hour)), time.Hour)
+		cache = appstatecache.NewCache(cacheutil.NewCache(cacheutil.NewRedisCache(client, time.Hour, cacheutil.RedisCompressionNone)), time.Hour)
 	} else {
 		cache, err = cacheSrc()
 		if err != nil {

diff --git a/cmd/argocd/commands/headless/headless.go b/cmd/argocd/commands/headless/headless.go
@@ -58,7 +58,7 @@ func (c *forwardCacheClient) doLazy(action func(client cache.CacheClient) error)
 		}
 
 		redisClient := redis.NewClient(&redis.Options{Addr: fmt.Sprintf("localhost:%d", redisPort)})
-		c.client = cache.NewRedisCache(redisClient, time.Hour)
+		c.client = cache.NewRedisCache(redisClient, time.Hour, cache.RedisCompressionNone)
 	})
 	if c.err != nil {
 		return c.err

diff --git a/docs/operator-manual/argocd-cmd-params-cm.yaml b/docs/operator-manual/argocd-cmd-params-cm.yaml
@@ -14,6 +14,8 @@ data:
 
   # Redis server hostname and port (e.g. argocd-redis:6379)
   redis.server: "argocd-redis:6379"
+  # Enable compression for data sent to Redis with the required compression algorithm. (default 'none')
+  redis.compression: none
   # Redis database
   redis.db:
 

diff --git a/docs/operator-manual/server-commands/argocd-application-controller.md b/docs/operator-manual/server-commands/argocd-application-controller.md
@@ -45,6 +45,7 @@ argocd-application-controller [flags]
       --redis-ca-certificate string           Path to Redis server CA certificate (e.g. /etc/certs/redis/ca.crt). If not specified, system trusted CAs will be used for server certificate validation.
       --redis-client-certificate string       Path to Redis client certificate (e.g. /etc/certs/redis/client.crt).
       --redis-client-key string               Path to Redis client key (e.g. /etc/certs/redis/client.crt).
+      --redis-compress string                 Enable compression for data sent to Redis with the required compression algorithm. (possible values: none, gzip) (default "none")
       --redis-insecure-skip-tls-verify        Skip Redis server certificate validation.
       --redis-use-tls                         Use TLS when connecting to Redis. 
       --redisdb int                           Redis database.

diff --git a/docs/operator-manual/server-commands/argocd-repo-server.md b/docs/operator-manual/server-commands/argocd-repo-server.md
@@ -29,6 +29,7 @@ argocd-repo-server [flags]
       --redis-ca-certificate string                    Path to Redis server CA certificate (e.g. /etc/certs/redis/ca.crt). If not specified, system trusted CAs will be used for server certificate validation.
       --redis-client-certificate string                Path to Redis client certificate (e.g. /etc/certs/redis/client.crt).
       --redis-client-key string                        Path to Redis client key (e.g. /etc/certs/redis/client.crt).
+      --redis-compress string                          Enable compression for data sent to Redis with the required compression algorithm. (possible values: none, gzip) (default "none")
       --redis-insecure-skip-tls-verify                 Skip Redis server certificate validation.
       --redis-use-tls                                  Use TLS when connecting to Redis. 
       --redisdb int                                    Redis database.

diff --git a/docs/operator-manual/server-commands/argocd-server.md b/docs/operator-manual/server-commands/argocd-server.md
@@ -51,6 +51,7 @@ argocd-server [flags]
       --redis-ca-certificate string                   Path to Redis server CA certificate (e.g. /etc/certs/redis/ca.crt). If not specified, system trusted CAs will be used for server certificate validation.
       --redis-client-certificate string               Path to Redis client certificate (e.g. /etc/certs/redis/client.crt).
       --redis-client-key string                       Path to Redis client key (e.g. /etc/certs/redis/client.crt).
+      --redis-compress string                         Enable compression for data sent to Redis with the required compression algorithm. (possible values: none, gzip) (default "none")
       --redis-insecure-skip-tls-verify                Skip Redis server certificate validation.
       --redis-use-tls                                 Use TLS when connecting to Redis. 
       --redisdb int                                   Redis database.

diff --git a/docs/user-guide/commands/argocd_admin_cluster_shards.md b/docs/user-guide/commands/argocd_admin_cluster_shards.md
@@ -30,6 +30,7 @@ argocd admin cluster shards [flags]
       --redis-ca-certificate string           Path to Redis server CA certificate (e.g. /etc/certs/redis/ca.crt). If not specified, system trusted CAs will be used for server certificate validation.
       --redis-client-certificate string       Path to Redis client certificate (e.g. /etc/certs/redis/client.crt).
       --redis-client-key string               Path to Redis client key (e.g. /etc/certs/redis/client.crt).
+      --redis-compress string                 Enable compression for data sent to Redis with the required compression algorithm. (possible values: none, gzip) (default "none")
       --redis-insecure-skip-tls-verify        Skip Redis server certificate validation.
       --redis-use-tls                         Use TLS when connecting to Redis. 
       --redisdb int                           Redis database.

diff --git a/docs/user-guide/commands/argocd_admin_cluster_stats.md b/docs/user-guide/commands/argocd_admin_cluster_stats.md
@@ -30,6 +30,7 @@ argocd admin cluster stats [flags]
       --redis-ca-certificate string           Path to Redis server CA certificate (e.g. /etc/certs/redis/ca.crt). If not specified, system trusted CAs will be used for server certificate validation.
       --redis-client-certificate string       Path to Redis client certificate (e.g. /etc/certs/redis/client.crt).
       --redis-client-key string               Path to Redis client key (e.g. /etc/certs/redis/client.crt).
+      --redis-compress string                 Enable compression for data sent to Redis with the required compression algorithm. (possible values: none, gzip) (default "none")
       --redis-insecure-skip-tls-verify        Skip Redis server certificate validation.
       --redis-use-tls                         Use TLS when connecting to Redis. 
       --redisdb int                           Redis database.

diff --git a/manifests/base/application-controller/argocd-application-controller-statefulset.yaml b/manifests/base/application-controller/argocd-application-controller-statefulset.yaml
@@ -107,6 +107,12 @@ spec:
                 name: argocd-cmd-params-cm
                 key: redis.server
                 optional: true
+        - name: REDIS_COMPRESSION
+          valueFrom:
+            configMapKeyRef:
+              name: argocd-cmd-params-cm
+              key: redis.compression
+              optional: true
         - name: REDISDB
           valueFrom:
               configMapKeyRef:

diff --git a/manifests/base/repo-server/argocd-repo-server-deployment.yaml b/manifests/base/repo-server/argocd-repo-server-deployment.yaml
@@ -83,6 +83,12 @@ spec:
                   name: argocd-cmd-params-cm
                   key: redis.server
                   optional: true
+          - name: REDIS_COMPRESSION
+            valueFrom:
+              configMapKeyRef:
+                name: argocd-cmd-params-cm
+                key: redis.compression
+                optional: true
           - name: REDISDB
             valueFrom:
                 configMapKeyRef:

diff --git a/manifests/base/server/argocd-server-deployment.yaml b/manifests/base/server/argocd-server-deployment.yaml
@@ -172,6 +172,12 @@ spec:
                 name: argocd-cmd-params-cm
                 key: redis.server
                 optional: true
+        - name: REDIS_COMPRESSION
+          valueFrom:
+            configMapKeyRef:
+              name: argocd-cmd-params-cm
+              key: redis.compression
+              optional: true
         - name: REDISDB
           valueFrom:
               configMapKeyRef:

diff --git a/manifests/core-install.yaml b/manifests/core-install.yaml
@@ -9810,6 +9810,12 @@ spec:
               key: redis.server
               name: argocd-cmd-params-cm
               optional: true
+        - name: REDIS_COMPRESSION
+          valueFrom:
+            configMapKeyRef:
+              key: redis.compression
+              name: argocd-cmd-params-cm
+              optional: true
         - name: REDISDB
           valueFrom:
             configMapKeyRef:
@@ -10074,6 +10080,12 @@ spec:
               key: redis.server
               name: argocd-cmd-params-cm
               optional: true
+        - name: REDIS_COMPRESSION
+          valueFrom:
+            configMapKeyRef:
+              key: redis.compression
+              name: argocd-cmd-params-cm
+              optional: true
         - name: REDISDB
           valueFrom:
             configMapKeyRef:

diff --git a/manifests/ha/install.yaml b/manifests/ha/install.yaml
@@ -11260,6 +11260,12 @@ spec:
               key: redis.server
               name: argocd-cmd-params-cm
               optional: true
+        - name: REDIS_COMPRESSION
+          valueFrom:
+            configMapKeyRef:
+              key: redis.compression
+              name: argocd-cmd-params-cm
+              optional: true
         - name: REDISDB
           valueFrom:
             configMapKeyRef:
@@ -11590,6 +11596,12 @@ spec:
               key: redis.server
               name: argocd-cmd-params-cm
               optional: true
+        - name: REDIS_COMPRESSION
+          valueFrom:
+            configMapKeyRef:
+              key: redis.compression
+              name: argocd-cmd-params-cm
+              optional: true
         - name: REDISDB
           valueFrom:
             configMapKeyRef:
@@ -11823,6 +11835,12 @@ spec:
               key: redis.server
               name: argocd-cmd-params-cm
               optional: true
+        - name: REDIS_COMPRESSION
+          valueFrom:
+            configMapKeyRef:
+              key: redis.compression
+              name: argocd-cmd-params-cm
+              optional: true
         - name: REDISDB
           valueFrom:
             configMapKeyRef:

diff --git a/manifests/ha/namespace-install.yaml b/manifests/ha/namespace-install.yaml
@@ -1949,6 +1949,12 @@ spec:
               key: redis.server
               name: argocd-cmd-params-cm
               optional: true
+        - name: REDIS_COMPRESSION
+          valueFrom:
+            configMapKeyRef:
+              key: redis.compression
+              name: argocd-cmd-params-cm
+              optional: true
         - name: REDISDB
           valueFrom:
             configMapKeyRef:
@@ -2279,6 +2285,12 @@ spec:
               key: redis.server
               name: argocd-cmd-params-cm
               optional: true
+        - name: REDIS_COMPRESSION
+          valueFrom:
+            configMapKeyRef:
+              key: redis.compression
+              name: argocd-cmd-params-cm
+              optional: true
         - name: REDISDB
           valueFrom:
             configMapKeyRef:
@@ -2512,6 +2524,12 @@ spec:
               key: redis.server
               name: argocd-cmd-params-cm
               optional: true
+        - name: REDIS_COMPRESSION
+          valueFrom:
+            configMapKeyRef:
+              key: redis.compression
+              name: argocd-cmd-params-cm
+              optional: true
         - name: REDISDB
           valueFrom:
             configMapKeyRef:

diff --git a/manifests/install.yaml b/manifests/install.yaml
@@ -10287,6 +10287,12 @@ spec:
               key: redis.server
               name: argocd-cmd-params-cm
               optional: true
+        - name: REDIS_COMPRESSION
+          valueFrom:
+            configMapKeyRef:
+              key: redis.compression
+              name: argocd-cmd-params-cm
+              optional: true
         - name: REDISDB
           valueFrom:
             configMapKeyRef:
@@ -10613,6 +10619,12 @@ spec:
               key: redis.server
               name: argocd-cmd-params-cm
               optional: true
+        - name: REDIS_COMPRESSION
+          valueFrom:
+            configMapKeyRef:
+              key: redis.compression
+              name: argocd-cmd-params-cm
+              optional: true
         - name: REDISDB
           valueFrom:
             configMapKeyRef:
@@ -10840,6 +10852,12 @@ spec:
               key: redis.server
               name: argocd-cmd-params-cm
               optional: true
+        - name: REDIS_COMPRESSION
+          valueFrom:
+            configMapKeyRef:
+              key: redis.compression
+              name: argocd-cmd-params-cm
+              optional: true
         - name: REDISDB
           valueFrom:
             configMapKeyRef:

diff --git a/manifests/namespace-install.yaml b/manifests/namespace-install.yaml
@@ -976,6 +976,12 @@ spec:
               key: redis.server
               name: argocd-cmd-params-cm
               optional: true
+        - name: REDIS_COMPRESSION
+          valueFrom:
+            configMapKeyRef:
+              key: redis.compression
+              name: argocd-cmd-params-cm
+              optional: true
         - name: REDISDB
           valueFrom:
             configMapKeyRef:
@@ -1302,6 +1308,12 @@ spec:
               key: redis.server
               name: argocd-cmd-params-cm
               optional: true
+        - name: REDIS_COMPRESSION
+          valueFrom:
+            configMapKeyRef:
+              key: redis.compression
+              name: argocd-cmd-params-cm
+              optional: true
         - name: REDISDB
           valueFrom:
             configMapKeyRef:
@@ -1529,6 +1541,12 @@ spec:
               key: redis.server
               name: argocd-cmd-params-cm
               optional: true
+        - name: REDIS_COMPRESSION
+          valueFrom:
+            configMapKeyRef:
+              key: redis.compression
+              name: argocd-cmd-params-cm
+              optional: true
         - name: REDISDB
           valueFrom:
             configMapKeyRef:

diff --git a/util/cache/cache.go b/util/cache/cache.go
@@ -83,6 +83,7 @@ func AddCacheFlagsToCmd(cmd *cobra.Command, opts ...func(client *redis.Client))
 	redisClientKey := ""
 	redisUseTLS := false
 	insecureRedis := false
+	compressionStr := ""
 	var defaultCacheExpiration time.Duration
 
 	cmd.Flags().StringVar(&redisAddress, "redis", env.StringFromEnv("REDIS_SERVER", ""), "Redis server hostname and port (e.g. argocd-redis:6379). ")
@@ -95,6 +96,7 @@ func AddCacheFlagsToCmd(cmd *cobra.Command, opts ...func(client *redis.Client))
 	cmd.Flags().StringVar(&redisClientKey, "redis-client-key", "", "Path to Redis client key (e.g. /etc/certs/redis/client.crt).")
 	cmd.Flags().BoolVar(&insecureRedis, "redis-insecure-skip-tls-verify", false, "Skip Redis server certificate validation.")
 	cmd.Flags().StringVar(&redisCACerticate, "redis-ca-certificate", "", "Path to Redis server CA certificate (e.g. /etc/certs/redis/ca.crt). If not specified, system trusted CAs will be used for server certificate validation.")
+	cmd.Flags().StringVar(&compressionStr, "redis-compress", env.StringFromEnv("REDIS_COMPRESSION", string(RedisCompressionNone)), "Enable compression for data sent to Redis with the required compression algorithm. (possible values: none, gzip)")
 	return func() (*Cache, error) {
 		var tlsConfig *tls.Config = nil
 		if redisUseTLS {
@@ -125,12 +127,16 @@ func AddCacheFlagsToCmd(cmd *cobra.Command, opts ...func(client *redis.Client))
 		password := os.Getenv(envRedisPassword)
 		username := os.Getenv(envRedisUsername)
 		maxRetries := env.ParseNumFromEnv(envRedisRetryCount, defaultRedisRetryCount, 0, math.MaxInt32)
+		compression, err := CompressionTypeFromString(compressionStr)
+		if err != nil {
+			return nil, err
+		}
 		if len(sentinelAddresses) > 0 {
 			client := buildFailoverRedisClient(sentinelMaster, password, username, redisDB, maxRetries, tlsConfig, sentinelAddresses)
 			for i := range opts {
 				opts[i](client)
 			}
-			return NewCache(NewRedisCache(client, defaultCacheExpiration)), nil
+			return NewCache(NewRedisCache(client, defaultCacheExpiration, compression)), nil
 		}
 		if redisAddress == "" {
 			redisAddress = common.DefaultRedisAddr
@@ -140,7 +146,7 @@ func AddCacheFlagsToCmd(cmd *cobra.Command, opts ...func(client *redis.Client))
 		for i := range opts {
 			opts[i](client)
 		}
-		return NewCache(NewRedisCache(client, defaultCacheExpiration)), nil
+		return NewCache(NewRedisCache(client, defaultCacheExpiration, compression)), nil
 	}
 }