k8s: Update resources/configs to use the latest recommendations

- Use graph-node.toml config file - Use dedicated block_ingestor pod - Improve query-node update strategy - Use specific image version tags rather than `latest` - Include cert manager
ozgrakkurt · Oct 11, 2022 · 0d10609 · 0d10609
1 parent 76bff5c
commit 0d10609
Show file tree

Hide file tree

Showing 24 changed files with 286 additions and 109 deletions.
diff --git a/k8s/base/block-ingestor/service.yaml b/k8s/base/block-ingestor/service.yaml
@@ -0,0 +1,16 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: index-node-block-ingestor
+  labels:
+    app: block-ingestor
+    network: all
+spec:
+  selector:
+    app: block-ingestor
+    network: all
+  ports:
+  - name: json-rpc
+    protocol: TCP
+    port: 80
+    targetPort: 8020
diff --git a/k8s/base/block-ingestor/stateful_set.yaml b/k8s/base/block-ingestor/stateful_set.yaml
@@ -0,0 +1,99 @@
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: index-node-block-ingestor
+spec:
+  selector:
+    matchLabels:
+      app: block-ingestor
+      network: all
+  serviceName: index-node-block-ingestor
+  replicas: 1
+  podManagementPolicy: Parallel
+  updateStrategy:
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        app: block-ingestor
+        network: all
+      annotations:
+        prometheus.io/scrape: "true"
+        prometheus.io/path: "/metrics"
+        prometheus.io/port: "8040"
+    spec:
+      nodeSelector:
+        index: "1"
+      tolerations:
+        - key: "index"
+          operator: "Exists"
+          effect: "NoSchedule"
+      volumes:
+        - name: nfs-shared
+          persistentVolumeClaim:
+            claimName: nfs-shared
+            readOnly: false
+        - name: graph-node-config
+          configMap:
+            name: graph-node-config
+      containers:
+        - name: graph-node
+          image: graph-node-image
+          resources:
+            requests:
+              cpu: 600m
+          ports:
+            - name: http
+              containerPort: 8000
+            - name: json-rpc
+              containerPort: 8020
+            - name: metrics
+              containerPort: 8040
+          volumeMounts:
+            - name: nfs-shared
+              mountPath: /var/lib/graph
+            - name: graph-node-config
+              mountPath: /etc/graph-node
+          env:
+            - name: node_id
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: GRAPH_NODE_CONFIG
+              value: /etc/graph-node/graph-node.toml
+            - name: GRAPH_KILL_IF_UNRESPONSIVE
+              value: "true"
+            - name: GRAPH_LOG
+              value: debug
+            - name: postgres_host
+              valueFrom:
+                secretKeyRef:
+                  name: postgres-credentials
+                  key: host
+            - name: postgres_user
+              valueFrom:
+                secretKeyRef:
+                  name: postgres-credentials
+                  key: user
+            - name: postgres_pass
+              valueFrom:
+                secretKeyRef:
+                  name: postgres-credentials
+                  key: password
+            - name: postgres_db
+              valueFrom:
+                secretKeyRef:
+                  name: postgres-credentials
+                  key: graph_db
+            - name: ipfs
+              value: https://ipfs.network.thegraph.com
+            - name: GRAPH_ALLOW_NON_DETERMINISTIC_IPFS
+              value: "true"
+            - name: GRAPH_ALLOW_NON_DETERMINISTIC_FULLTEXT_SEARCH
+              value: "true"
+
+            # Avoid some subgraphs fail with "Gas limit exceeded" error
+            - name: GRAPH_MAX_GAS_PER_HANDLER
+              value: "1_000_000_000_000_000"
+            - name: GRAPH_ETHEREUM_REQUEST_RETRIES
+              value: "30"
diff --git a/k8s/base/config/graph-node.toml b/k8s/base/config/graph-node.toml
@@ -0,0 +1,25 @@
+[general]
+query = "query_node_.*"
+
+[store]
+[store.primary]
+connection = "postgresql://$postgres_user:$postgres_pass@$postgres_host/$postgres_db"
+pool_size = [
+    { node = "query_node_.*", size = 60 },
+    { node = "index_node_.*", size = 35 },
+    { node = "index_node_block_ingestor_.*", size = 4 },
+]
+
+[deployment]
+[[deployment.rule]]
+store = "primary"
+indexers = [ "index_node_0", "index_node_1" ]
+
+[chains]
+ingestor = "index_node_block_ingestor_0"
+
+[chains.mainnet]
+shard = "primary"
+provider = [
+    { label = "mainnet-0", url = "https://eth-mainnet.node/", features = ["archive", "traces"] }
+]
diff --git a/k8s/base/grafana.yaml b/k8s/base/grafana.yaml
@@ -3226,15 +3226,14 @@ spec:
     spec:
       containers:
       - name: grafana
-        image: grafana/grafana:latest
+        image: grafana/grafana:8.3.3
         ports:
           - name: service
             containerPort: 80
             protocol: TCP
           - name: grafana
             containerPort: 3000
-        resources:
-          {}
+        resources: {}
         volumeMounts:
           - mountPath: /var/lib/grafana
             name: grafana-storage
@@ -3269,22 +3268,21 @@ spec:
           emptyDir: {}
         - name: grafana-datasources
           configMap:
-              defaultMode: 420
-              name: grafana-datasources
+            defaultMode: 420
+            name: grafana-datasources
         - name: grafana-dashboards
           configMap:
-              defaultMode: 420
-              name: grafana-dashboards
+            defaultMode: 420
+            name: grafana-dashboards
 
 ---
-
 apiVersion: v1
 kind: Service
 metadata:
   name: grafana
   annotations:
-      prometheus.io/scrape: 'true'
-      prometheus.io/port:   '3000'
+    prometheus.io/scrape: "true"
+    prometheus.io/port: "3000"
 spec:
   selector:
     app: grafana

diff --git a/k8s/base/index-node/ethereum_networks.yaml b/k8s/base/index-node/ethereum_networks.yaml
diff --git a/k8s/base/index-node/stateful_set.yaml b/k8s/base/index-node/stateful_set.yaml
@@ -31,6 +31,9 @@ spec:
           persistentVolumeClaim:
             claimName: nfs-shared
             readOnly: false
+        - name: graph-node-config
+          configMap:
+            name: graph-node-config
       containers:
         - name: graph-node
           image: graph-node-image
@@ -47,10 +50,15 @@ spec:
           volumeMounts:
             - name: nfs-shared
               mountPath: /var/lib/graph
+            - name: graph-node-config
+              mountPath: /etc/graph-node
           env:
-            # The name of the pod that will do block ingestion
-            - name: BLOCK_INGESTOR
-              value: index-node-0
+            - name: node_id
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: GRAPH_NODE_CONFIG
+              value: /etc/graph-node/graph-node.toml
             - name: GRAPH_KILL_IF_UNRESPONSIVE
               value: "true"
             - name: postgres_host
@@ -73,16 +81,9 @@ spec:
                 secretKeyRef:
                   name: postgres-credentials
                   key: graph_db
-            - name: node_role
-              value: index-node
-            - name: node_id
-              valueFrom:
-                fieldRef:
-                  fieldPath: metadata.name
-            - name: ethereum
-              valueFrom:
-                configMapKeyRef:
-                  name: ethereum-networks
-                  key: networks
             - name: ipfs
               value: https://ipfs.network.thegraph.com
+
+            # Avoid some subgraphs fail with "Gas limit exceeded" error
+            - name: GRAPH_MAX_GAS_PER_HANDLER
+              value: "1_000_000_000_000_000"
diff --git a/k8s/base/indexer-agent/service.yaml b/k8s/base/indexer-agent/service.yaml
@@ -10,4 +10,4 @@ spec:
     - name: syncing
       protocol: TCP
       port: 8002
-      targetPort: 8002
+      targetPort: 8000
diff --git a/k8s/base/indexer-agent/statefulset.yaml b/k8s/base/indexer-agent/statefulset.yaml
@@ -97,3 +97,7 @@ spec:
                 secretKeyRef:
                   name: network-subgraph
                   key: endpoint
+            - name: INDEXER_AGENT_LOG_LEVEL
+              value: debug
+            - name: INDEXER_AGENT_ALLOCATION_MANAGEMENT
+              value: oversight
diff --git a/k8s/base/indexer-service/backend_config.yaml b/k8s/base/indexer-service/backend_config.yaml
@@ -3,6 +3,5 @@ kind: BackendConfig
 metadata:
   name: indexer-service-backend-config
 spec:
-  # Use a long timeout for WebSocket connections
   connectionDraining:
     drainingTimeoutSec: 0
diff --git a/k8s/base/indexer-service/statefulset.yaml b/k8s/base/indexer-service/statefulset.yaml
@@ -52,12 +52,10 @@ spec:
                 secretKeyRef:
                   name: ethereum
                   key: url
-            - name: INDEXER_SERVICE_NETWORK_SUBGRAPH_ENDPOINT
-              value: http://index-eragent.default.svc.cluster.local:8002/network
             - name: INDEXER_SERVICE_GRAPH_NODE_QUERY_ENDPOINT
-              value: http://query-node.default.svc.cluster.local/
+              value: http://query-node.default.svc.cluster.local:8000/
             - name: INDEXER_SERVICE_GRAPH_NODE_STATUS_ENDPOINT
-              value: http://query-node.default.svc.cluster.local/index-node/graphql
+              value: http://query-node.default.svc.cluster.local:8030/graphql
             - name: INDEXER_SERVICE_NETWORK_SUBGRAPH_ENDPOINT
               value: https://gateway.network.thegraph.com/network
             - name: INDEXER_SERVICE_POSTGRES_HOST
@@ -90,3 +88,7 @@ spec:
                 secretKeyRef:
                   name: scalar
                   key: client_signer_address
+            - name: INDEXER_SERVICE_SERVE_NETWORK_SUBGRAPH
+              value: "true"
+            - name: INDEXER_SERVICE_GCLOUD_PROFILING
+              value: "true"
diff --git a/k8s/base/issuer.yaml b/k8s/base/issuer.yaml
@@ -0,0 +1,7 @@
+apiVersion: cert-manager.io/v1
+kind: Issuer
+metadata:
+  name: self-signed-issuer
+  namespace: default
+spec:
+  selfSigned: {}
diff --git a/k8s/base/kustomization.yaml b/k8s/base/kustomization.yaml
@@ -1,10 +1,11 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
-  - index-node/ethereum_networks.yaml
+  - https://github.com/jetstack/cert-manager/releases/download/v1.9.1/cert-manager.yaml
   - index-node/stateful_set.yaml
   - index-node/service.yaml
-  - query-node/backend_config.yaml
+  - block-ingestor/stateful_set.yaml
+  - block-ingestor/service.yaml
   - query-node/deployment.yaml
   - query-node/proxy.yaml
   - query-node/service.yaml
@@ -18,5 +19,8 @@ resources:
   - prometheus.yaml
   - grafana.yaml
   - ingress.yaml
-
+configMapGenerator:
+  - name: graph-node-config
+    files:
+      - config/graph-node.toml
 namespace: default
diff --git a/k8s/base/nfs.yaml b/k8s/base/nfs.yaml
@@ -22,7 +22,7 @@ spec:
         name: nfs-server
         imagePullPolicy: IfNotPresent
         securityContext:
-            privileged: true
+          privileged: true
         volumeMounts:
           - name: nfs
             mountPath: /nfs-share
@@ -68,7 +68,7 @@ metadata:
   name: nfs-shared
 spec:
   accessModes:
-  - ReadWriteMany
+    - ReadWriteMany
   storageClassName: "standard"
   volumeName: nfs-shared
   resources: