diff --git a/k8s/base/block-ingestor/service.yaml b/k8s/base/block-ingestor/service.yaml new file mode 100644 index 000000000..79086b9ce --- /dev/null +++ b/k8s/base/block-ingestor/service.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Service +metadata: + name: index-node-block-ingestor + labels: + app: block-ingestor + network: all +spec: + selector: + app: block-ingestor + network: all + ports: + - name: json-rpc + protocol: TCP + port: 80 + targetPort: 8020 diff --git a/k8s/base/block-ingestor/stateful_set.yaml b/k8s/base/block-ingestor/stateful_set.yaml new file mode 100644 index 000000000..b6e0e6f1e --- /dev/null +++ b/k8s/base/block-ingestor/stateful_set.yaml @@ -0,0 +1,99 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: index-node-block-ingestor +spec: + selector: + matchLabels: + app: block-ingestor + network: all + serviceName: index-node-block-ingestor + replicas: 1 + podManagementPolicy: Parallel + updateStrategy: + type: RollingUpdate + template: + metadata: + labels: + app: block-ingestor + network: all + annotations: + prometheus.io/scrape: "true" + prometheus.io/path: "/metrics" + prometheus.io/port: "8040" + spec: + nodeSelector: + index: "1" + tolerations: + - key: "index" + operator: "Exists" + effect: "NoSchedule" + volumes: + - name: nfs-shared + persistentVolumeClaim: + claimName: nfs-shared + readOnly: false + - name: graph-node-config + configMap: + name: graph-node-config + containers: + - name: graph-node + image: graph-node-image + resources: + requests: + cpu: 600m + ports: + - name: http + containerPort: 8000 + - name: json-rpc + containerPort: 8020 + - name: metrics + containerPort: 8040 + volumeMounts: + - name: nfs-shared + mountPath: /var/lib/graph + - name: graph-node-config + mountPath: /etc/graph-node + env: + - name: node_id + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: GRAPH_NODE_CONFIG + value: /etc/graph-node/graph-node.toml + - name: GRAPH_KILL_IF_UNRESPONSIVE + value: "true" + - name: GRAPH_LOG + value: debug + - name: postgres_host + valueFrom: + secretKeyRef: + name: postgres-credentials + key: host + - name: postgres_user + valueFrom: + secretKeyRef: + name: postgres-credentials + key: user + - name: postgres_pass + valueFrom: + secretKeyRef: + name: postgres-credentials + key: password + - name: postgres_db + valueFrom: + secretKeyRef: + name: postgres-credentials + key: graph_db + - name: ipfs + value: https://ipfs.network.thegraph.com + - name: GRAPH_ALLOW_NON_DETERMINISTIC_IPFS + value: "true" + - name: GRAPH_ALLOW_NON_DETERMINISTIC_FULLTEXT_SEARCH + value: "true" + + # Avoid some subgraphs fail with "Gas limit exceeded" error + - name: GRAPH_MAX_GAS_PER_HANDLER + value: "1_000_000_000_000_000" + - name: GRAPH_ETHEREUM_REQUEST_RETRIES + value: "30" diff --git a/k8s/base/config/graph-node.toml b/k8s/base/config/graph-node.toml new file mode 100644 index 000000000..9d1ae1106 --- /dev/null +++ b/k8s/base/config/graph-node.toml @@ -0,0 +1,25 @@ +[general] +query = "query_node_.*" + +[store] +[store.primary] +connection = "postgresql://$postgres_user:$postgres_pass@$postgres_host/$postgres_db" +pool_size = [ + { node = "query_node_.*", size = 60 }, + { node = "index_node_.*", size = 35 }, + { node = "index_node_block_ingestor_.*", size = 4 }, +] + +[deployment] +[[deployment.rule]] +store = "primary" +indexers = [ "index_node_0", "index_node_1" ] + +[chains] +ingestor = "index_node_block_ingestor_0" + +[chains.mainnet] +shard = "primary" +provider = [ + { label = "mainnet-0", url = "https://eth-mainnet.node/", features = ["archive", "traces"] } +] diff --git a/k8s/base/grafana.yaml b/k8s/base/grafana.yaml index 52e30cfa7..dfa053861 100644 --- a/k8s/base/grafana.yaml +++ b/k8s/base/grafana.yaml @@ -3226,15 +3226,14 @@ spec: spec: containers: - name: grafana - image: grafana/grafana:latest + image: grafana/grafana:8.3.3 ports: - name: service containerPort: 80 protocol: TCP - name: grafana containerPort: 3000 - resources: - {} + resources: {} volumeMounts: - mountPath: /var/lib/grafana name: grafana-storage @@ -3269,22 +3268,21 @@ spec: emptyDir: {} - name: grafana-datasources configMap: - defaultMode: 420 - name: grafana-datasources + defaultMode: 420 + name: grafana-datasources - name: grafana-dashboards configMap: - defaultMode: 420 - name: grafana-dashboards + defaultMode: 420 + name: grafana-dashboards --- - apiVersion: v1 kind: Service metadata: name: grafana annotations: - prometheus.io/scrape: 'true' - prometheus.io/port: '3000' + prometheus.io/scrape: "true" + prometheus.io/port: "3000" spec: selector: app: grafana diff --git a/k8s/base/index-node/ethereum_networks.yaml b/k8s/base/index-node/ethereum_networks.yaml deleted file mode 100644 index ca84b64da..000000000 --- a/k8s/base/index-node/ethereum_networks.yaml +++ /dev/null @@ -1,8 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: ethereum-networks - namespace: default -data: - networks: >- - mainnet:https://eth-mainnet.node/ \ No newline at end of file diff --git a/k8s/base/index-node/stateful_set.yaml b/k8s/base/index-node/stateful_set.yaml index 6d81aba4c..89c0b1753 100644 --- a/k8s/base/index-node/stateful_set.yaml +++ b/k8s/base/index-node/stateful_set.yaml @@ -31,6 +31,9 @@ spec: persistentVolumeClaim: claimName: nfs-shared readOnly: false + - name: graph-node-config + configMap: + name: graph-node-config containers: - name: graph-node image: graph-node-image @@ -47,10 +50,15 @@ spec: volumeMounts: - name: nfs-shared mountPath: /var/lib/graph + - name: graph-node-config + mountPath: /etc/graph-node env: - # The name of the pod that will do block ingestion - - name: BLOCK_INGESTOR - value: index-node-0 + - name: node_id + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: GRAPH_NODE_CONFIG + value: /etc/graph-node/graph-node.toml - name: GRAPH_KILL_IF_UNRESPONSIVE value: "true" - name: postgres_host @@ -73,16 +81,9 @@ spec: secretKeyRef: name: postgres-credentials key: graph_db - - name: node_role - value: index-node - - name: node_id - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: ethereum - valueFrom: - configMapKeyRef: - name: ethereum-networks - key: networks - name: ipfs value: https://ipfs.network.thegraph.com + + # Avoid some subgraphs fail with "Gas limit exceeded" error + - name: GRAPH_MAX_GAS_PER_HANDLER + value: "1_000_000_000_000_000" diff --git a/k8s/base/indexer-agent/service.yaml b/k8s/base/indexer-agent/service.yaml index b9ac58251..02ad77f1d 100644 --- a/k8s/base/indexer-agent/service.yaml +++ b/k8s/base/indexer-agent/service.yaml @@ -10,4 +10,4 @@ spec: - name: syncing protocol: TCP port: 8002 - targetPort: 8002 + targetPort: 8000 diff --git a/k8s/base/indexer-agent/statefulset.yaml b/k8s/base/indexer-agent/statefulset.yaml index 987f3d0ce..5330a9d21 100644 --- a/k8s/base/indexer-agent/statefulset.yaml +++ b/k8s/base/indexer-agent/statefulset.yaml @@ -97,3 +97,7 @@ spec: secretKeyRef: name: network-subgraph key: endpoint + - name: INDEXER_AGENT_LOG_LEVEL + value: debug + - name: INDEXER_AGENT_ALLOCATION_MANAGEMENT + value: oversight diff --git a/k8s/base/indexer-service/backend_config.yaml b/k8s/base/indexer-service/backend_config.yaml index 145f1c350..37ad9d183 100644 --- a/k8s/base/indexer-service/backend_config.yaml +++ b/k8s/base/indexer-service/backend_config.yaml @@ -3,6 +3,5 @@ kind: BackendConfig metadata: name: indexer-service-backend-config spec: - # Use a long timeout for WebSocket connections connectionDraining: drainingTimeoutSec: 0 diff --git a/k8s/base/indexer-service/statefulset.yaml b/k8s/base/indexer-service/statefulset.yaml index 503c6bff5..dd10e4264 100644 --- a/k8s/base/indexer-service/statefulset.yaml +++ b/k8s/base/indexer-service/statefulset.yaml @@ -52,12 +52,10 @@ spec: secretKeyRef: name: ethereum key: url - - name: INDEXER_SERVICE_NETWORK_SUBGRAPH_ENDPOINT - value: http://index-eragent.default.svc.cluster.local:8002/network - name: INDEXER_SERVICE_GRAPH_NODE_QUERY_ENDPOINT - value: http://query-node.default.svc.cluster.local/ + value: http://query-node.default.svc.cluster.local:8000/ - name: INDEXER_SERVICE_GRAPH_NODE_STATUS_ENDPOINT - value: http://query-node.default.svc.cluster.local/index-node/graphql + value: http://query-node.default.svc.cluster.local:8030/graphql - name: INDEXER_SERVICE_NETWORK_SUBGRAPH_ENDPOINT value: https://gateway.network.thegraph.com/network - name: INDEXER_SERVICE_POSTGRES_HOST @@ -90,3 +88,7 @@ spec: secretKeyRef: name: scalar key: client_signer_address + - name: INDEXER_SERVICE_SERVE_NETWORK_SUBGRAPH + value: "true" + - name: INDEXER_SERVICE_GCLOUD_PROFILING + value: "true" diff --git a/k8s/base/issuer.yaml b/k8s/base/issuer.yaml new file mode 100644 index 000000000..048fc6800 --- /dev/null +++ b/k8s/base/issuer.yaml @@ -0,0 +1,7 @@ +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: self-signed-issuer + namespace: default +spec: + selfSigned: {} diff --git a/k8s/base/kustomization.yaml b/k8s/base/kustomization.yaml index 78ce4f356..b12f4979b 100644 --- a/k8s/base/kustomization.yaml +++ b/k8s/base/kustomization.yaml @@ -1,10 +1,11 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - - index-node/ethereum_networks.yaml + - https://github.com/jetstack/cert-manager/releases/download/v1.9.1/cert-manager.yaml - index-node/stateful_set.yaml - index-node/service.yaml - - query-node/backend_config.yaml + - block-ingestor/stateful_set.yaml + - block-ingestor/service.yaml - query-node/deployment.yaml - query-node/proxy.yaml - query-node/service.yaml @@ -18,5 +19,8 @@ resources: - prometheus.yaml - grafana.yaml - ingress.yaml - +configMapGenerator: + - name: graph-node-config + files: + - config/graph-node.toml namespace: default diff --git a/k8s/base/nfs.yaml b/k8s/base/nfs.yaml index c6e041049..802b3bfb8 100644 --- a/k8s/base/nfs.yaml +++ b/k8s/base/nfs.yaml @@ -22,7 +22,7 @@ spec: name: nfs-server imagePullPolicy: IfNotPresent securityContext: - privileged: true + privileged: true volumeMounts: - name: nfs mountPath: /nfs-share @@ -68,7 +68,7 @@ metadata: name: nfs-shared spec: accessModes: - - ReadWriteMany + - ReadWriteMany storageClassName: "standard" volumeName: nfs-shared resources: diff --git a/k8s/base/prometheus.yaml b/k8s/base/prometheus.yaml index 0b0ec45f0..7f621186c 100644 --- a/k8s/base/prometheus.yaml +++ b/k8s/base/prometheus.yaml @@ -3,24 +3,22 @@ kind: ClusterRole metadata: name: prometheus rules: -- apiGroups: [""] - resources: - - nodes - - nodes/proxy - - services - - endpoints - - pods - verbs: ["get", "list", "watch"] -- apiGroups: - - extensions - resources: - - ingresses - verbs: ["get", "list", "watch"] -- nonResourceURLs: ["/metrics"] - verbs: ["get"] - + - apiGroups: [""] + resources: + - nodes + - nodes/proxy + - services + - endpoints + - pods + verbs: ["get", "list", "watch"] + - apiGroups: + - extensions + resources: + - ingresses + verbs: ["get", "list", "watch"] + - nonResourceURLs: ["/metrics"] + verbs: ["get"] --- - apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: @@ -30,17 +28,16 @@ roleRef: kind: ClusterRole name: prometheus subjects: -- kind: ServiceAccount - name: default - namespace: default - + - kind: ServiceAccount + name: default + namespace: default --- - apiVersion: apps/v1 -kind: Deployment +kind: StatefulSet metadata: name: prometheus spec: + serviceName: prometheus replicas: 1 selector: matchLabels: @@ -93,9 +90,7 @@ spec: - name: prometheus-storage persistentVolumeClaim: claimName: prometheus - --- - apiVersion: v1 kind: ConfigMap metadata: @@ -129,9 +124,7 @@ data: - source_labels: [__meta_kubernetes_pod_name] action: replace target_label: kubernetes_pod_name - --- - apiVersion: v1 kind: ConfigMap metadata: @@ -151,21 +144,23 @@ data: } location /prometheus { + auth_basic "prometheus"; + auth_basic_user_file /etc/nginx/.htpasswd; proxy_pass http://localhost:9099; } } } stream {} - + .htpasswd: | + prometheus:$6$$a62UOs1FosG118RnKdLAxpMOxv5Tt2CSxMef/JGpM6Ug/t2IJbFDyUMzZeZHCvA/Td4PLBh90DWmoUAYZmOeX0 --- - apiVersion: v1 kind: Service metadata: name: prometheus annotations: prometheus.io/scrape: 'true' - prometheus.io/port: '9099' + prometheus.io/port: '9099' spec: type: NodePort selector: diff --git a/k8s/base/query-node/deployment.yaml b/k8s/base/query-node/deployment.yaml index 9f7587ec7..f53672821 100644 --- a/k8s/base/query-node/deployment.yaml +++ b/k8s/base/query-node/deployment.yaml @@ -7,6 +7,12 @@ spec: matchLabels: app: query-node replicas: 1 + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 0 + maxUnavailable: 1 + minReadySeconds: 60 # warm up the cache before deleting old pods template: metadata: labels: @@ -30,6 +36,9 @@ spec: persistentVolumeClaim: claimName: nfs-shared readOnly: false + - name: graph-node-config + configMap: + name: graph-node-config containers: # Proxy that forwards HTTP requests to port 8000 and # WebSocket requests to port 8001 @@ -43,6 +52,11 @@ spec: subPath: nginx.conf # The query node itself - name: graph-node + resources: + requests: + # Set this to a little more than half the number of CPUs + # that one VM in the node pool has + cpu: 5 image: graph-node-image ports: - name: http @@ -51,20 +65,30 @@ spec: containerPort: 8001 - name: index-node containerPort: 8030 + - name: metrics + containerPort: 8040 volumeMounts: - name: nfs-shared mountPath: /var/lib/graph + - name: graph-node-config + mountPath: /etc/graph-node env: # Query nodes connect to Ethereum but don't call the Ethereum node # for anything; this argument could be made obsolete - - name: ethereum - value: mainnet:https://eth-mainnet.node/ + - name: node_role + value: query-node - name: ipfs value: https://ipfs.network.thegraph.com - name: EXPERIMENTAL_SUBGRAPH_VERSION_SWITCHING_MODE value: synced - name: GRAPH_KILL_IF_UNRESPONSIVE value: "true" + - name: GRAPH_QUERY_CACHE_MAX_MEM + value: "3000" + - name: GRAPH_QUERY_CACHE_BLOCKS + value: "6" + - name: GRAPH_QUERY_CACHE_STALE_PERIOD + value: "1000" - name: postgres_host valueFrom: secretKeyRef: @@ -82,9 +106,7 @@ spec: key: password - name: postgres_db value: graph - - name: node_role - value: query-node - name: node_id valueFrom: fieldRef: - fieldPath: metadata.name \ No newline at end of file + fieldPath: metadata.name diff --git a/k8s/base/query-node/service.yaml b/k8s/base/query-node/service.yaml index a07d8ef57..0e6c4b44a 100644 --- a/k8s/base/query-node/service.yaml +++ b/k8s/base/query-node/service.yaml @@ -2,15 +2,17 @@ apiVersion: v1 kind: Service metadata: name: query-node - annotations: - cloud.google.com/neg: '{"ingress": true, "exposed_ports": {"80":{}}}' - beta.cloud.google.com/backend-config: '{"ports": {"80":"query-node-backend-config"}}' spec: - type: LoadBalancer + type: NodePort selector: app: query-node ports: - - name: http + - name: http-nginx protocol: TCP port: 80 - targetPort: 80 + - name: http + port: 8000 + - name: ws + port: 8001 + - name: index-node + port: 8030 diff --git a/k8s/base/shell.yaml b/k8s/base/shell.yaml index 4276b17cc..af5a8f94d 100644 --- a/k8s/base/shell.yaml +++ b/k8s/base/shell.yaml @@ -23,7 +23,7 @@ spec: env: - name: ethereum valueFrom: - configMapKeyRef: + secretKeyRef: name: ethereum-networks key: networks - name: PGHOST diff --git a/k8s/overlays/block_ingestor.yaml b/k8s/overlays/block_ingestor.yaml new file mode 100644 index 000000000..9b1a7c46e --- /dev/null +++ b/k8s/overlays/block_ingestor.yaml @@ -0,0 +1,19 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: index-node-block-ingestor +spec: + # Set this to the number of VM's in the index_pool node pool + replicas: 2 + template: + spec: + containers: + - name: graph-node + resources: + requests: + # Set this to a little more than half the number of CPUs + # that one VM in the node pool has + cpu: 600m + env: + - name: GRAPH_LOG + value: debug diff --git a/k8s/overlays/ethereum_networks.yaml b/k8s/overlays/ethereum_networks.yaml deleted file mode 100644 index 9fc041dfd..000000000 --- a/k8s/overlays/ethereum_networks.yaml +++ /dev/null @@ -1,10 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: ethereum-networks - namespace: default -data: - # List all the networks from which you want to index below. This should - # end up being one string with space separated entries. - networks: >- - mainnet:https://eth-mainnet.node/ \ No newline at end of file diff --git a/k8s/overlays/index_node.yaml b/k8s/overlays/index_node.yaml index 6db7ac266..ac0c8e5ec 100644 --- a/k8s/overlays/index_node.yaml +++ b/k8s/overlays/index_node.yaml @@ -17,3 +17,7 @@ spec: env: - name: GRAPH_LOG value: debug + - name: GRAPH_ALLOW_NON_DETERMINISTIC_IPFS + value: "true" + - name: GRAPH_ALLOW_NON_DETERMINISTIC_FULLTEXT_SEARCH + value: "true" diff --git a/k8s/overlays/indexer_agent.yaml b/k8s/overlays/indexer_agent.yaml index d38f80180..f2c951062 100644 --- a/k8s/overlays/indexer_agent.yaml +++ b/k8s/overlays/indexer_agent.yaml @@ -12,7 +12,7 @@ spec: - name: INDEXER_AGENT_INDEX_NODE_IDS value: index_node_0,index_node_1 - # Set this to your indexer's geo location + # Set this to your indexer's lat, long coordinates - name: INDEXER_AGENT_INDEXER_GEO_COORDINATES value: "0.00 0.00" diff --git a/k8s/overlays/indexer_service.yaml b/k8s/overlays/indexer_service.yaml index 3afab224a..488a65774 100644 --- a/k8s/overlays/indexer_service.yaml +++ b/k8s/overlays/indexer_service.yaml @@ -3,7 +3,7 @@ kind: StatefulSet metadata: name: indexer-service spec: - replicas: 4 + replicas: 2 template: spec: containers: diff --git a/k8s/overlays/kustomization.yaml b/k8s/overlays/kustomization.yaml index 421f09206..db9edfcb6 100644 --- a/k8s/overlays/kustomization.yaml +++ b/k8s/overlays/kustomization.yaml @@ -5,31 +5,30 @@ bases: images: - name: graph-node-image newName: graphprotocol/graph-node - newTag: latest + newTag: v0.27.0 - name: graph-node-debug newName: graphprotocol/graph-node-debug - newTag: latest + newTag: v0.27.0 - name: indexer-agent-image newName: ghcr.io/graphprotocol/indexer-agent - newTag: latest + newTag: v0.20.4 - name: indexer-service-image newName: ghcr.io/graphprotocol/indexer-service - newTag: latest + newTag: v0.20.4 - name: prometheus-image newName: prom/prometheus - newTag: v2.23.0 + newTag: v2.32.1 - name: busybox-image newName: busybox newTag: "1.30" - name: nginx-image newName: nginx newTag: latest - - name: vector-node - newName: connextproject/vector_node - newTag: a340c8a4 + - name: gists/nfs-server + newTag: 2.5.4 patchesStrategicMerge: - - ethereum_networks.yaml - index_node.yaml + - block_ingestor.yaml - query_node.yaml - indexer_agent.yaml - indexer_service.yaml diff --git a/k8s/overlays/query_node.yaml b/k8s/overlays/query_node.yaml index a40e1e5f3..00ea8c340 100644 --- a/k8s/overlays/query_node.yaml +++ b/k8s/overlays/query_node.yaml @@ -15,6 +15,5 @@ spec: # that one VM in the node pool has cpu: 600m env: - # Set this to your Ethereum node - - name: ethereum - value: mainnet:https://eth-mainnet.node/ \ No newline at end of file + - name: GRAPH_LOG + value: debug