Helm: add alertmanager fallback config (grafana#3360)

Adds the possibility to define a fallback config for alertmanager from the helm chart config Signed-off-by: Dimitar Dimitrov <[email protected]> Co-authored-by: duncan485 <[email protected]>
nullren · Nov 7, 2022 · 2a8da5d · 2a8da5d
1 parent 76ad50b
commit 2a8da5d
Show file tree

Hide file tree

Showing 50 changed files with 501 additions and 5 deletions.
diff --git a/operations/compare-helm-with-jsonnet/components/config/kustomization.yaml b/operations/compare-helm-with-jsonnet/components/config/kustomization.yaml
@@ -144,3 +144,10 @@ patches:
       # This makes it simpler to just scale up the compactor and improve compaction time
       - op: remove
         path: /config/limits/compactor_tenant_shard_size
+
+  - target:
+      kind: MimirConfig
+      name: 'compactor|distributor|ingester|overrides-exporter|querier|query-frontend|query-scheduler|ruler|store-gateway'
+    patch: |-
+      - op: remove
+        path: /config/alertmanager/fallback_config_file
diff --git a/operations/helm/charts/mimir-distributed/CHANGELOG.md b/operations/helm/charts/mimir-distributed/CHANGELOG.md
@@ -84,6 +84,7 @@ Entries should include a reference to the Pull Request that introduced the chang
   * Unset `frontend.log_queries_longer_than` (Was `10s`, now defaults to `0`, which is disabled)
 * [ENHANCEMENT] Added `usage_stats.installation_mode` configuration to track the installation mode via the anonymous usage statistics. #3294
 * [ENHANCEMENT] Update grafana-agent-operator subchart to 0.2.8. Notable changes are being able to configure Pod's SecurityContext and Container's SecurityContext. #3350
+* [ENHANCEMENT] Add possibility to configure fallbackConfig for alertmanager and set it by default. Now tenants without an alertmanager config will not see errors accessing the alertmanager UI or when using the alertmanager API. #3360
 * [BUGFIX] Fix an issue that caused metamonitoring secrets to be created incorrectly #3170
 * [BUGFIX] Nginx: fixed `imagePullSecret` value reference inconsistency. #3208
 * [BUGFIX] Move the activity tracker log from /data to /active-query-tracker to remove ignore log messages. #3169

diff --git a/operations/helm/charts/mimir-distributed/templates/alertmanager/alertmanager-config.yaml b/operations/helm/charts/mimir-distributed/templates/alertmanager/alertmanager-config.yaml
@@ -0,0 +1,16 @@
+{{- if .Values.alertmanager.enabled -}}
+{{- if .Values.alertmanager.fallbackConfig -}}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ include "mimir.resourceName" (dict "ctx" . "component" "alertmanager") }}-fallback-config
+  labels:
+    {{- include "mimir.labels" (dict "ctx" . "component" "alertmanager" "memberlist" true) | nindent 4 }}
+  annotations:
+  {{- toYaml .Values.alertmanager.annotations | nindent 4 }} 
+  namespace: {{ .Release.Namespace | quote }}
+data:
+  alertmanager_fallback_config.yaml: |
+    {{- .Values.alertmanager.fallbackConfig | nindent 4 }}
+{{- end -}}
+{{- end -}}
diff --git a/operations/helm/charts/mimir-distributed/templates/alertmanager/alertmanager-dep.yaml b/operations/helm/charts/mimir-distributed/templates/alertmanager/alertmanager-dep.yaml
@@ -22,6 +22,9 @@ spec:
         {{- include "mimir.podLabels" (dict "ctx" . "component" "alertmanager" "memberlist" true) | nindent 8 }}
       annotations:
         {{- include "mimir.podAnnotations" (dict "ctx" . "component" "alertmanager") | nindent 8 }}
+        {{- if .Values.alertmanager.fallbackConfig }}
+        checksum/alertmanager-fallback-config: {{ include (print .Template.BasePath "/alertmanager/alertmanager-config.yaml") . | sha256sum }}
+        {{- end }}
       namespace: {{ .Release.Namespace | quote }}
     spec:
       serviceAccountName: {{ template "mimir.serviceAccountName" . }}
@@ -64,6 +67,10 @@ spec:
             - name: storage
               mountPath: "/data"
               subPath: {{ .Values.alertmanager.persistence.subPath }}
+            {{- if .Values.alertmanager.fallbackConfig }}
+            - name: alertmanager-fallback-config
+              mountPath: /configs/
+            {{- end }}
             - name: tmp
               mountPath: /tmp
             - name: active-queries
@@ -132,5 +139,10 @@ spec:
           emptyDir: {}
         - name: active-queries
           emptyDir: {}
+        {{- if .Values.alertmanager.fallbackConfig }}
+        - name: alertmanager-fallback-config
+          configMap:
+            name: {{ include "mimir.resourceName" (dict "ctx" . "component" "alertmanager") }}-fallback-config
+        {{- end }}
 {{- end -}}
 {{- end -}}
diff --git a/...ations/helm/charts/mimir-distributed/templates/alertmanager/alertmanager-statefulset.yaml b/...ations/helm/charts/mimir-distributed/templates/alertmanager/alertmanager-statefulset.yaml
@@ -54,6 +54,9 @@ spec:
         {{- include "mimir.podLabels" $args | nindent 8 }}
       annotations:
         {{- include "mimir.podAnnotations" $args | nindent 8 }}
+        {{- if .Values.alertmanager.fallbackConfig }}
+        checksum/alertmanager-fallback-config: {{ include (print .Template.BasePath "/alertmanager/alertmanager-config.yaml") . | sha256sum }}
+        {{- end }}
       namespace: {{ .Release.Namespace | quote }}
     spec:
       serviceAccountName: {{ template "mimir.serviceAccountName" . }}
@@ -96,11 +99,16 @@ spec:
         {{- end }}
         - name: tmp
           emptyDir: {}
+        - name: active-queries
+          emptyDir: {}
+        {{- if .Values.alertmanager.fallbackConfig }}
+        - name: alertmanager-fallback-config
+          configMap:
+            name: {{ include "mimir.resourceName" (dict "ctx" . "component" "alertmanager") }}-fallback-config
+        {{- end }}
         {{- if .Values.alertmanager.extraVolumes }}
         {{ toYaml .Values.alertmanager.extraVolumes | nindent 8 }}
         {{- end }}
-        - name: active-queries
-          emptyDir: {}
       containers:
         {{- if .Values.alertmanager.extraContainers }}
         {{ toYaml .Values.alertmanager.extraContainers | nindent 8 }}
@@ -140,6 +148,10 @@ spec:
               subPath: {{ .Values.alertmanager.persistentVolume.subPath }}
               {{- else }}
               {{- end }}
+            {{- if .Values.alertmanager.fallbackConfig }}
+            - name: alertmanager-fallback-config
+              mountPath: /configs/
+            {{- end }}
             - name: tmp
               mountPath: /tmp
             - name: active-queries
@@ -181,4 +193,4 @@ spec:
 {{ end }}
 {{ end }}
 {{ end }}
-{{ end }}
+{{ end }}
diff --git a/operations/helm/charts/mimir-distributed/values.yaml b/operations/helm/charts/mimir-distributed/values.yaml
@@ -116,6 +116,9 @@ mimir:
       sharding_ring:
         zone_awareness_enabled: true
       {{- end }}
+      {{- if .Values.alertmanager.fallbackConfig }}
+      fallback_config_file: /configs/alertmanager_fallback_config.yaml
+      {{- end }}
 
     {{- if .Values.minio.enabled }}
     alertmanager_storage:
@@ -384,6 +387,14 @@ alertmanager:
       cpu: 10m
       memory: 32Mi
 
+  # -- Fallback config for alertmanager.
+  # When a tenant doesn't have an Alertmanager configuration, the Grafana Mimir Alertmanager uses the fallback configuration.
+  fallbackConfig: |
+    receivers:
+        - name: default-receiver
+    route:
+        receiver: default-receiver
+
   extraArgs: {}
 
   # Pod Labels

diff --git a/...rprise-values-generated/mimir-distributed/templates/alertmanager/alertmanager-config.yaml b/...rprise-values-generated/mimir-distributed/templates/alertmanager/alertmanager-config.yaml
@@ -0,0 +1,21 @@
+---
+# Source: mimir-distributed/templates/alertmanager/alertmanager-config.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: gateway-enterprise-values-mimir-alertmanager-fallback-config
+  labels:
+    app.kubernetes.io/name: mimir
+    app.kubernetes.io/instance: gateway-enterprise-values
+    app.kubernetes.io/component: alertmanager
+    app.kubernetes.io/part-of: memberlist
+    app.kubernetes.io/managed-by: Helm
+  annotations:
+    {} 
+  namespace: "citestns"
+data:
+  alertmanager_fallback_config.yaml: |
+    receivers:
+        - name: default-receiver
+    route:
+        receiver: default-receiver
diff --git a/...e-values-generated/mimir-distributed/templates/alertmanager/alertmanager-statefulset.yaml b/...e-values-generated/mimir-distributed/templates/alertmanager/alertmanager-statefulset.yaml
@@ -41,6 +41,7 @@ spec:
         app.kubernetes.io/component: alertmanager
         app.kubernetes.io/part-of: memberlist
       annotations:
+        checksum/alertmanager-fallback-config: 5323106a127a419852c256c1d33f7917c7006d4b3d22e97db42c61e7c0e239c6
       namespace: "citestns"
     spec:
       serviceAccountName: gateway-enterprise-values-mimir
@@ -86,6 +87,9 @@ spec:
           emptyDir: {}
         - name: active-queries
           emptyDir: {}
+        - name: alertmanager-fallback-config
+          configMap:
+            name: gateway-enterprise-values-mimir-alertmanager-fallback-config
       containers:
         - name: alertmanager
           imagePullPolicy: IfNotPresent
@@ -102,6 +106,8 @@ spec:
               mountPath: /var/mimir
             - name: storage
               mountPath: "/data"
+            - name: alertmanager-fallback-config
+              mountPath: /configs/
             - name: tmp
               mountPath: /tmp
             - name: active-queries

diff --git a/...m/tests/gateway-enterprise-values-generated/mimir-distributed/templates/mimir-config.yaml b/...m/tests/gateway-enterprise-values-generated/mimir-distributed/templates/mimir-config.yaml
@@ -33,6 +33,7 @@ data:
       data_dir: /data
       enable_api: true
       external_url: /alertmanager
+      fallback_config_file: /configs/alertmanager_fallback_config.yaml
     alertmanager_storage:
       backend: s3
       s3:

diff --git a/...-nginx-values-generated/mimir-distributed/templates/alertmanager/alertmanager-config.yaml b/...-nginx-values-generated/mimir-distributed/templates/alertmanager/alertmanager-config.yaml
@@ -0,0 +1,21 @@
+---
+# Source: mimir-distributed/templates/alertmanager/alertmanager-config.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: gateway-nginx-values-mimir-alertmanager-fallback-config
+  labels:
+    app.kubernetes.io/name: mimir
+    app.kubernetes.io/instance: gateway-nginx-values
+    app.kubernetes.io/component: alertmanager
+    app.kubernetes.io/part-of: memberlist
+    app.kubernetes.io/managed-by: Helm
+  annotations:
+    {} 
+  namespace: "citestns"
+data:
+  alertmanager_fallback_config.yaml: |
+    receivers:
+        - name: default-receiver
+    route:
+        receiver: default-receiver
diff --git a/...x-values-generated/mimir-distributed/templates/alertmanager/alertmanager-statefulset.yaml b/...x-values-generated/mimir-distributed/templates/alertmanager/alertmanager-statefulset.yaml
@@ -41,6 +41,7 @@ spec:
         app.kubernetes.io/component: alertmanager
         app.kubernetes.io/part-of: memberlist
       annotations:
+        checksum/alertmanager-fallback-config: 003b0404961e045dea31576565c2a4aeedf068f825fa30cd888a6f6fdde77154
       namespace: "citestns"
     spec:
       serviceAccountName: gateway-nginx-values-mimir
@@ -83,6 +84,9 @@ spec:
           emptyDir: {}
         - name: active-queries
           emptyDir: {}
+        - name: alertmanager-fallback-config
+          configMap:
+            name: gateway-nginx-values-mimir-alertmanager-fallback-config
       containers:
         - name: alertmanager
           imagePullPolicy: IfNotPresent
@@ -97,6 +101,8 @@ spec:
               mountPath: /var/mimir
             - name: storage
               mountPath: "/data"
+            - name: alertmanager-fallback-config
+              mountPath: /configs/
             - name: tmp
               mountPath: /tmp
             - name: active-queries

diff --git a/...s/helm/tests/gateway-nginx-values-generated/mimir-distributed/templates/mimir-config.yaml b/...s/helm/tests/gateway-nginx-values-generated/mimir-distributed/templates/mimir-config.yaml
@@ -18,6 +18,7 @@ data:
       data_dir: /data
       enable_api: true
       external_url: /alertmanager
+      fallback_config_file: /configs/alertmanager_fallback_config.yaml
     alertmanager_storage:
       backend: s3
       s3:

diff --git a/...nabled-values-generated/mimir-distributed/templates/alertmanager/alertmanager-config.yaml b/...nabled-values-generated/mimir-distributed/templates/alertmanager/alertmanager-config.yaml
@@ -0,0 +1,21 @@
+---
+# Source: mimir-distributed/templates/alertmanager/alertmanager-config.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: graphite-enabled-values-mimir-alertmanager-fallback-config
+  labels:
+    app.kubernetes.io/name: mimir
+    app.kubernetes.io/instance: graphite-enabled-values
+    app.kubernetes.io/component: alertmanager
+    app.kubernetes.io/part-of: memberlist
+    app.kubernetes.io/managed-by: Helm
+  annotations:
+    {} 
+  namespace: "citestns"
+data:
+  alertmanager_fallback_config.yaml: |
+    receivers:
+        - name: default-receiver
+    route:
+        receiver: default-receiver
diff --git a/...d-values-generated/mimir-distributed/templates/alertmanager/alertmanager-statefulset.yaml b/...d-values-generated/mimir-distributed/templates/alertmanager/alertmanager-statefulset.yaml
@@ -41,6 +41,7 @@ spec:
         app.kubernetes.io/component: alertmanager
         app.kubernetes.io/part-of: memberlist
       annotations:
+        checksum/alertmanager-fallback-config: bda77839b7c05d228d55793cf7573c093548fc9f54274e407a32d643d77753b9
       namespace: "citestns"
     spec:
       serviceAccountName: graphite-enabled-values-mimir
@@ -86,6 +87,9 @@ spec:
           emptyDir: {}
         - name: active-queries
           emptyDir: {}
+        - name: alertmanager-fallback-config
+          configMap:
+            name: graphite-enabled-values-mimir-alertmanager-fallback-config
       containers:
         - name: alertmanager
           imagePullPolicy: IfNotPresent
@@ -102,6 +106,8 @@ spec:
               mountPath: /var/mimir
             - name: storage
               mountPath: "/data"
+            - name: alertmanager-fallback-config
+              mountPath: /configs/
             - name: tmp
               mountPath: /tmp
             - name: active-queries

diff --git a/...elm/tests/graphite-enabled-values-generated/mimir-distributed/templates/mimir-config.yaml b/...elm/tests/graphite-enabled-values-generated/mimir-distributed/templates/mimir-config.yaml
@@ -33,6 +33,7 @@ data:
       data_dir: /data
       enable_api: true
       external_url: /alertmanager
+      fallback_config_file: /configs/alertmanager_fallback_config.yaml
     alertmanager_storage:
       backend: s3
       s3:

diff --git a/.../large-values-generated/mimir-distributed/templates/alertmanager/alertmanager-config.yaml b/.../large-values-generated/mimir-distributed/templates/alertmanager/alertmanager-config.yaml
@@ -0,0 +1,21 @@
+---
+# Source: mimir-distributed/templates/alertmanager/alertmanager-config.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: large-values-mimir-alertmanager-fallback-config
+  labels:
+    app.kubernetes.io/name: mimir
+    app.kubernetes.io/instance: large-values
+    app.kubernetes.io/component: alertmanager
+    app.kubernetes.io/part-of: memberlist
+    app.kubernetes.io/managed-by: Helm
+  annotations:
+    {} 
+  namespace: "citestns"
+data:
+  alertmanager_fallback_config.yaml: |
+    receivers:
+        - name: default-receiver
+    route:
+        receiver: default-receiver
diff --git a/...e-values-generated/mimir-distributed/templates/alertmanager/alertmanager-statefulset.yaml b/...e-values-generated/mimir-distributed/templates/alertmanager/alertmanager-statefulset.yaml
@@ -41,6 +41,7 @@ spec:
         app.kubernetes.io/component: alertmanager
         app.kubernetes.io/part-of: memberlist
       annotations:
+        checksum/alertmanager-fallback-config: 731f8fd3804831553b6a885137af73c8213943869f297618520e050fc315f34d
       namespace: "citestns"
     spec:
       serviceAccountName: large-values-mimir
@@ -83,6 +84,9 @@ spec:
           emptyDir: {}
         - name: active-queries
           emptyDir: {}
+        - name: alertmanager-fallback-config
+          configMap:
+            name: large-values-mimir-alertmanager-fallback-config
       containers:
         - name: alertmanager
           imagePullPolicy: IfNotPresent
@@ -97,6 +101,8 @@ spec:
               mountPath: /var/mimir
             - name: storage
               mountPath: "/data"
+            - name: alertmanager-fallback-config
+              mountPath: /configs/
             - name: tmp
               mountPath: /tmp
             - name: active-queries

diff --git a/operations/helm/tests/large-values-generated/mimir-distributed/templates/mimir-config.yaml b/operations/helm/tests/large-values-generated/mimir-distributed/templates/mimir-config.yaml
@@ -18,6 +18,7 @@ data:
       data_dir: /data
       enable_api: true
       external_url: /alertmanager
+      fallback_config_file: /configs/alertmanager_fallback_config.yaml
     blocks_storage:
       backend: s3
       bucket_store:

diff --git a/...toring-values-generated/mimir-distributed/templates/alertmanager/alertmanager-config.yaml b/...toring-values-generated/mimir-distributed/templates/alertmanager/alertmanager-config.yaml
@@ -0,0 +1,21 @@
+---
+# Source: mimir-distributed/templates/alertmanager/alertmanager-config.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: metamonitoring-values-mimir-alertmanager-fallback-config
+  labels:
+    app.kubernetes.io/name: mimir
+    app.kubernetes.io/instance: metamonitoring-values
+    app.kubernetes.io/component: alertmanager
+    app.kubernetes.io/part-of: memberlist
+    app.kubernetes.io/managed-by: Helm
+  annotations:
+    {} 
+  namespace: "citestns"
+data:
+  alertmanager_fallback_config.yaml: |
+    receivers:
+        - name: default-receiver
+    route:
+        receiver: default-receiver