Merge pull request prometheus-operator#3333 from paulfantom/mixin

Makefile

@@ -135,7 +135,7 @@ tidy:
 	cd scripts && go mod tidy -v -modfile=go.mod
 
 .PHONY: generate
-generate: $(DEEPCOPY_TARGET) generate-crds bundle.yaml $(shell find Documentation -type f)
+generate: $(DEEPCOPY_TARGET) generate-crds bundle.yaml example/mixin/alerts.yaml $(shell find Documentation -type f)
 
 .PHONY: generate-crds
 generate-crds: $(CONTROLLER_GEN_BINARY) $(GOJSONTOYAML_BINARY) $(TYPES_V1_TARGET)
@@ -192,6 +192,10 @@ scripts/generate/vendor: $(JB_BINARY) $(shell find jsonnet/prometheus-operator -
 example/non-rbac/prometheus-operator.yaml: scripts/generate/vendor scripts/generate/prometheus-operator-non-rbac.jsonnet $(shell find jsonnet -type f)
 	scripts/generate/build-non-rbac-prometheus-operator.sh
 
+example/mixin/alerts.yaml: $(JSONNET_BINARY) $(GOJSONTOYAML_BINARY)
+	-mkdir -p example/alerts
+	$(JSONNET_BINARY) jsonnet/mixin/alerts.jsonnet | $(GOJSONTOYAML_BINARY) > $@
+
 RBAC_MANIFESTS = example/rbac/prometheus-operator/prometheus-operator-cluster-role.yaml example/rbac/prometheus-operator/prometheus-operator-cluster-role-binding.yaml example/rbac/prometheus-operator/prometheus-operator-service-account.yaml example/rbac/prometheus-operator/prometheus-operator-deployment.yaml
 $(RBAC_MANIFESTS): scripts/generate/vendor scripts/generate/prometheus-operator-rbac.jsonnet $(shell find jsonnet -type f)
 	scripts/generate/build-rbac-prometheus-operator.sh

example/mixin/alerts.yaml

@@ -0,0 +1,33 @@
+groups:
+- name: prometheus-operator
+  rules:
+  - alert: PrometheusOperatorWatchErrors
+    annotations:
+      description: Errors while performing watch operations in controller {{$labels.controller}}
+        in {{$labels.namespace}} namespace.
+      summary: Errors while performing watch operations in controller.
+    expr: |
+      (sum by (controller,namespace) (rate(prometheus_operator_watch_operations_failed_total{job="prometheus-operator"}[1h])) / sum by (controller,namespace) (rate(prometheus_operator_watch_operations_total{job="prometheus-operator"}[1h]))) > 0.1
+    for: 15m
+    labels:
+      severity: warning
+  - alert: PrometheusOperatorReconcileErrors
+    annotations:
+      description: Errors while reconciling {{ $labels.controller }} in {{ $labels.namespace
+        }} Namespace.
+      summary: Errors while reconciling controller.
+    expr: |
+      rate(prometheus_operator_reconcile_errors_total{job="prometheus-operator"}[5m]) > 0.1
+    for: 10m
+    labels:
+      severity: warning
+  - alert: PrometheusOperatorNodeLookupErrors
+    annotations:
+      description: Errors while reconciling Prometheus in {{ $labels.namespace }}
+        Namespace.
+      summary: Errors while reconciling Prometheus.
+    expr: |
+      rate(prometheus_operator_node_address_lookup_errors_total{job="prometheus-operator"}[5m]) > 0.1
+    for: 10m
+    labels:
+      severity: warning

jsonnet/mixin/alerts.jsonnet

@@ -0,0 +1,3 @@
+(
+  import 'mixin.libsonnet'
+).prometheusAlerts

jsonnet/mixin/alerts/alerts.libsonnet

@@ -0,0 +1,53 @@
+{
+  prometheusAlerts+:: {
+    groups+: [
+      {
+        name: 'prometheus-operator',
+        rules: [
+          {
+            alert: 'PrometheusOperatorWatchErrors',
+            expr: |||
+              (sum by (controller,namespace) (rate(prometheus_operator_watch_operations_failed_total{%(prometheusOperatorSelector)s}[1h])) / sum by (controller,namespace) (rate(prometheus_operator_watch_operations_total{%(prometheusOperatorSelector)s}[1h]))) > 0.1
+            ||| % $._config,
+            labels: {
+              severity: 'warning',
+            },
+            annotations: {
+              description: 'Errors while performing watch operations in controller {{$labels.controller}} in {{$labels.namespace}} namespace.',
+              summary: 'Errors while performing watch operations in controller.',
+            },
+            'for': '15m',
+          },
+          {
+            alert: 'PrometheusOperatorReconcileErrors',
+            expr: |||
+              rate(prometheus_operator_reconcile_errors_total{%(prometheusOperatorSelector)s}[5m]) > 0.1
+            ||| % $._config,
+            labels: {
+              severity: 'warning',
+            },
+            annotations: {
+              description: 'Errors while reconciling {{ $labels.controller }} in {{ $labels.namespace }} Namespace.',
+              summary: 'Errors while reconciling controller.',
+            },
+            'for': '10m',
+          },
+          {
+            alert: 'PrometheusOperatorNodeLookupErrors',
+            expr: |||
+              rate(prometheus_operator_node_address_lookup_errors_total{%(prometheusOperatorSelector)s}[5m]) > 0.1
+            ||| % $._config,
+            labels: {
+              severity: 'warning',
+            },
+            annotations: {
+              description: 'Errors while reconciling Prometheus in {{ $labels.namespace }} Namespace.',
+              summary: 'Errors while reconciling Prometheus.',
+            },
+            'for': '10m',
+          },
+        ],
+      },
+    ],
+  },
+}

jsonnet/mixin/config.libsonnet

@@ -0,0 +1,5 @@
+{
+  _config+:: {
+    prometheusOperatorSelector: 'job="prometheus-operator"',
+  },
+}

jsonnet/mixin/mixin.libsonnet

@@ -0,0 +1,2 @@
+(import 'config.libsonnet') +
+(import 'alerts/alerts.libsonnet')