Add support of vmalertmanager.spec.templates and autoreload dirs for …

…templates and configmaps (VictoriaMetrics#592) (VictoriaMetrics#596)

* Add support of vmalertmanager.spec.templates and autoreload dirs for templates and configmaps in alertmanager pods (VictoriaMetrics#592)

* Fix dirs watching list for config-reloader container

api/v1beta1/additional.go

@@ -364,3 +364,11 @@ func (ds *DiscoverySelector) AsListOptions() (*client.ListOptions, error) {
 		LabelSelector: s,
 	}, nil
 }
+
+// ConfigMapKeyReference refers to a key in a ConfigMap.
+type ConfigMapKeyReference struct {
+	// The ConfigMap to refer to.
+	v1.LocalObjectReference `json:",inline"`
+	// The ConfigMap key to refer to.
+	Key string `json:"key"`
+}

api/v1beta1/vmalertmanager_types.go

@@ -68,6 +68,11 @@ type VMAlertmanagerSpec struct {
 	// The ConfigMaps are mounted into /etc/vm/configs/<configmap-name>.
 	// +optional
 	ConfigMaps []string `json:"configMaps,omitempty"`
+	// Templates is a list of ConfigMap key references for ConfigMaps in the same namespace as the VMAlertmanager
+	// object, which shall be mounted into the VMAlertmanager Pods.
+	// The Templates are mounted into /etc/vm/templates/<configmap-name>/<configmap-key>.
+	// +optional
+	Templates []ConfigMapKeyReference `json:"templates,omitempty"`
 
 	// ConfigRawYaml - raw configuration for alertmanager,
 	// it helps it to start without secret.

api/victoriametrics/v1beta1/additional.go

@@ -364,3 +364,11 @@ func (ds *DiscoverySelector) AsListOptions() (*client.ListOptions, error) {
 		LabelSelector: s,
 	}, nil
 }
+
+// ConfigMapKeyReference refers to a key in a ConfigMap.
+type ConfigMapKeyReference struct {
+	// The ConfigMap to refer to.
+	v1.LocalObjectReference `json:",inline"`
+	// The ConfigMap key to refer to.
+	Key string `json:"key"`
+}

api/victoriametrics/v1beta1/vmalertmanager_types.go

@@ -68,6 +68,11 @@ type VMAlertmanagerSpec struct {
 	// The ConfigMaps are mounted into /etc/vm/configs/<configmap-name>.
 	// +optional
 	ConfigMaps []string `json:"configMaps,omitempty"`
+	// Templates is a list of ConfigMap key references for ConfigMaps in the same namespace as the VMAlertmanager
+	// object, which shall be mounted into the VMAlertmanager Pods.
+	// The Templates are mounted into /etc/vm/templates/<configmap-name>/<configmap-key>.
+	// +optional
+	Templates []ConfigMapKeyReference `json:"templates,omitempty"`
 
 	// ConfigRawYaml - raw configuration for alertmanager,
 	// it helps it to start without secret.

config/crd/bases/operator.victoriametrics.com_vmalertmanagers.yaml

@@ -1207,6 +1207,26 @@ spec:
                         type: object
                     type: object
                 type: object
+              templates:
+                description: Templates is a list of ConfigMap key references for ConfigMaps
+                  in the same namespace as the VMAlertmanager object, which shall
+                  be mounted into the VMAlertmanager Pods. The Templates are mounted
+                  into /etc/vm/templates/<configmap-name>/<configmap-key>.
+                items:
+                  description: ConfigMapKeyReference refers to a key in a ConfigMap.
+                  properties:
+                    key:
+                      description: The ConfigMap key to refer to.
+                      type: string
+                    name:
+                      description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+                        TODO: Add other useful fields. apiVersion, kind, uid?'
+                      type: string
+                  required:
+                  - key
+                  type: object
+                  x-kubernetes-map-type: atomic
+                type: array
               terminationGracePeriodSeconds:
                 description: TerminationGracePeriodSeconds period for container graceful
                   termination

controllers/factory/alertmanager.go

@@ -354,6 +354,14 @@ func makeStatefulSetSpec(cr *victoriametricsv1beta1.VMAlertmanager, c *config.Ba
 		})
 	}
 
+	crVolumeMounts := []v1.VolumeMount{
+		{
+			Name:      "config-volume",
+			MountPath: alertmanagerConfDir,
+			ReadOnly:  true,
+		},
+	}
+
 	for _, c := range cr.Spec.ConfigMaps {
 		volumes = append(volumes, v1.Volume{
 			Name: k8stools.SanitizeVolumeName("configmap-" + c),
@@ -365,11 +373,37 @@ func makeStatefulSetSpec(cr *victoriametricsv1beta1.VMAlertmanager, c *config.Ba
 				},
 			},
 		})
-		amVolumeMounts = append(amVolumeMounts, v1.VolumeMount{
+		cmVolumeMount := v1.VolumeMount{
 			Name:      k8stools.SanitizeVolumeName("configmap-" + c),
 			ReadOnly:  true,
 			MountPath: path.Join(ConfigMapsDir, c),
+		}
+		amVolumeMounts = append(amVolumeMounts, cmVolumeMount)
+		crVolumeMounts = append(crVolumeMounts, cmVolumeMount)
+	}
+
+	volumeByName := make(map[string]struct{})
+	for _, t := range cr.Spec.Templates {
+		// Deduplicate configmaps by name
+		if _, ok := volumeByName[t.Name]; ok {
+			continue
+		}
+		volumeByName[t.Name] = struct{}{}
+		volumes = append(volumes, v1.Volume{
+			Name: k8stools.SanitizeVolumeName("templates-" + t.Name),
+			VolumeSource: v1.VolumeSource{
+				ConfigMap: &v1.ConfigMapVolumeSource{
+					LocalObjectReference: t.LocalObjectReference,
+				},
+			},
 		})
+		tmplVolumeMount := v1.VolumeMount{
+			Name:      k8stools.SanitizeVolumeName("templates-" + t.Name),
+			MountPath: path.Join(TemplatesDir, t.Name),
+			ReadOnly:  true,
+		}
+		amVolumeMounts = append(amVolumeMounts, tmplVolumeMount)
+		crVolumeMounts = append(crVolumeMounts, tmplVolumeMount)
 	}
 
 	amVolumeMounts = append(amVolumeMounts, cr.Spec.VolumeMounts...)
@@ -415,27 +449,22 @@ func makeStatefulSetSpec(cr *victoriametricsv1beta1.VMAlertmanager, c *config.Ba
 		TerminationMessagePolicy: v1.TerminationMessageFallbackToLogsOnError,
 	}
 	vmaContainer = buildProbe(vmaContainer, cr) // cr.Spec.EmbeddedProbes, healthPath, cr.Spec.PortName, true)
-	defaultContainers := []v1.Container{
-		vmaContainer,
-		{
-			Name:  "config-reloader",
-			Image: fmt.Sprintf("%s", formatContainerImage(c.ContainerRegistry, c.VMAlertManager.ConfigReloaderImage)),
-			Args: []string{
-				fmt.Sprintf("-webhook-url=%s", localReloadURL),
-				fmt.Sprintf("-volume-dir=%s", alertmanagerConfDir),
-			},
-			VolumeMounts: []v1.VolumeMount{
-				{
-					Name:      "config-volume",
-					ReadOnly:  true,
-					MountPath: alertmanagerConfDir,
-				},
-			},
-			Resources:                resources,
-			TerminationMessagePolicy: v1.TerminationMessageFallbackToLogsOnError,
+	configReloaderContainer := v1.Container{
+		Name:  "config-reloader",
+		Image: fmt.Sprintf("%s", formatContainerImage(c.ContainerRegistry, c.VMAlertManager.ConfigReloaderImage)),
+		Args: []string{
+			fmt.Sprintf("-webhook-url=%s", localReloadURL),
 		},
+		VolumeMounts:             crVolumeMounts,
+		Resources:                resources,
+		TerminationMessagePolicy: v1.TerminationMessageFallbackToLogsOnError,
+	}
+	// Add watching for every volume mount in config-reloader
+	for _, vm := range crVolumeMounts {
+		configReloaderContainer.Args = append(configReloaderContainer.Args, fmt.Sprintf("-volume-dir=%s", vm.MountPath))
 	}
 
+	defaultContainers := []v1.Container{vmaContainer, configReloaderContainer}
 	containers, err := k8stools.MergePatchContainers(defaultContainers, cr.Spec.Containers)
 	if err != nil {
 		return nil, fmt.Errorf("failed to merge containers spec: %w", err)
@@ -531,6 +560,20 @@ func createDefaultAMConfig(ctx context.Context, cr *victoriametricsv1beta1.VMAle
 	if len(alertmananagerConfig) == 0 {
 		alertmananagerConfig = []byte(defaultAMConfig)
 	}
+
+	// add templates from CR to alermanager config
+	if len(cr.Spec.Templates) > 0 {
+		templatePaths := make([]string, 0, len(cr.Spec.Templates))
+		for _, template := range cr.Spec.Templates {
+			templatePaths = append(templatePaths, path.Join(TemplatesDir, template.Name, template.Key))
+		}
+		mergedCfg, err := alertmanager.AddConfigTemplates(alertmananagerConfig, templatePaths)
+		if err != nil {
+			return fmt.Errorf("cannot build alertmanager config with templates, err: %w", err)
+		}
+		alertmananagerConfig = mergedCfg
+	}
+
 	newAMSecretConfig := &v1.Secret{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:            cr.ConfigSecretName(),

controllers/factory/alertmanager/config.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"path"
 	"sort"
+	"strings"
 
 	operatorv1beta1 "github.com/VictoriaMetrics/operator/api/v1beta1"
 	"gopkg.in/yaml.v2"
@@ -94,6 +95,31 @@ OUTER:
 	return &ParsedConfig{Data: result, BadObjectsCount: badObjectsCount, ParseErrors: parseErrors}, nil
 }
 
+func AddConfigTemplates(baseCfg []byte, templates []string) ([]byte, error) {
+	if len(templates) == 0 {
+		return baseCfg, nil
+	}
+	var baseYAMlCfg alertmanagerConfig
+	if err := yaml.Unmarshal(baseCfg, &baseYAMlCfg); err != nil {
+		return nil, fmt.Errorf("cannot parse base cfg :%w", err)
+	}
+	templatesSet := make(map[string]struct{})
+	for _, v := range baseYAMlCfg.Templates {
+		templatesSet[v] = struct{}{}
+	}
+	for _, v := range templates {
+		if len(strings.TrimSpace(v)) == 0 {
+			continue
+		}
+		if _, ok := templatesSet[v]; ok {
+			continue
+		}
+		baseYAMlCfg.Templates = append(baseYAMlCfg.Templates, v)
+		templatesSet[v] = struct{}{}
+	}
+	return yaml.Marshal(baseYAMlCfg)
+}
+
 func buildGlobalTimeIntervals(cr *operatorv1beta1.VMAlertmanagerConfig) []yaml.MapSlice {
 	var r []yaml.MapSlice
 	tis := cr.Spec.TimeIntervals