[refactor] support alarm threshold bind monitors (#2933)

Signed-off-by: tomsun28 <[email protected]>

hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/calculate/RealTimeAlertCalculator.java

@@ -23,6 +23,9 @@
 import java.util.Map;
 import java.util.Objects;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.jexl3.JexlException;
 import org.apache.commons.jexl3.JexlExpression;
@@ -37,10 +40,8 @@
 import org.apache.hertzbeat.common.entity.alerter.SingleAlert;
 import org.apache.hertzbeat.common.entity.message.CollectRep;
 import org.apache.hertzbeat.common.queue.CommonDataQueue;
-import org.apache.hertzbeat.common.support.event.MonitorDeletedEvent;
 import org.apache.hertzbeat.common.util.CommonUtil;
 import org.apache.hertzbeat.common.util.JexlExpressionRunner;
-import org.springframework.context.event.EventListener;
 import org.springframework.stereotype.Component;
 import org.springframework.util.CollectionUtils;
 
@@ -50,10 +51,22 @@
 @Component
 @Slf4j
 public class RealTimeAlertCalculator {
-
-    private static final String SYSTEM_VALUE_ROW_COUNT = "system_value_row_count";
 
     private static final int CALCULATE_THREADS = 3;
+
+    private static final String KEY_INSTANCE = "__instance__";
+    private static final String KEY_APP = "__app__";
+    private static final String KEY_METRICS = "__metrics__";
+    private static final String KEY_PRIORITY = "__priority__";
+    private static final String KEY_CODE = "__code__";
+    private static final String KEY_AVAILABLE  = "__available__";
+    private static final String UP = "up";
+    private static final String DOWN = "down";
+    private static final String KEY_ROW = "__row__";
+
+    private static final Pattern APP_PATTERN = Pattern.compile("equals\\(__app__,\"([^\"]+)\"\\)");
+    private static final Pattern INSTANCE_PATTERN = Pattern.compile("equals\\(__instance__,\"(\\d+)\"\\)");
+    private static final Pattern METRICS_PATTERN = Pattern.compile("equals\\(__metrics__,\"([^\"]+)\"\\)");
 
     /**
      * The alarm in the process is triggered
@@ -109,37 +122,50 @@ private void startCalculate() {
 
     private void calculate(CollectRep.MetricsData metricsData) {
         long currentTimeMilli = System.currentTimeMillis();
-        long instance = metricsData.getId();
+        String instance = String.valueOf(metricsData.getId());
         String app = metricsData.getApp();
         String metrics = metricsData.getMetrics();
-        Integer priority = metricsData.getPriority();
+        int priority = metricsData.getPriority();
         String code = metricsData.getCode().name();
-        // todo get all alert define cache
         List<AlertDefine> thresholds = this.alertDefineService.getRealTimeAlertDefines();
-        // todo filter some thresholds by app metrics instance
+        // Filter thresholds by app, metrics and instance
+        thresholds = filterThresholdsByAppAndMetrics(thresholds, app, metrics, instance);
+        if (thresholds.isEmpty()) {
+            return;
+        }
         Map<String, Object> commonContext = new HashMap<>(8);
-        commonContext.put("instance", instance);
-        commonContext.put("app", app);
-        commonContext.put("priority", priority);
-        commonContext.put("code", code);
-        commonContext.put("metrics", metrics);
-
+        commonContext.put(KEY_INSTANCE, instance);
+        commonContext.put(KEY_APP, app);
+        commonContext.put(KEY_PRIORITY, priority);
+        commonContext.put(KEY_CODE, code);
+        commonContext.put(KEY_METRICS, metrics);
+        if (priority == 0) {
+            commonContext.put(KEY_AVAILABLE, metricsData.getCode() == CollectRep.Code.SUCCESS ? UP : DOWN);
+        }
         List<CollectRep.Field> fields = metricsData.getFields();
         Map<String, Object> fieldValueMap = new HashMap<>(8);
         int valueRowCount = metricsData.getValuesCount();
         for (AlertDefine define : thresholds) {
+            if (define.getLabels() == null) {
+                define.setLabels(new HashMap<>(8));
+            }
+            if (define.getAnnotations() == null) {
+                define.setAnnotations(new HashMap<>(8));
+            }
+            fieldValueMap.clear();
+            fieldValueMap.putAll(commonContext);
             final String expr = define.getExpr();
             if (StringUtils.isBlank(expr)) {
                 continue;
             }
-            fieldValueMap.putAll(commonContext);
             {
                 // trigger the expr before the metrics data, due the available up down or others
                 try {
                     boolean match = execAlertExpression(fieldValueMap, expr);
                     try {
                         Map<String, String> fingerPrints = new HashMap<>(8);
-                        fingerPrints.put(CommonConstants.LABEL_INSTANCE, String.valueOf(instance));
+                        fingerPrints.put(CommonConstants.LABEL_INSTANCE, instance);
+                        // here use the alert name as finger, not care the alert name may be changed
                         fingerPrints.put(CommonConstants.LABEL_ALERT_NAME, define.getName());
                         fingerPrints.putAll(define.getLabels());
                         if (match) {
@@ -161,10 +187,10 @@ private void calculate(CollectRep.MetricsData metricsData) {
                     continue;
                 }
                 fieldValueMap.clear();
-                fieldValueMap.put(SYSTEM_VALUE_ROW_COUNT, valueRowCount);
+                fieldValueMap.put(KEY_ROW, valueRowCount);
                 fieldValueMap.putAll(commonContext);
                 fingerPrints.clear();
-                fingerPrints.put(CommonConstants.LABEL_INSTANCE, String.valueOf(instance));
+                fingerPrints.put(CommonConstants.LABEL_INSTANCE, instance);
                 fingerPrints.put(CommonConstants.LABEL_ALERT_NAME, define.getName());
                 fingerPrints.putAll(define.getLabels());
                 for (int index = 0; index < valueRow.getColumnsList().size(); index++) {
@@ -212,6 +238,54 @@ private void calculate(CollectRep.MetricsData metricsData) {
         }
     }
 
+    /**
+     * Filter alert definitions by app, metrics and instance
+     * @param thresholds Alert definitions to filter
+     * @param app Current app name
+     * @param metrics Current metrics name
+     * @param instance Current instance id
+     * @return Filtered alert definitions
+     */
+    private List<AlertDefine> filterThresholdsByAppAndMetrics(List<AlertDefine> thresholds, String app, String metrics, String instance) {
+        return thresholds.stream()
+                .filter(define -> {
+                    if (StringUtils.isBlank(define.getExpr())) {
+                        return false;
+                    }
+                    String expr = define.getExpr();
+
+                    // Extract and check app - required
+                    Matcher appMatcher = APP_PATTERN.matcher(expr);
+                    if (!appMatcher.find() || !app.equals(appMatcher.group(1))) {
+                        return false;
+                    }
+
+                    // Extract and check metrics - optional
+                    Matcher metricsMatcher = METRICS_PATTERN.matcher(expr);
+                    if (metricsMatcher.find() && !metrics.equals(metricsMatcher.group(1))) {
+                        return false;
+                    }
+
+                    // Extract and check instance - optional with multiple values
+                    Matcher instanceMatcher = INSTANCE_PATTERN.matcher(expr);
+                    // If no instance specified in expr, accept all instances
+                    if (!instanceMatcher.find()) {
+                        return true;
+                    }
+
+                    // Reset matcher to check all instances
+                    instanceMatcher.reset();
+                    // If instances specified, current instance must match one of them
+                    while (instanceMatcher.find()) {
+                        if (Objects.equals(instance, instanceMatcher.group(1))) {
+                            return true;
+                        }
+                    }
+                    return false;
+                })
+                .collect(Collectors.toList());
+    }
+
     private void handleRecoveredAlert(Map<String, String> fingerprints) {
         String fingerprint = calculateFingerprint(fingerprints);
         SingleAlert firingAlert = firingAlertMap.remove(fingerprint);
@@ -275,7 +349,7 @@ private boolean execAlertExpression(Map<String, Object> fieldValueMap, String ex
         try {
             expression = JexlExpressionRunner.compile(expr);
         } catch (JexlException jexlException) {
-            log.error("Alarm Rule: {} Compile Error: {}.", expr, jexlException.getMessage());
+            log.warn("Alarm Rule: {} Compile Error: {}.", expr, jexlException.getMessage());
             throw jexlException;
         } catch (Exception e) {
             log.error("Alarm Rule: {} Unknown Error: {}.", expr, e.getMessage());
@@ -285,7 +359,7 @@ private boolean execAlertExpression(Map<String, Object> fieldValueMap, String ex
         try {
             match = (Boolean) JexlExpressionRunner.evaluate(expression, fieldValueMap);
         } catch (JexlException jexlException) {
-            log.error("Alarm Rule: {} Run Error: {}.", expr, jexlException.getMessage());
+            log.warn("Alarm Rule: {} Run Error: {}.", expr, jexlException.getMessage());
             throw jexlException;
         } catch (Exception e) {
             log.error("Alarm Rule: {} Unknown Error: {}.", expr, e.getMessage());
@@ -300,11 +374,4 @@ private String calculateFingerprint(Map<String, String> fingerPrints) {
         return Arrays.hashCode(keyList.toArray(new String[0])) + "-"
                 + Arrays.hashCode(valueList.toArray(new String[0]));
     }
-
-    @EventListener(MonitorDeletedEvent.class)
-    public void onMonitorDeletedEvent(MonitorDeletedEvent event) {
-        log.info("calculate alarm receive monitor {} has been deleted.", event.getMonitorId());
-        this.pendingAlertMap.remove(String.valueOf(event.getMonitorId()));
-    }
-
 }

hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/controller/AlertDefineController.java

@@ -23,12 +23,9 @@
 import io.swagger.v3.oas.annotations.Parameter;
 import io.swagger.v3.oas.annotations.tags.Tag;
 import jakarta.validation.Valid;
-import java.util.List;
 import java.util.Objects;
-import java.util.stream.Collectors;
 import org.apache.hertzbeat.alert.service.AlertDefineService;
 import org.apache.hertzbeat.common.entity.alerter.AlertDefine;
-import org.apache.hertzbeat.common.entity.alerter.AlertDefineMonitorBind;
 import org.apache.hertzbeat.common.entity.dto.Message;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.http.ResponseEntity;
@@ -93,24 +90,4 @@ public ResponseEntity<Message<Void>> deleteAlertDefine(
         return ResponseEntity.ok(Message.success("Delete success"));
     }
 
-    @PostMapping(path = "/{alertDefineId}/monitors")
-    @Operation(summary = "Application alarm definition is associated with monitoring",
-            description = "Applies the association between specified alarm definitions and monitoring")
-    public ResponseEntity<Message<Void>> applyAlertDefineMonitorsBind(
-            @Parameter(description = "Alarm Definition ID", example = "6565463543") @PathVariable("alertDefineId") long alertDefineId,
-            @RequestBody List<AlertDefineMonitorBind> alertDefineMonitorBinds) {
-        alertDefineService.applyBindAlertDefineMonitors(alertDefineId, alertDefineMonitorBinds);
-        return ResponseEntity.ok(Message.success("Apply success"));
-    }
-
-    @GetMapping(path = "/{alertDefineId}/monitors")
-    @Operation(summary = "Application alarm definition is associated with monitoring",
-            description = "Applies the association between specified alarm definitions and monitoring")
-    public ResponseEntity<Message<List<AlertDefineMonitorBind>>> getAlertDefineMonitorsBind(
-            @Parameter(description = "Alarm Definition ID", example = "6565463543") @PathVariable("alertDefineId") long alertDefineId) {
-        List<AlertDefineMonitorBind> defineBinds = alertDefineService.getBindAlertDefineMonitors(alertDefineId);
-        defineBinds = defineBinds.stream().filter(item -> item.getMonitor() != null).collect(Collectors.toList());
-        return ResponseEntity.ok(Message.success(defineBinds));
-    }
-
 }

hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/controller/AlertsController.java

@@ -26,6 +26,7 @@
 import org.apache.hertzbeat.alert.dto.AlertSummary;
 import org.apache.hertzbeat.alert.service.AlertService;
 import org.apache.hertzbeat.common.entity.alerter.GroupAlert;
+import org.apache.hertzbeat.common.entity.alerter.SingleAlert;
 import org.apache.hertzbeat.common.entity.dto.Message;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.data.domain.Page;
@@ -49,8 +50,21 @@ public class AlertsController {
     @Autowired
     private AlertService alertService;
 
+    @GetMapping
+    @Operation(summary = "Query Alarms")
+    public ResponseEntity<Message<Page<SingleAlert>>> getAlerts(
+            @Parameter(description = "Alarm Status", example = "resolved") @RequestParam(required = false) String status,
+            @Parameter(description = "Alarm content fuzzy query", example = "linux") @RequestParam(required = false) String search,
+            @Parameter(description = "Sort field, default id", example = "name") @RequestParam(defaultValue = "id") String sort,
+            @Parameter(description = "Sort Type", example = "desc") @RequestParam(defaultValue = "desc") String order,
+            @Parameter(description = "List current page", example = "0") @RequestParam(defaultValue = "0") int pageIndex,
+            @Parameter(description = "Number of list pagination", example = "8") @RequestParam(defaultValue = "8") int pageSize) {
+        Page<SingleAlert> alertPage = alertService.getSingleAlerts(status, search, sort, order, pageIndex, pageSize);
+        return ResponseEntity.ok(Message.success(alertPage));
+    }
+
     @GetMapping("/group")
-    @Operation(summary = "Get a list of alarm information based on query filter items", description = "according to the query filter items to obtain a list of alarm information")
+    @Operation(summary = "Query Group Alarms")
     public ResponseEntity<Message<Page<GroupAlert>>> getGroupAlerts(
             @Parameter(description = "Alarm Status", example = "resolved") @RequestParam(required = false) String status,
             @Parameter(description = "Alarm content fuzzy query", example = "linux") @RequestParam(required = false) String search,

hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/dao/AlertDefineDao.java

@@ -18,12 +18,11 @@
 package org.apache.hertzbeat.alert.dao;
 
 import java.util.List;
+import java.util.Optional;
 import java.util.Set;
 import org.apache.hertzbeat.common.entity.alerter.AlertDefine;
 import org.springframework.data.jpa.repository.JpaRepository;
 import org.springframework.data.jpa.repository.JpaSpecificationExecutor;
-import org.springframework.data.jpa.repository.Query;
-import org.springframework.data.repository.query.Param;
 
 /**
  * AlertDefine Dao
@@ -37,24 +36,16 @@ public interface AlertDefineDao extends JpaRepository<AlertDefine, Long>, JpaSpe
     void deleteAlertDefinesByIdIn(Set<Long> alertDefineIds);
 
     /**
-     * Query the default alarm thresholds based on the monitoring metrics type
-     * @param app monitoring type
-     * @param metric metrics
+     * Query the alarm define by type
+     * @param type alarm type 
      * @return alarm defines
      */
-    List<AlertDefine> queryAlertDefinesByAppAndMetricAndPresetTrueAndEnableTrue(String app, String metric);
-
+    List<AlertDefine> findAlertDefinesByTypeAndEnableTrue(String type);
 
     /**
-     * Query the alarm definition list associated with the monitoring ID
-     * @param monitorId monitor id
-     * @param app monitor type
-     * @param metrics metrics
-     * @return Alarm Definition List
+     * Query the alarm define by name
+     * @param name alarm name
+     * @return alarm define
      */
-    @Query("select define from AlertDefine define join AlertDefineMonitorBind bind on bind.alertDefineId = define.id "
-            + "where bind.monitorId = :monitorId and define.app = :app and define.metric = :metrics and define.enable = true and define.preset = false")
-    List<AlertDefine> queryAlertDefinesByMonitor(@Param(value = "monitorId") Long monitorId,
-                                                 @Param(value = "app") String app,
-                                                 @Param(value = "metrics") String metrics);
+    Optional<AlertDefine> findAlertDefineByName(String name);
 }

hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/dao/AlertSilenceDao.java

@@ -17,6 +17,7 @@
 
 package org.apache.hertzbeat.alert.dao;
 
+import java.util.List;
 import java.util.Set;
 import org.apache.hertzbeat.common.entity.alerter.AlertSilence;
 import org.springframework.data.jpa.repository.JpaRepository;
@@ -35,4 +36,10 @@ public interface AlertSilenceDao extends JpaRepository<AlertSilence, Long>, JpaS
      */
     @Modifying
     void deleteAlertSilencesByIdIn(Set<Long> silenceIds);
+
+    /**
+     * find enabled alert silence
+     * @return enabled alert silence list
+     */
+    List<AlertSilence> findAlertSilencesByEnableTrue();
 }

hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/reduce/AlarmGroupReduce.java

@@ -106,6 +106,9 @@ public void checkAndSendGroups() {
      * Process single alert and group by defined rules
      */
     public void processGroupAlert(SingleAlert alert) {
+        // Generate alert fingerprint
+        String fingerprint = generateAlertFingerprint(alert);
+        alert.setFingerprint(fingerprint);
         Map<String, String> labels = alert.getLabels();
         if (labels == null || labels.isEmpty() || groupDefines.isEmpty()) {
             sendSingleAlert(alert);
@@ -153,10 +156,7 @@ private void processAlertByGroupDefine(SingleAlert alert, String defineName, Ale
             newCache.setAlertFingerprints(new HashMap<>());
             return newCache;
         });
-
-        // Generate alert fingerprint
-        String fingerprint = generateAlertFingerprint(alert);
-
+        String fingerprint = alert.getFingerprint();
         // Check if this is a duplicate alert
         SingleAlert existingAlert = cache.getAlertFingerprints().get(fingerprint);
         if (existingAlert != null) {

hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/reduce/AlarmSilenceReduce.java

@@ -48,16 +48,12 @@ public class AlarmSilenceReduce {
     public void silenceAlarm(GroupAlert groupAlert) {
         List<AlertSilence> alertSilenceList = CacheFactory.getAlertSilenceCache();
         if (alertSilenceList == null) {
-            alertSilenceList = alertSilenceDao.findAll();
+            alertSilenceList = alertSilenceDao.findAlertSilencesByEnableTrue();
             CacheFactory.setAlertSilenceCache(alertSilenceList);
         }
 
         // Check each silence rule
         for (AlertSilence alertSilence : alertSilenceList) {
-            if (!alertSilence.isEnable()) {
-                continue;
-            }
-
             // Check if alert matches silence rule
             boolean match = alertSilence.isMatchAll();
             if (!match && groupAlert.getGroupLabels() != null) {