fix: set rules as silenced if all alerts under them are silenced

Author: PeterYurkovich

web/src/components/alerting/AlertRulesPage.tsx

@@ -204,7 +204,7 @@ const AlertRulesPage_: FC = () => {
           onFilterChange={onFilterChange}
           rowFilters={rowFilters}
         />
-        {silences.loadError && <SilencesNotLoadedWarning silencesLoadError={silences.loadError} />}
+        {silences?.loadError && <SilencesNotLoadedWarning silencesLoadError={silences.loadError} />}
         <div id="alert-rules-table-scroll">
           <VirtualizedTable<Rule>
             aria-label={t('Alerting rules')}

web/src/components/alerting/AlertUtils.tsx

@@ -5,9 +5,14 @@ import {
   Alert,
   AlertSeverity,
   AlertStates,
+  PrometheusAlert,
   PrometheusLabels,
+  PrometheusRule,
   RowFilter,
   Rule,
+  RuleStates,
+  Silence,
+  SilenceStates,
   Timestamp,
 } from '@openshift-console/dynamic-plugin-sdk';
 import { AlertSource } from '../types';
@@ -420,3 +425,147 @@ export const NamespaceGroupVersionKind = {
   kind: NamespaceModel.kind,
   version: null,
 };
+
+// This function looks to take a alerts and rules and then apply a set of silences to them
+// This function mutates the arrays in place and then returns them
+export const applySilences = ({
+  alerts,
+  silences,
+  rules,
+}: {
+  silences: Array<Silence>;
+  alerts: Array<Alert>;
+  rules: Array<Rule>;
+}): { silences: Array<Silence>; alerts: Array<Alert>; rules: Array<Rule> } => {
+  // We only need to check alerts that are either firing or silenced for if they are still silenced
+  const firingAlerts = alerts.filter(isAlertFiring);
+  applySilencesToAlerts({ firingAlerts, silences });
+
+  // Only check rules that are firing, silenced or pending to see if they are still silenced
+  const firingRules = rules.filter(isRuleFiring);
+  applySilencesToRules({ firingRules, silences });
+
+  // Add each alert that is being effected by a silence to the firingAlerts list on the silence
+  const appliedSilences = silences.map((silence) => {
+    silence.firingAlerts = firingAlerts.filter((firingAlert) =>
+      isAlertSilenced(firingAlert, silence),
+    );
+    return silence;
+  });
+  return { alerts, silences: appliedSilences, rules };
+};
+
+// This fucntion mutates the firingAlerts parameter in place to set silence fields on each alert
+const applySilencesToAlerts = ({
+  firingAlerts,
+  silences,
+}: {
+  silences: Array<Silence>;
+  firingAlerts: Array<Alert>;
+}) => {
+  // For each firing alert, store a list of the Silences that are silencing it
+  // and set its state to show it is silenced
+  firingAlerts.forEach((firingAlert) => {
+    firingAlert.silencedBy = silences.filter(
+      (silence) =>
+        silence.status?.state === SilenceStates.Active && isAlertSilenced(firingAlert, silence),
+    );
+
+    if (firingAlert.silencedBy.length) {
+      firingAlert.state = AlertStates.Silenced;
+      // Also set the state of Alerts in `rule.alerts`
+
+      firingAlert.rule.alerts.forEach((ruleAlert) => {
+        if (firingAlert.silencedBy?.some((silence) => isAlertSilenced(ruleAlert, silence))) {
+          ruleAlert.state = AlertStates.Silenced;
+        }
+      });
+
+      if (
+        firingAlert.rule.alerts.length !== 0 &&
+        firingAlert.rule.alerts.every((alert) => alert.state === AlertStates.Silenced)
+      ) {
+        firingAlert.rule.state = RuleStates.Silenced;
+
+        firingAlert.rule.silencedBy = silences.filter(
+          (silence) =>
+            silence.status?.state === SilenceStates.Active &&
+            firingAlert.rule.alerts.some((alert) => isAlertSilenced(alert, silence)),
+        );
+      }
+    }
+  });
+
+  return firingAlerts;
+};
+
+// This fucntion mutates the firingRules parameter in place to set silence fields on each rule
+const applySilencesToRules = ({
+  firingRules,
+  silences,
+}: {
+  silences: Array<Silence>;
+  firingRules: Array<Rule>;
+}) => {
+  // For each firing alert, store a list of the Silences that are silencing it
+  // and set its state to show it is silenced
+  firingRules.forEach((firingRule) => {
+    firingRule.silencedBy = silences.filter(
+      (silence) =>
+        silence.status?.state === SilenceStates.Active && isRuleSilenced(firingRule, silence),
+    );
+
+    if (firingRule.silencedBy.length) {
+      firingRule.state = RuleStates.Silenced;
+
+      firingRule.alerts.forEach((ruleAlert) => {
+        if (firingRule.silencedBy?.some((silence) => isAlertSilenced(ruleAlert, silence))) {
+          ruleAlert.state = AlertStates.Silenced;
+        }
+      });
+
+      if (
+        firingRule.alerts.length !== 0 &&
+        firingRule.alerts.every((alert) => alert.state === AlertStates.Silenced)
+      ) {
+        firingRule.state = RuleStates.Silenced;
+
+        firingRule.silencedBy = silences.filter(
+          (silence) =>
+            silence.status?.state === SilenceStates.Active &&
+            firingRule.alerts.some((alert) => isAlertSilenced(alert, silence)),
+        );
+      }
+    }
+  });
+
+  return firingRules;
+};
+
+const isAlertFiring = (alert: PrometheusAlert) =>
+  alert?.state === AlertStates.Firing || alert?.state === AlertStates.Silenced;
+
+const isRuleFiring = (rule: PrometheusRule) =>
+  rule?.state === RuleStates.Firing ||
+  rule?.state === RuleStates.Silenced ||
+  rule?.state === RuleStates.Pending;
+
+// Determine if an Alert is silenced by a Silence (if all of the Silence's matchers match one of the
+// Alert's labels)
+const isAlertSilenced = (alert: PrometheusAlert, silence: Silence): boolean => {
+  return (
+    isAlertFiring(alert) &&
+    silence.matchers.every((matcher) => {
+      const alertValue = alert.labels[matcher.name] ?? '';
+      const isMatch = matcher.isRegex
+        ? new RegExp(`^${matcher.value}$`).test(alertValue)
+        : alertValue === matcher.value;
+      return matcher.isEqual === false && alertValue ? !isMatch : isMatch;
+    })
+  );
+};
+
+// Determine if an Rule is silenced by a Silence (if all alerts for a rule are silenced)
+export const isRuleSilenced = (rule: PrometheusRule, silence: Silence): boolean => {
+  return isRuleFiring(rule) && rule.alerts.every((alert) => isAlertSilenced(alert, silence));
+};

web/src/components/utils.ts

@@ -125,21 +125,6 @@ export const getSilenceName = (silence: Silence) => {
 export const alertDescription = (alert: PrometheusAlert | Rule): string =>
   alert.annotations?.description || alert.annotations?.message || alert.labels?.alertname;
 
-// Determine if an Alert is silenced by a Silence (if all of the Silence's matchers match one of the
-// Alert's labels)
-export const isSilenced = (alert: PrometheusAlert, silence: Silence): boolean => {
-  return (
-    [AlertStates.Firing, AlertStates.Silenced].includes(alert.state) &&
-    silence.matchers.every((matcher) => {
-      const alertValue = alert.labels[matcher.name] ?? '';
-      const isMatch = matcher.isRegex
-        ? new RegExp(`^${matcher.value}$`).test(alertValue)
-        : alertValue === matcher.value;
-      return matcher.isEqual === false && alertValue ? !isMatch : isMatch;
-    })
-  );
-};
-
 export type ListOrder = (number | string)[];
 
 // Severity sort order is "critical" > "warning" > (anything else in A-Z order) > "none"

web/src/store/reducers.ts

@@ -1,15 +1,7 @@
 import * as _ from 'lodash-es';
-import {
-  Alert,
-  AlertStates,
-  RuleStates,
-  Silence,
-  SilenceStates,
-} from '@openshift-console/dynamic-plugin-sdk';
 
 import { ActionType, ObserveAction } from './actions';
 
-import { isSilenced } from '../components/utils';
 import { MONITORING_DASHBOARDS_VARIABLE_ALL_OPTION_KEY } from '../components/dashboards/legacy/utils';
 import {
   defaultObserveState,
@@ -19,6 +11,7 @@ import {
   QueryStructure,
 } from './store';
 import { produce } from 'immer';
+import { applySilences } from '../components/alerting/AlertUtils';
 
 const monitoringReducer = produce((draft: ObserveState, action: ObserveAction): ObserveState => {
   if (!draft) {
@@ -179,23 +172,16 @@ const monitoringReducer = produce((draft: ObserveState, action: ObserveAction):
           loadError: null,
         };
       }
-
-      const alerts = draft.alerting[datasource][identifier].alerts;
-      const silences = draft.alerting[datasource][identifier].silences;
-
-      const firingAlerts = alerts.filter(isAlertFiring);
-
-      const updatedAlerts = silenceFiringAlerts(firingAlerts, silences.data);
-      draft.alerting[datasource][identifier].alerts = updatedAlerts;
-
-      silences.data = silences.data.map((silence) => {
-        silence.firingAlerts = firingAlerts.filter((firingAlert) =>
-          isSilenced(firingAlert, silence),
-        );
-        return silence;
+      const { alerts, rules, silences } = applySilences({
+        alerts: draft.alerting[datasource][identifier].alerts,
+        silences: draft.alerting[datasource][identifier].silences.data,
+        rules: draft.alerting[datasource][identifier].rules,
       });
 
-      draft.alerting[datasource][identifier].silences = silences;
+      draft.alerting[datasource][identifier].alerts = alerts;
+      draft.alerting[datasource][identifier].rules = rules;
+      draft.alerting[datasource][identifier].silences.data = silences;
+
       break;
     }
 
@@ -406,54 +392,4 @@ const monitoringReducer = produce((draft: ObserveState, action: ObserveAction):
   return draft;
 });
 
-export type NotificationAlerts = {
-  data: Alert[];
-  loaded: boolean;
-  loadError?: {
-    message?: string;
-  };
-};
-
-const isAlertFiring = (alert: Alert) =>
-  alert?.state === AlertStates.Firing || alert?.state === AlertStates.Silenced;
-
-const silenceFiringAlerts = (
-  firingAlerts: Array<Alert>,
-  silences: Array<Silence>,
-): Array<Alert> => {
-  // For each firing alert, store a list of the Silences that are silencing it
-  // and set its state to show it is silenced
-  firingAlerts.forEach((firingAlert) => {
-    firingAlert.silencedBy = silences.filter(
-      (silence) =>
-        silence.status?.state === SilenceStates.Active && isSilenced(firingAlert, silence),
-    );
-
-    if (firingAlert.silencedBy.length) {
-      firingAlert.state = AlertStates.Silenced;
-      // Also set the state of Alerts in `rule.alerts`
-
-      firingAlert.rule.alerts.forEach((ruleAlert) => {
-        if (firingAlert.silencedBy?.some((silence) => isSilenced(ruleAlert, silence))) {
-          ruleAlert.state = AlertStates.Silenced;
-        }
-      });
-
-      if (
-        firingAlert.rule.alerts.length !== 0 &&
-        firingAlert.rule.alerts.every((alert) => alert.state === AlertStates.Silenced)
-      ) {
-        firingAlert.rule.state = RuleStates.Silenced;
-
-        firingAlert.rule.silencedBy = silences.filter(
-          (silence) =>
-            silence.status?.state === SilenceStates.Active &&
-            firingAlert.rule.alerts.some((alert) => isSilenced(alert, silence)),
-        );
-      }
-    }
-  });
-  return firingAlerts;
-};
-
 export default monitoringReducer;