Merge pull request #2598 from jan--f/pull-OCPBUGS-33691-fix

OCPBUGS-33691: mixin: longer rate interval for Alertmanager[Cluster]FailedToSendAlerts

Author: openshift-merge-bot[bot]

assets/alertmanager/prometheus-rule.yaml

@@ -48,9 +48,9 @@ spec:
         summary: An Alertmanager instance failed to send notifications.
       expr: |
         (
-          rate(alertmanager_notifications_failed_total{job=~"alertmanager-main|alertmanager-user-workload"}[5m])
+          rate(alertmanager_notifications_failed_total{job=~"alertmanager-main|alertmanager-user-workload"}[15m])
         /
-          ignoring (reason) group_left rate(alertmanager_notifications_total{job=~"alertmanager-main|alertmanager-user-workload"}[5m])
+          ignoring (reason) group_left rate(alertmanager_notifications_total{job=~"alertmanager-main|alertmanager-user-workload"}[15m])
         )
         > 0.01
       for: 5m
@@ -63,9 +63,9 @@ spec:
         summary: All Alertmanager instances in a cluster failed to send notifications to a critical integration.
       expr: |
         min by (namespace,service, integration) (
-          rate(alertmanager_notifications_failed_total{job=~"alertmanager-main|alertmanager-user-workload", integration=~`.*`}[5m])
+          rate(alertmanager_notifications_failed_total{job=~"alertmanager-main|alertmanager-user-workload", integration=~`.*`}[15m])
         /
-          ignoring (reason) group_left rate(alertmanager_notifications_total{job=~"alertmanager-main|alertmanager-user-workload", integration=~`.*`}[5m])
+          ignoring (reason) group_left rate(alertmanager_notifications_total{job=~"alertmanager-main|alertmanager-user-workload", integration=~`.*`}[15m])
         )
         > 0.01
       for: 5m

jsonnet/utils/sanitize-rules.libsonnet

@@ -167,6 +167,28 @@ local patchedRules = [
         labels: {
           severity: 'warning',
         },
+        expr: |||
+          min by (namespace,service, integration) (
+            rate(alertmanager_notifications_failed_total{job=~"alertmanager-main|alertmanager-user-workload", integration=~`.*`}[15m])
+          /
+            ignoring (reason) group_left rate(alertmanager_notifications_total{job=~"alertmanager-main|alertmanager-user-workload", integration=~`.*`}[15m])
+          )
+          > 0.01
+        |||,
+      },
+      {
+        alert: 'AlertmanagerFailedToSendAlerts',
+        labels: {
+          severity: 'warning',
+        },
+        expr: |||
+          (
+            rate(alertmanager_notifications_failed_total{job=~"alertmanager-main|alertmanager-user-workload"}[15m])
+          /
+            ignoring (reason) group_left rate(alertmanager_notifications_total{job=~"alertmanager-main|alertmanager-user-workload"}[15m])
+          )
+          > 0.01
+        |||,
       },
       {
         alert: 'AlertmanagerConfigInconsistent',