apps sc: update kube-state-metrics alerts

Author: anders-elastisys

helmfile.d/charts/prometheus-alerts/CHANGELOG.md

@@ -7,6 +7,11 @@
 1. helmfile.d/charts/prometheus-alerts/templates/alerts/kubernetes-resources.yaml
     - MODIFIED - KubeCPUOvercommit properly compute per cluster as done upstream
     - MODIFIED - KubeMemoryOvercommit properly compute per cluster as done upstream
+1. helmfile.d/charts/prometheus-alerts/templates/alerts/kube-state-metrics.yaml
+    - MODIFIED - KubeStateMetricsListErrors properly compute per cluster as done upstream
+    - MODIFIED - KubeStateMetricsWatchErrors properly compute per cluster as done upstream
+    - MODIFIED - KubeStateMetricsShardingMismatch properly compute per cluster as done upstream
+    - MODIFIED - KubeStateMetricsShardsMissing properly compute per cluster as done upstream
 
 ## 2022.12.01
 

helmfile.d/charts/prometheus-alerts/templates/alerts/kube-state-metrics.yaml

@@ -23,9 +23,9 @@ spec:
         runbook_url: {{ .Values.defaultRules.runbookUrl }}kube-state-metrics/kubestatemetricslisterrors
         summary: kube-state-metrics is experiencing errors in list operations.
       expr: |-
-        (sum(rate(kube_state_metrics_list_total{job="kube-state-metrics",result="error"}[5m]))
+        (sum(rate(kube_state_metrics_list_total{job="kube-state-metrics",result="error"}[5m])) by (cluster)
           /
-        sum(rate(kube_state_metrics_list_total{job="kube-state-metrics"}[5m])))
+        sum(rate(kube_state_metrics_list_total{job="kube-state-metrics"}[5m])) by (cluster))
         > 0.01
       for: 15m
       labels:
@@ -39,9 +39,9 @@ spec:
         runbook_url: {{ .Values.defaultRules.runbookUrl }}kube-state-metrics/kubestatemetricswatcherrors
         summary: kube-state-metrics is experiencing errors in watch operations.
       expr: |-
-        (sum(rate(kube_state_metrics_watch_total{job="kube-state-metrics",result="error"}[5m]))
+        (sum(rate(kube_state_metrics_watch_total{job="kube-state-metrics",result="error"}[5m])) by (cluster)
           /
-        sum(rate(kube_state_metrics_watch_total{job="kube-state-metrics"}[5m])))
+        sum(rate(kube_state_metrics_watch_total{job="kube-state-metrics"}[5m])) by (cluster))
         > 0.01
       for: 15m
       labels:
@@ -54,7 +54,7 @@ spec:
         description: kube-state-metrics pods are running with different --total-shards configuration, some Kubernetes objects may be exposed multiple times or not exposed at all.
         runbook_url: {{ .Values.defaultRules.runbookUrl }}kube-state-metrics/kubestatemetricsshardingmismatch
         summary: kube-state-metrics sharding is misconfigured.
-      expr: stdvar (kube_state_metrics_total_shards{job="kube-state-metrics"}) != 0
+      expr: stdvar (kube_state_metrics_total_shards{job="kube-state-metrics"}) by (cluster) != 0
       for: 15m
       labels:
         severity: critical
@@ -67,9 +67,9 @@ spec:
         runbook_url: {{ .Values.defaultRules.runbookUrl }}kube-state-metrics/kubestatemetricsshardsmissing
         summary: kube-state-metrics shards are missing.
       expr: |-
-        2^max(kube_state_metrics_total_shards{job="kube-state-metrics"}) - 1
+        2^max(kube_state_metrics_total_shards{job="kube-state-metrics"}) by (cluster) - 1
           -
-        sum( 2 ^ max by (shard_ordinal) (kube_state_metrics_shard_ordinal{job="kube-state-metrics"}) )
+        sum( 2 ^ max by (shard_ordinal) (kube_state_metrics_shard_ordinal{job="kube-state-metrics"})) by (cluster)
         != 0
       for: 15m
       labels: