Merge pull request #1922 from tjungblu/OCPBUGS-15430

OCPBUGS-15430: move alerting rules from CMO over

Author: openshift-merge-bot[bot]

bindata/assets/alerts/kube-apiserver-down.yaml

@@ -0,0 +1,47 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: kubernetes-system-apiserver
+  namespace: openshift-kube-apiserver
+spec:
+  groups:
+    - name: kubernetes-system-apiserver
+      rules:
+        - alert: KubeAggregatedAPIErrors
+          annotations:
+            description: Kubernetes aggregated API {{ $labels.instance }}/{{ $labels.name }} has reported {{ $labels.reason }} errors.
+            runbook_url: https://github.com/openshift/runbooks/blob/master/alerts/cluster-kube-apiserver-operator/KubeAggregatedAPIErrors.md
+            summary: Kubernetes aggregated API has reported errors.
+          expr: |
+            sum by(cluster, instance, name, reason)(increase(aggregator_unavailable_apiservice_total{job="apiserver"}[1m])) > 0
+          for: 10m
+          labels:
+            severity: warning
+        - alert: KubeAggregatedAPIDown
+          annotations:
+            description: Kubernetes aggregated API {{ $labels.name }}/{{ $labels.namespace }} has been only {{ $value | humanize }}% available over the last 10m.
+            summary: Kubernetes aggregated API is down.
+          expr: |
+            (1 - max by(name, namespace, cluster)(avg_over_time(aggregator_unavailable_apiservice{job="apiserver"}[10m]))) * 100 < 85
+          for: 15m
+          labels:
+            severity: warning
+        - alert: KubeAPIDown
+          annotations:
+            description: KubeAPI has disappeared from Prometheus target discovery.
+            runbook_url: https://github.com/openshift/runbooks/blob/master/alerts/cluster-kube-apiserver-operator/KubeAPIDown.md
+            summary: Target disappeared from Prometheus target discovery.
+          expr: |
+            absent(up{job="apiserver"} == 1)
+          for: 15m
+          labels:
+            severity: critical
+        - alert: KubeAPITerminatedRequests
+          annotations:
+            description: The kubernetes apiserver has terminated {{ $value | humanizePercentage }} of its incoming requests.
+            summary: The kubernetes apiserver has terminated {{ $value | humanizePercentage }} of its incoming requests.
+          expr: |
+            sum by(cluster) (rate(apiserver_request_terminations_total{job="apiserver"}[10m])) / ( sum by(cluster) (rate(apiserver_request_total{job="apiserver"}[10m])) + sum by(cluster) (rate(apiserver_request_terminations_total{job="apiserver"}[10m])) ) > 0.20
+          for: 5m
+          labels:
+            severity: warning

pkg/operator/starter.go

@@ -253,6 +253,7 @@ func RunOperator(ctx context.Context, controllerContext *controllercmd.Controlle
 			"assets/kube-apiserver/storage-version-migration-prioritylevelconfiguration-v1beta3.yaml",
 			"assets/alerts/api-usage.yaml",
 			"assets/alerts/audit-errors.yaml",
+			"assets/alerts/kube-apiserver-down.yaml",
 			"assets/alerts/kube-apiserver-requests.yaml",
 			"assets/alerts/kube-apiserver-slos-basic.yaml",
 			"assets/alerts/podsecurity-violations.yaml",

pkg/test/assets_test.go

@@ -36,6 +36,10 @@ func readAllYaml(path string, t *testing.T) {
 			return false, nil
 		}
 		// there is an alert message containing $labels strings that cause the reader to fail.
+		if strings.HasSuffix(info.Name(), "kube-apiserver-down.yaml") {
+			return false, nil
+		}
+		// there is an alert message containing $labels strings that cause the reader to fail.
 		if strings.HasSuffix(info.Name(), "api-usage.yaml") {
 			return false, nil
 		}