Merge pull request #845 from VannTen/feat/stuck_rollout

feat: Add KubeDeploymentRolloutStuck

Author: povilasv

alerts/apps_alerts.libsonnet

@@ -87,6 +87,21 @@
             'for': '15m',
             alert: 'KubeDeploymentReplicasMismatch',
           },
+          {
+            expr: |||
+              kube_deployment_status_condition{condition="Progressing", status="false",%(prefixedNamespaceSelector)s%(kubeStateMetricsSelector)s}
+              != 0
+            ||| % $._config,
+            labels: {
+              severity: 'warning',
+            },
+            annotations: {
+              description: 'Rollout of deployment {{ $labels.namespace }}/{{ $labels.deployment }} is not progressing for longer than 15 minutes.',
+              summary: 'Deployment rollout is not progressing.',
+            },
+            'for': '15m',
+            alert: 'KubeDeploymentRolloutStuck',
+          },
           {
             expr: |||
               (

runbook.md

@@ -37,6 +37,9 @@ This page collects this repositories alerts and begins the process of describing
 ##### Alert Name: "KubeDeploymentReplicasMismatch"
 + *Message*: `Deployment {{ $labels.namespace }}/{{ $labels.deployment }} replica mismatch`
 + *Severity*: warning
+##### Alert Name: "KubeDeploymentRolloutStuck"
++ *Message*: `Rollout of deployment {{ $labels.namespace }}/{{ $labels.deployment }} is not progressing`
++ *Severity*: warning
 ##### Alert Name: "KubeStatefulSetReplicasMismatch"
 + *Message*: `StatefulSet {{ $labels.namespace }}/{{ $labels.statefulset }} replica mismatch`
 + *Severity*: warning

tests.yaml

@@ -813,6 +813,30 @@ tests:
         runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubedaemonsetrolloutstuck
   - eval_time: 34m
     alertname: KubeDaemonSetRolloutStuck
+# KubeDeploymentRolloutStuck
+- interval: 1m
+  input_series:
+  - series: 'kube_deployment_status_condition{job="kube-state-metrics",namespace="monitoring",deployment="stuck", condition="Progressing", status="false"}'
+    values: '1+0x17 0+0x5'
+  alert_rule_test:
+  - eval_time: 14m
+    alertname: KubeDeploymentRolloutStuck
+  - eval_time: 16m
+    alertname: KubeDeploymentRolloutStuck
+    exp_alerts:
+    - exp_labels:
+        job: kube-state-metrics
+        namespace: monitoring
+        deployment: stuck
+        severity: warning
+        condition: Progressing
+        status: "false"
+      exp_annotations:
+        summary: 'Deployment rollout is not progressing.'
+        description: 'Rollout of deployment monitoring/stuck is not progressing for longer than 15 minutes.'
+        runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubedeploymentrolloutstuck
+  - eval_time: 18m
+    alertname: KubeDeploymentRolloutStuck
 - interval: 1m
   # Misscheduled is non zero.
   input_series: