diff --git a/alerts/apps_alerts.libsonnet b/alerts/apps_alerts.libsonnet index 46363745d..5d2d8f373 100644 --- a/alerts/apps_alerts.libsonnet +++ b/alerts/apps_alerts.libsonnet @@ -167,13 +167,13 @@ local utils = import '../lib/utils.libsonnet'; unless kube_statefulset_status_update_revision{%(prefixedNamespaceSelector)s%(kubeStateMetricsSelector)s} ) - * + * on(namespace, statefulset, job, %(clusterLabel)s) ( kube_statefulset_replicas{%(prefixedNamespaceSelector)s%(kubeStateMetricsSelector)s} != kube_statefulset_status_replicas_updated{%(prefixedNamespaceSelector)s%(kubeStateMetricsSelector)s} ) - ) and ( + ) and on(namespace, statefulset, job, %(clusterLabel)s) ( changes(kube_statefulset_status_replicas_updated{%(prefixedNamespaceSelector)s%(kubeStateMetricsSelector)s}[5m]) == 0 diff --git a/tests/apps_alerts-test.yaml b/tests/apps_alerts-test.yaml index 803c61944..3ab42d042 100644 --- a/tests/apps_alerts-test.yaml +++ b/tests/apps_alerts-test.yaml @@ -25,3 +25,31 @@ tests: description: "PDB ns1/pdb1 expects 1 more healthy pods. The desired number of healthy pods has not been met for at least 15m." runbook_url: "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepdbnotenoughhealthypods" summary: "PDB does not have enough healthy pods." + +- interval: 1m + name: KubeStatefulSetUpdateNotRolledOut still fires even if another label (e.g. instance) is present + input_series: + - series: 'kube_statefulset_status_current_revision{job="kube-state-metrics", cluster="c1", namespace="ns1", statefulset="ss1", revision="foo", instance="custom"}' + values: '1x15' + - series: 'kube_statefulset_status_update_revision{job="kube-state-metrics", cluster="c1", namespace="ns1", statefulset="ss1", revision="bar", instance="custom"}' + values: '1x15' + - series: 'kube_statefulset_replicas{job="kube-state-metrics", cluster="c1", namespace="ns1", statefulset="ss1", instance="custom"}' + values: '5x15' + - series: 'kube_statefulset_status_replicas_updated{job="kube-state-metrics", cluster="c1", namespace="ns1", statefulset="ss1", instance="custom"}' + values: '1x15' + alert_rule_test: + - eval_time: 14m + alertname: KubeStatefulSetUpdateNotRolledOut + - eval_time: 15m + alertname: KubeStatefulSetUpdateNotRolledOut + exp_alerts: + - exp_labels: + cluster: "c1" + job: "kube-state-metrics" + namespace: "ns1" + severity: "warning" + statefulset: "ss1" + exp_annotations: + description: "StatefulSet ns1/ss1 update has not been rolled out." + runbook_url: "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubestatefulsetupdatenotrolledout" + summary: "StatefulSet update has not been rolled out."