diff --git a/alerts/kubelet.libsonnet b/alerts/kubelet.libsonnet index 854a77692..c47aff89c 100644 --- a/alerts/kubelet.libsonnet +++ b/alerts/kubelet.libsonnet @@ -12,6 +12,9 @@ local utils = import '../lib/utils.libsonnet'; kubeletCertExpirationWarningSeconds: 7 * 24 * 3600, kubeletCertExpirationCriticalSeconds: 1 * 24 * 3600, + + // Evictions per second that will trigger an alert. The default value will trigger on any evictions. + KubeNodeEvictionRateThreshold: 0.0, }, prometheusAlerts+:: { @@ -37,6 +40,24 @@ local utils = import '../lib/utils.libsonnet'; 'for': '15m', alert: 'KubeNodeNotReady', }, + { + alert: 'KubeNodePressure', + expr: ||| + kube_node_status_condition{%(kubeStateMetricsSelector)s,condition=~"(MemoryPressure|DiskPressure|PIDPressure)",status="true"} == 1 + and on (%(clusterLabel)s, node) + kube_node_spec_unschedulable{%(kubeStateMetricsSelector)s} == 0 + ||| % $._config, + labels: { + severity: 'info', + }, + 'for': '10m', + annotations: { + description: '{{ $labels.node }}%s has active Condition {{ $labels.condition }}. This is caused by resource usage exceeding eviction thresholds.' % [ + utils.ifShowMultiCluster($._config, ' on cluster {{ $labels.%(clusterLabel)s }}' % $._config), + ], + summary: 'Node has as active Condition.', + }, + }, { expr: ||| (kube_node_spec_taint{%(kubeStateMetricsSelector)s,key="node.kubernetes.io/unreachable",effect="NoSchedule"} unless ignoring(key,value) kube_node_spec_taint{%(kubeStateMetricsSelector)s,key=~"%(kubeNodeUnreachableIgnoreKeys)s"}) == 1 @@ -103,6 +124,27 @@ local utils = import '../lib/utils.libsonnet'; summary: 'Node readiness status is flapping.', }, }, + { + alert: 'KubeNodeEviction', + expr: ||| + sum(rate(kubelet_evictions{%(kubeletSelector)s}[15m])) by(%(clusterLabel)s, eviction_signal, instance) + * on (%(clusterLabel)s, instance) group_left(node) + max by (%(clusterLabel)s, instance, node) ( + kubelet_node_name{%(kubeletSelector)s} + ) + > %(KubeNodeEvictionRateThreshold)s + ||| % $._config, + labels: { + severity: 'info', + }, + 'for': '0s', + annotations: { + description: 'Node {{ $labels.node }}%s is evicting Pods due to {{ $labels.eviction_signal }}. Eviction occurs when eviction thresholds are crossed, typically caused by Pods exceeding RAM/ephemeral-storage limits.' % [ + utils.ifShowMultiCluster($._config, ' on {{ $labels.%(clusterLabel)s }}' % $._config), + ], + summary: 'Node is evicting pods.', + }, + }, { alert: 'KubeletPlegDurationHigh', expr: ||| diff --git a/runbook.md b/runbook.md index f30238902..7a33cd03e 100644 --- a/runbook.md +++ b/runbook.md @@ -182,10 +182,15 @@ This page collects this repositories alerts and begins the process of describing ### Group Name: "kubernetes-system" ##### Alert Name: "KubeNodeNotReady" -+ *Message*: `{{ $labels.node }} has been unready for more than 15 minutes."` ++ *Message*: `{{ $labels.node }} has been unready for more than 15 minutes.` + *Severity*: warning + *Runbook*: [Link](https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubenodenotready/) +##### Alert Name: "KubeNodePressure" ++ *Message*: `{{ $labels.node }} has active Condition {{ $labels.condition }}. This is caused by resource usage exceeding eviction thresholds.` ++ *Severity*: info ++ *Runbook*: [Link](https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubenodepressure/) + ##### Alert Name: "KubeNodeUnreachable" + *Message*: `{{ $labels.node }} is unreachable and some workloads may be rescheduled.` + *Severity*: warning @@ -201,6 +206,11 @@ This page collects this repositories alerts and begins the process of describing + *Severity*: warning + *Runbook*: [Link](https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubenodereadinessflapping/) +##### Alert Name: "KubeNodeEviction" ++ *Message*: `Node {{ $labels.node }} is evicting Pods due to {{ $labels.eviction_signal }}. Eviction occurs when eviction thresholds are crossed, typically caused by Pods exceeding RAM/ephemeral-storage limits.` ++ *Severity*: info ++ *Runbook*: [Link](https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubenodeeviction) + ##### Alert Name: "KubeletPlegDurationHigh" + *Message*: `The Kubelet Pod Lifecycle Event Generator has a 99th percentile duration of {{ $value }} seconds on node {{ $labels.node }}.` + *Severity*: warning diff --git a/tests/tests.yaml b/tests/tests.yaml index 49a81c730..24dcfba67 100644 --- a/tests/tests.yaml +++ b/tests/tests.yaml @@ -626,6 +626,43 @@ tests: description: 'minikube has been unready for more than 15 minutes.' runbook_url: 'https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubenodenotready' +- interval: 1m + input_series: + # node=minikube is uncordoned so we expect the alert to fire + - series: 'kube_node_status_condition{condition="MemoryPressure",status="true",endpoint="https-main",cluster="kubernetes",instance="10.0.2.15:10250",job="kube-state-metrics",namespace="monitoring",node="minikube",pod="kube-state-metrics-b894d84cc-d6htw",service="kube-state-metrics"}' + values: '0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1' + - series: 'kube_node_spec_unschedulable{endpoint="https-main",cluster="kubernetes",instance="10.0.2.15:10250",job="kube-state-metrics",namespace="monitoring",node="minikube",pod="kube-state-metrics-b894d84cc-d6htw",service="kube-state-metrics"}' + values: '0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0' + # node=minikube2 is cordoned so we expect the alert to not fire + - series: 'kube_node_status_condition{condition="MemoryPressure",status="true",endpoint="https-main",cluster="kubernetes",instance="10.0.2.15:10250",job="kube-state-metrics",namespace="monitoring",node="minikube2",pod="kube-state-metrics-b894d84cc-d6htw",service="kube-state-metrics"}' + values: '0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1' + - series: 'kube_node_spec_unschedulable{endpoint="https-main",cluster="kubernetes",instance="10.0.2.15:10250",job="kube-state-metrics",namespace="monitoring",node="minikube2",pod="kube-state-metrics-b894d84cc-d6htw",service="kube-state-metrics"}' + values: '1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1' + alert_rule_test: + # does not meet 'for: 10m' threshold + - eval_time: 12m + alertname: KubeNodePressure + exp_alerts: [] + - eval_time: 15m + alertname: KubeNodePressure + exp_alerts: + - exp_labels: + condition: MemoryPressure + status: "true" + cluster: kubernetes + node: minikube + severity: info + endpoint: https-main + instance: 10.0.2.15:10250 + job: kube-state-metrics + namespace: monitoring + pod: kube-state-metrics-b894d84cc-d6htw + service: kube-state-metrics + exp_annotations: + summary: "Node has as active Condition." + description: 'minikube has active Condition MemoryPressure. This is caused by resource usage exceeding eviction thresholds.' + runbook_url: 'https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubenodepressure' + - interval: 1m input_series: # node=minikube is uncordoned so we expect the alert to fire @@ -651,6 +688,29 @@ tests: description: 'The readiness status of node minikube has changed 9 times in the last 15 minutes.' runbook_url: 'https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubenodereadinessflapping' +- interval: 1m + input_series: + # This metric only appears with non-zero values, meaning the transition from 0 --> 1 doesn't trigger an alert + # However, since that's undesired behavior it'd be kinda pointless to test for it, even though it's expected. + - series: 'kubelet_evictions{eviction_signal="memory.available",endpoint="https-main",cluster="kubernetes",instance="10.0.2.15:10250",job="kubelet",namespace="monitoring",node="minikube",pod="kube-state-metrics-b894d84cc-d6htw",service="kube-state-metrics"}' + values: '1 1 1 1 1 1 2 2 2 2 2 2 2 2 2 3 3 3 3 3' + - series: 'kubelet_node_name{cluster="kubernetes", instance="10.0.2.15:10250", node="minikube", job="kubelet"}' + values: '1x20' + alert_rule_test: + - eval_time: 18m + alertname: KubeNodeEviction + exp_alerts: + - exp_labels: + eviction_signal: memory.available + cluster: kubernetes + node: minikube + instance: 10.0.2.15:10250 + severity: info + exp_annotations: + summary: "Node is evicting pods." + description: 'Node minikube is evicting Pods due to memory.available. Eviction occurs when eviction thresholds are crossed, typically caused by Pods exceeding RAM/ephemeral-storage limits.' + runbook_url: 'https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubenodeeviction' + # Verify that node:node_num_cpu:sum triggers no many-to-many errors. - interval: 1m input_series: