Skip to content

Commit faf83e3

Browse files
baurmattbaurmatt
committed
Add inodes alert
1 parent f2b5aab commit faf83e3

File tree

2 files changed

+246
-0
lines changed

2 files changed

+246
-0
lines changed

‎alerts/storage_alerts.libsonnet‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,56 @@
6666
summary: 'PersistentVolume is filling up.',
6767
},
6868
},
69+
{
70+
alert: 'KubePersistentVolumeInodesFillingUp',
71+
expr: |||
72+
(
73+
kubelet_volume_stats_inodes_free{%(prefixedNamespaceSelector)s%(kubeletSelector)s}
74+
/
75+
kubelet_volume_stats_inodes{%(prefixedNamespaceSelector)s%(kubeletSelector)s}
76+
) < 0.03
77+
and
78+
kubelet_volume_stats_inodes_used{%(prefixedNamespaceSelector)s%(kubeletSelector)s} > 0
79+
unless on(namespace, persistentvolumeclaim)
80+
kube_persistentvolumeclaim_access_mode{%(prefixedNamespaceSelector)s access_mode="ReadOnlyMany"} == 1
81+
unless on(namespace, persistentvolumeclaim)
82+
kube_persistentvolumeclaim_labels{%(prefixedNamespaceSelector)s%(pvExcludedSelector)s} == 1
83+
||| % $._config,
84+
'for': '1m',
85+
labels: {
86+
severity: 'critical',
87+
},
88+
annotations: {
89+
description: 'The PersistentVolume claimed by {{ $labels.persistentvolumeclaim }} in Namespace {{ $labels.namespace }} only has {{ $value | humanizePercentage }} free inodes.',
90+
summary: 'PersistentVolumeInodes are filling up.',
91+
},
92+
},
93+
{
94+
alert: 'KubePersistentVolumeInodesFillingUp',
95+
expr: |||
96+
(
97+
kubelet_volume_stats_inodes_free{%(prefixedNamespaceSelector)s%(kubeletSelector)s}
98+
/
99+
kubelet_volume_stats_inodes{%(prefixedNamespaceSelector)s%(kubeletSelector)s}
100+
) < 0.15
101+
and
102+
kubelet_volume_stats_inodes_used{%(prefixedNamespaceSelector)s%(kubeletSelector)s} > 0
103+
and
104+
predict_linear(kubelet_volume_stats_inodes_free{%(prefixedNamespaceSelector)s%(kubeletSelector)s}[%(volumeFullPredictionSampleTime)s], 4 * 24 * 3600) < 0
105+
unless on(namespace, persistentvolumeclaim)
106+
kube_persistentvolumeclaim_access_mode{%(prefixedNamespaceSelector)s access_mode="ReadOnlyMany"} == 1
107+
unless on(namespace, persistentvolumeclaim)
108+
kube_persistentvolumeclaim_labels{%(prefixedNamespaceSelector)s%(pvExcludedSelector)s} == 1
109+
||| % $._config,
110+
'for': '1h',
111+
labels: {
112+
severity: 'warning',
113+
},
114+
annotations: {
115+
description: 'Based on recent sampling, the PersistentVolume claimed by {{ $labels.persistentvolumeclaim }} in Namespace {{ $labels.namespace }} is expected to run out of inodes within four days. Currently {{ $value | humanizePercentage }} of its inodes are free.',
116+
summary: 'PersistentVolumeInodes are filling up.',
117+
},
118+
},
69119
{
70120
alert: 'KubePersistentVolumeErrors',
71121
expr: |||

‎tests.yaml‎

Lines changed: 196 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ rule_files:
55
evaluation_interval: 1m
66

77
tests:
8+
# PersistentVolume disk space
89
- interval: 1m
910
input_series:
1011
- series: 'kubelet_volume_stats_available_bytes{job="kubelet",namespace="monitoring",persistentvolumeclaim="somepvc"}'
@@ -199,6 +200,201 @@ tests:
199200
- eval_time: 61m
200201
alertname: KubePersistentVolumeFillingUp
201202

203+
# PersistentVolume inodes
204+
- interval: 1m
205+
input_series:
206+
- series: 'kubelet_volume_stats_inodes_free{job="kubelet",namespace="monitoring",persistentvolumeclaim="somepvc"}'
207+
values: '1024 512 64 16'
208+
- series: 'kubelet_volume_stats_inodes{job="kubelet",namespace="monitoring",persistentvolumeclaim="somepvc"}'
209+
values: '1024 1024 1024 1024'
210+
- series: 'kubelet_volume_stats_inodes_used{job="kubelet",namespace="monitoring",persistentvolumeclaim="somepvc"}'
211+
values: '16 64 512 1024'
212+
- series: 'kube_persistentvolumeclaim_access_mode{job="ksm",namespace="monitoring",persistentvolumeclaim="somepvc", access_mode="ReadWriteOnce", service="kube-state-metrics"}'
213+
values: '1 1 1 1'
214+
alert_rule_test:
215+
- eval_time: 1m
216+
alertname: KubePersistentVolumeInodesFillingUp
217+
- eval_time: 2m
218+
alertname: KubePersistentVolumeInodesFillingUp
219+
- eval_time: 3m
220+
alertname: KubePersistentVolumeInodesFillingUp
221+
- eval_time: 4m
222+
alertname: KubePersistentVolumeInodesFillingUp
223+
exp_alerts:
224+
- exp_labels:
225+
job: kubelet
226+
namespace: monitoring
227+
persistentvolumeclaim: somepvc
228+
severity: critical
229+
exp_annotations:
230+
summary: "PersistentVolumeInodes are filling up."
231+
description: 'The PersistentVolume claimed by somepvc in Namespace monitoring only has 1.562% free inodes.'
232+
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepersistentvolumeinodesfillingup
233+
234+
# Don't alert when PVC access_mode is ReadOnlyMany
235+
- interval: 1m
236+
input_series:
237+
- series: 'kubelet_volume_stats_inodes_free{job="kubelet",namespace="monitoring",persistentvolumeclaim="somepvc"}'
238+
values: '1024 512 64 16'
239+
- series: 'kubelet_volume_stats_inodes{job="kubelet",namespace="monitoring",persistentvolumeclaim="somepvc"}'
240+
values: '1024 1024 1024 1024'
241+
- series: 'kubelet_volume_stats_inodes_used{job="kubelet",namespace="monitoring",persistentvolumeclaim="somepvc"}'
242+
values: '16 64 512 1024'
243+
- series: 'kube_persistentvolumeclaim_access_mode{job="ksm",namespace="monitoring",persistentvolumeclaim="somepvc", access_mode="ReadOnlyMany", service="kube-state-metrics"}'
244+
values: '1 1 1 1'
245+
alert_rule_test:
246+
- eval_time: 1m
247+
alertname: KubePersistentVolumeInodesFillingUp
248+
- eval_time: 2m
249+
alertname: KubePersistentVolumeInodesFillingUp
250+
- eval_time: 3m
251+
alertname: KubePersistentVolumeInodesFillingUp
252+
- eval_time: 4m
253+
alertname: KubePersistentVolumeInodesFillingUp
254+
255+
# Block volume mounts can report 0 for the kubelet_volume_stats_inodes_used metric but it shouldn't trigger the KubePersistentVolumeInodesFillingUp alert.
256+
# See https://github.com/kubernetes/kubernetes/commit/b997e0e4d6ccbead435a47d6ac75b0db3d17252f for details.
257+
- interval: 1m
258+
input_series:
259+
- series: 'kubelet_volume_stats_inodes_free{job="kubelet",namespace="monitoring",persistentvolumeclaim="somepvc"}'
260+
values: '1024 512 64 16'
261+
- series: 'kubelet_volume_stats_inodes{job="kubelet",namespace="monitoring",persistentvolumeclaim="somepvc"}'
262+
values: '1024 1024 1024 1024'
263+
- series: 'kubelet_volume_stats_inodes_used{job="kubelet",namespace="monitoring",persistentvolumeclaim="somepvc"}'
264+
values: '0 0 0 0'
265+
alert_rule_test:
266+
- eval_time: 1m
267+
alertname: KubePersistentVolumeInodesFillingUp
268+
- eval_time: 2m
269+
alertname: KubePersistentVolumeInodesFillingUp
270+
- eval_time: 3m
271+
alertname: KubePersistentVolumeInodesFillingUp
272+
- eval_time: 4m
273+
alertname: KubePersistentVolumeInodesFillingUp
274+
275+
# Don't alert when PVC has been labelled as fully utilised
276+
- interval: 1m
277+
input_series:
278+
- series: 'kubelet_volume_stats_inodes_free{job="kubelet",namespace="monitoring",persistentvolumeclaim="somepvc"}'
279+
values: '1024 512 64 16'
280+
- series: 'kubelet_volume_stats_inodes{job="kubelet",namespace="monitoring",persistentvolumeclaim="somepvc"}'
281+
values: '1024 1024 1024 1024'
282+
- series: 'kubelet_volume_stats_inodes_used{job="kubelet",namespace="monitoring",persistentvolumeclaim="somepvc"}'
283+
values: '16 64 512 1024'
284+
- series: 'kube_persistentvolumeclaim_access_mode{job="ksm",namespace="monitoring",persistentvolumeclaim="somepvc", access_mode="ReadWriteOnce", service="kube-state-metrics"}'
285+
values: '1 1 1 1'
286+
- series: 'kube_persistentvolumeclaim_labels{job="kubelet",namespace="monitoring",persistentvolumeclaim="somepvc",label_excluded_from_alerts="true"}'
287+
values: '1 1 1 1'
288+
alert_rule_test:
289+
- eval_time: 1m
290+
alertname: KubePersistentVolumeInodesFillingUp
291+
- eval_time: 2m
292+
alertname: KubePersistentVolumeInodesFillingUp
293+
- eval_time: 3m
294+
alertname: KubePersistentVolumeInodesFillingUp
295+
- eval_time: 4m
296+
alertname: KubePersistentVolumeInodesFillingUp
297+
298+
- interval: 1m
299+
input_series:
300+
- series: 'kubelet_volume_stats_inodes_free{job="kubelet",namespace="monitoring",persistentvolumeclaim="somepvc"}'
301+
values: '1024-10x61'
302+
- series: 'kubelet_volume_stats_inodes{job="kubelet",namespace="monitoring",persistentvolumeclaim="somepvc"}'
303+
values: '32768+0x61'
304+
- series: 'kubelet_volume_stats_inodes_used{job="kubelet",namespace="monitoring",persistentvolumeclaim="somepvc"}'
305+
values: '1024+10x61'
306+
alert_rule_test:
307+
- eval_time: 1h
308+
alertname: KubePersistentVolumeInodesFillingUp
309+
exp_alerts:
310+
- exp_labels:
311+
job: kubelet
312+
namespace: monitoring
313+
persistentvolumeclaim: somepvc
314+
severity: critical
315+
exp_annotations:
316+
summary: "PersistentVolumeInodes are filling up."
317+
description: 'The PersistentVolume claimed by somepvc in Namespace monitoring only has 1.294% free inodes.'
318+
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepersistentvolumeinodesfillingup
319+
320+
- interval: 1m
321+
input_series:
322+
- series: 'kubelet_volume_stats_inodes_free{job="kubelet",namespace="monitoring",persistentvolumeclaim="somepvc"}'
323+
values: '1024-10x61'
324+
- series: 'kubelet_volume_stats_inodes{job="kubelet",namespace="monitoring",persistentvolumeclaim="somepvc"}'
325+
values: '32768+0x61'
326+
- series: 'kubelet_volume_stats_inodes_used{job="kubelet",namespace="monitoring",persistentvolumeclaim="somepvc"}'
327+
values: '1024+10x61'
328+
- series: 'kube_persistentvolumeclaim_access_mode{job="ksm",namespace="monitoring",persistentvolumeclaim="somepvc", access_mode="ReadWriteOnce", service="kube-state-metrics"}'
329+
values: '1x61'
330+
alert_rule_test:
331+
- eval_time: 61m
332+
alertname: KubePersistentVolumeInodesFillingUp
333+
exp_alerts:
334+
- exp_labels:
335+
job: kubelet
336+
namespace: monitoring
337+
persistentvolumeclaim: somepvc
338+
severity: warning
339+
exp_annotations:
340+
summary: "PersistentVolumeInodes are filling up."
341+
description: 'Based on recent sampling, the PersistentVolume claimed by somepvc in Namespace monitoring is expected to run out of inodes within four days. Currently 1.263% of its inodes are free.'
342+
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepersistentvolumeinodesfillingup
343+
- exp_labels:
344+
job: kubelet
345+
namespace: monitoring
346+
persistentvolumeclaim: somepvc
347+
severity: critical
348+
exp_annotations:
349+
summary: "PersistentVolumeInodes are filling up."
350+
description: 'The PersistentVolume claimed by somepvc in Namespace monitoring only has 1.263% free inodes.'
351+
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepersistentvolumeinodesfillingup
352+
353+
# Block volume mounts can report 0 for the kubelet_volume_stats_inodes_used metric but it shouldn't trigger the KubePersistentVolumeInodesFillingUp alert.
354+
# See https://github.com/kubernetes/kubernetes/commit/b997e0e4d6ccbead435a47d6ac75b0db3d17252f for details.
355+
- interval: 1m
356+
input_series:
357+
- series: 'kubelet_volume_stats_inodes_free{job="kubelet",namespace="monitoring",persistentvolumeclaim="somepvc"}'
358+
values: '1024-10x61'
359+
- series: 'kubelet_volume_stats_inodes{job="kubelet",namespace="monitoring",persistentvolumeclaim="somepvc"}'
360+
values: '32768+0x61'
361+
- series: 'kubelet_volume_stats_inodes_used{job="kubelet",namespace="monitoring",persistentvolumeclaim="somepvc"}'
362+
values: '0x61'
363+
alert_rule_test:
364+
- eval_time: 61m
365+
alertname: KubePersistentVolumeInodesFillingUp
366+
367+
# Don't alert when PVC access_mode is ReadOnlyMany
368+
- interval: 1m
369+
input_series:
370+
- series: 'kubelet_volume_stats_inodes_free{job="kubelet",namespace="monitoring",persistentvolumeclaim="somepvc"}'
371+
values: '1024-10x61'
372+
- series: 'kubelet_volume_stats_inodes{job="kubelet",namespace="monitoring",persistentvolumeclaim="somepvc"}'
373+
values: '32768+0x61'
374+
- series: 'kubelet_volume_stats_used_bytes{job="kubelet",namespace="monitoring",persistentvolumeclaim="somepvc"}'
375+
values: '1x61'
376+
- series: 'kube_persistentvolumeclaim_access_mode{job="ksm",namespace="monitoring",persistentvolumeclaim="somepvc", access_mode="ReadOnlyMany", service="kube-state-metrics"}'
377+
values: '1x61'
378+
alert_rule_test:
379+
- eval_time: 61m
380+
alertname: KubePersistentVolumeInodesFillingUp
381+
382+
- interval: 1m
383+
input_series:
384+
- series: 'kubelet_volume_stats_inodes_free{job="kubelet",namespace="monitoring",persistentvolumeclaim="somepvc"}'
385+
values: '1024-10x61'
386+
- series: 'kubelet_volume_stats_capacity_bytes{job="kubelet",namespace="monitoring",persistentvolumeclaim="somepvc"}'
387+
values: '32768+0x61'
388+
- series: 'kubelet_volume_stats_used_bytes{job="kubelet",namespace="monitoring",persistentvolumeclaim="somepvc"}'
389+
values: '1024+10x61'
390+
- series: 'kube_persistentvolumeclaim_access_mode{job="ksm",namespace="monitoring",persistentvolumeclaim="somepvc", access_mode="ReadWriteOnce", service="kube-state-metrics"}'
391+
values: '1x61'
392+
- series: 'kube_persistentvolumeclaim_labels{job="ksm",namespace="monitoring",persistentvolumeclaim="somepvc",label_excluded_from_alerts="true"}'
393+
values: '1x61'
394+
alert_rule_test:
395+
- eval_time: 61m
396+
alertname: KubePersistentVolumeInodesFillingUp
397+
202398
- interval: 1m
203399
input_series:
204400
- series: 'kube_node_status_capacity{resource="pods",instance="172.17.0.5:8443",node="minikube",job="kube-state-metrics", namespace="kube-system"}'

0 commit comments

Comments
 (0)