From e8312f27fee18af464186ef9aa9647296b44a0f3 Mon Sep 17 00:00:00 2001 From: Michael Act <86778470+michaelact@users.noreply.github.com> Date: Tue, 23 Sep 2025 11:26:42 +0700 Subject: [PATCH 1/2] chore: storage usage alert Signed-off-by: Michael Act <86778470+michaelact@users.noreply.github.com> --- .../rules-1.14/kubernetes-storage.yaml | 30 ++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kubernetes-storage.yaml b/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kubernetes-storage.yaml index eb3b7fcfd45c..354531d6b6e6 100644 --- a/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kubernetes-storage.yaml +++ b/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kubernetes-storage.yaml @@ -214,4 +214,32 @@ spec: {{- end }} {{- end }} {{- end }} -{{- end }} \ No newline at end of file +{{- if not (.Values.defaultRules.disabled.KubernetesStorageUsage85Percent | default false) }} + - alert: KubernetesStorageUsage85Percent + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesStorage }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesStorage | indent 8 }} +{{- end }} + description: Storage usage of PVC {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.persistentvolumeclaim {{`}}`}} has exceeded 85% for the last 5 minutes. + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodefilesystemspacefillingup/#:~:text=A%20filesystem%20running%20full%20is%20very%20bad%20for%20any%20process%20in%20need%20to%20write%20to%20the%20filesystem.%20But%20even%20before%20a%20filesystem%20runs%20full%2C%20performance%20is%20usually%20degrading. + summary: High storage usage detected (PVC {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.persistentvolumeclaim {{`}}`}}) + expr: kubelet_volume_stats_used_bytes{job="kubelet", namespace{{ $namespaceOperator }}"{{ $targetNamespace }}", metrics_path="/metrics"} / kubelet_volume_stats_capacity_bytes{job="kubelet", namespace{{ $namespaceOperator }}"{{ $targetNamespace }}", metrics_path="/metrics"} * 100 > 85 + for: {{ dig "KubernetesStorageUsage85Percent" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubernetesStorageUsage85Percent" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesStorage }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesStorage }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} From 62c5dfc4facaa5d99cef38c8d48776ce4a7d8d4e Mon Sep 17 00:00:00 2001 From: Michael Act <86778470+michaelact@users.noreply.github.com> Date: Tue, 23 Sep 2025 11:30:48 +0700 Subject: [PATCH 2/2] chore: bump chart version Signed-off-by: Michael Act <86778470+michaelact@users.noreply.github.com> --- charts/kube-prometheus-stack/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/kube-prometheus-stack/Chart.yaml b/charts/kube-prometheus-stack/Chart.yaml index d5044a9594b0..687d5632c30b 100644 --- a/charts/kube-prometheus-stack/Chart.yaml +++ b/charts/kube-prometheus-stack/Chart.yaml @@ -31,7 +31,7 @@ name: kube-prometheus-stack sources: - https://github.com/prometheus-community/helm-charts - https://github.com/prometheus-operator/kube-prometheus -version: 77.10.0 +version: 77.11.0 # renovate: github=prometheus-operator/prometheus-operator appVersion: v0.85.0 kubeVersion: ">=1.25.0-0"