Merge pull request #754 from phillebaba/more/multicluster

Fix multicluster labels

Author: metalmatze

alerts/kubelet.libsonnet

@@ -52,11 +52,11 @@
             // Some node has a capacity of 1 like AWS's Fargate and only exists while a pod is running on it.
             // We have to ignore this special node in the KubeletTooManyPods alert.
             expr: |||
-              count by(node) (
-                (kube_pod_status_phase{%(kubeStateMetricsSelector)s,phase="Running"} == 1) * on(instance,pod,namespace,cluster) group_left(node) topk by(instance,pod,namespace,cluster) (1, kube_pod_info{%(kubeStateMetricsSelector)s})
+              count by(%(clusterLabel)s, node) (
+                (kube_pod_status_phase{%(kubeStateMetricsSelector)s,phase="Running"} == 1) * on(instance,pod,namespace,%(clusterLabel)s) group_left(node) topk by(instance,pod,namespace,%(clusterLabel)s) (1, kube_pod_info{%(kubeStateMetricsSelector)s})
               )
               /
-              max by(node) (
+              max by(%(clusterLabel)s, node) (
                 kube_node_status_capacity{%(kubeStateMetricsSelector)s,resource="pods"} != 1
               ) > 0.95
             ||| % $._config,
@@ -72,7 +72,7 @@
           {
             alert: 'KubeNodeReadinessFlapping',
             expr: |||
-              sum(changes(kube_node_status_condition{status="true",condition="Ready"}[15m])) by (node) > 2
+              sum(changes(kube_node_status_condition{status="true",condition="Ready"}[15m])) by (%(clusterLabel)s, node) > 2
             ||| % $._config,
             'for': '15m',
             labels: {

alerts/system_alerts.libsonnet

@@ -28,11 +28,11 @@
             // this is normal and an expected error, therefore it should be
             // ignored in this alert.
             expr: |||
-              (sum(rate(rest_client_requests_total{code=~"5.."}[5m])) by (instance, job, namespace)
+              (sum(rate(rest_client_requests_total{code=~"5.."}[5m])) by (%(clusterLabel)s, instance, job, namespace)
                 /
-              sum(rate(rest_client_requests_total[5m])) by (instance, job, namespace))
+              sum(rate(rest_client_requests_total[5m])) by (%(clusterLabel)s, instance, job, namespace))
               > 0.01
-            |||,
+            ||| % $._config,
             'for': '15m',
             labels: {
               severity: 'warning',

rules/node.libsonnet

@@ -20,8 +20,8 @@
             // below for instance.
             record: 'node_namespace_pod:kube_pod_info:',
             expr: |||
-              topk by(namespace, %(podLabel)s) (1,
-                max by (node, namespace, %(podLabel)s) (
+              topk by(%(clusterLabel)s, namespace, %(podLabel)s) (1,
+                max by (%(clusterLabel)s, node, namespace, %(podLabel)s) (
                   label_replace(kube_pod_info{%(kubeStateMetricsSelector)s,node!=""}, "%(podLabel)s", "$1", "pod", "(.*)")
               ))
             ||| % $._config,