Add metric for pod tolerations (Ref: #1744)

Author: Julian van den Berkmortel

docs/pod-metrics.md

@@ -47,6 +47,7 @@
 | kube_pod_status_reason | Gauge | The pod status reasons | |`pod`=&lt;pod-name&gt; <br> `namespace`=&lt;pod-namespace&gt; <br> `reason`=&lt;Evicted\|NodeAffinity\|NodeLost\|Shutdown\|UnexpectedAdmissionError&gt; <br> `uid`=&lt;pod-uid&gt; | EXPERIMENTAL | - |
 | kube_pod_status_scheduled_time | Gauge | Unix timestamp when pod moved into scheduled status | seconds |`pod`=&lt;pod-name&gt; <br> `namespace`=&lt;pod-namespace&gt; <br> `uid`=&lt;pod-uid&gt; | STABLE | - |
 | kube_pod_status_unschedulable | Gauge | Describes the unschedulable status for the pod | |`pod`=&lt;pod-name&gt; <br> `namespace`=&lt;pod-namespace&gt; <br> `uid`=&lt;pod-uid&gt; | STABLE | - |
+| kube_pod_tolerations | Gauge | Information about the pod tolerations | | `pod`=&lt;pod-name&gt; <br> `namespace`=&lt;pod-namespace&gt; <br> `uid`=&lt;pod-uid&gt; <br> `key`=&lt;toleration-key&gt; <br> `operator`=&lt;toleration-operator&gt; <br> `value`=&lt;toleration-value&gt; <br> `effect`=&lt;toleration-effect&gt; `toleration_seconds`=&lt;toleration-seconds&gt; | EXPERIMENTAL | - |
 
 ## Useful metrics queries
 

internal/store/pod.go

@@ -85,6 +85,7 @@ func podMetricFamilies(allowAnnotationsList, allowLabelsList []string) []generat
 		createPodStatusScheduledFamilyGenerator(),
 		createPodStatusScheduledTimeFamilyGenerator(),
 		createPodStatusUnschedulableFamilyGenerator(),
+		createPodTolerationsFamilyGenerator(),
 		createPodNodeSelectorsFamilyGenerator(),
 	}
 }
@@ -1403,6 +1404,62 @@ func createPodStatusUnschedulableFamilyGenerator() generator.FamilyGenerator {
 	)
 }
 
+func createPodTolerationsFamilyGenerator() generator.FamilyGenerator {
+	return *generator.NewFamilyGenerator(
+		"kube_pod_tolerations",
+		"Information about the pod tolerations",
+		metric.Gauge,
+		"",
+		wrapPodFunc(func(p *v1.Pod) *metric.Family {
+			var ms []*metric.Metric
+
+			for _, t := range p.Spec.Tolerations {
+				var labelKeys []string
+				var labelValues []string
+
+				if t.Key != "" {
+					labelKeys = append(labelKeys, "key")
+					labelValues = append(labelValues, t.Key)
+				}
+
+				if t.Operator != "" {
+					labelKeys = append(labelKeys, "operator")
+					labelValues = append(labelValues, string(t.Operator))
+				}
+
+				if t.Value != "" {
+					labelKeys = append(labelKeys, "value")
+					labelValues = append(labelValues, t.Value)
+				}
+
+				if t.Effect != "" {
+					labelKeys = append(labelKeys, "effect")
+					labelValues = append(labelValues, string(t.Effect))
+				}
+
+				if t.TolerationSeconds != nil {
+					labelKeys = append(labelKeys, "toleration_seconds")
+					labelValues = append(labelValues, strconv.FormatInt(*t.TolerationSeconds, 10))
+				}
+
+				if len(labelKeys) == 0 {
+					continue
+				}
+
+				ms = append(ms, &metric.Metric{
+					LabelKeys:   labelKeys,
+					LabelValues: labelValues,
+					Value:       1,
+				})
+			}
+
+			return &metric.Family{
+				Metrics: ms,
+			}
+		}),
+	)
+}
+
 func createPodNodeSelectorsFamilyGenerator() generator.FamilyGenerator {
 	return *generator.NewOptInFamilyGenerator(
 		"kube_pod_nodeselectors",

internal/store/pod_test.go

@@ -1926,6 +1926,47 @@ func TestPodStore(t *testing.T) {
 				"kube_pod_annotations",
 			},
 		},
+		{
+			Obj: &v1.Pod{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "pod1",
+					Namespace: "ns1",
+					UID:       "uid1",
+				},
+				Spec: v1.PodSpec{
+					Tolerations: []v1.Toleration{
+						{
+							Key:      "key1",
+							Operator: v1.TolerationOpEqual,
+							Value:    "value1",
+							Effect:   v1.TaintEffectNoSchedule,
+						},
+						{
+							Key:      "key2",
+							Operator: v1.TolerationOpExists,
+						},
+						{
+							Key:      "key3",
+							Operator: v1.TolerationOpEqual,
+							Value:    "value3",
+						},
+						{
+							// an empty toleration to ensure that an empty toleration does not result in a metric
+						},
+					},
+				},
+			},
+			Want: `
+				# HELP kube_pod_tolerations Information about the pod tolerations
+				# TYPE kube_pod_tolerations gauge
+				kube_pod_tolerations{namespace="ns1",pod="pod1",uid="uid1",key="key1",operator="Equal",value="value1",effect="NoSchedule"} 1
+				kube_pod_tolerations{namespace="ns1",pod="pod1",uid="uid1",key="key2",operator="Exists"} 1
+				kube_pod_tolerations{namespace="ns1",pod="pod1",uid="uid1",key="key3",operator="Equal",value="value3"} 1
+			`,
+			MetricNames: []string{
+				"kube_pod_tolerations",
+			},
+		},
 	}
 
 	for i, c := range cases {
@@ -2018,7 +2059,7 @@ func BenchmarkPodStore(b *testing.B) {
 		},
 	}
 
-	expectedFamilies := 45
+	expectedFamilies := 46
 	for n := 0; n < b.N; n++ {
 		families := f(pod)
 		if len(families) != expectedFamilies {

pkg/app/server_test.go

@@ -234,6 +234,7 @@ func TestFullScrapeCycle(t *testing.T) {
 # HELP kube_pod_status_scheduled Describes the status of the scheduling process for the pod.
 # HELP kube_pod_status_scheduled_time Unix timestamp when pod moved into scheduled status
 # HELP kube_pod_status_unschedulable Describes the unschedulable status for the pod.
+# HELP kube_pod_tolerations Information about the pod tolerations
 # TYPE kube_pod_annotations gauge
 # TYPE kube_pod_completion_time gauge
 # TYPE kube_pod_container_info gauge
@@ -278,6 +279,7 @@ func TestFullScrapeCycle(t *testing.T) {
 # TYPE kube_pod_status_scheduled gauge
 # TYPE kube_pod_status_scheduled_time gauge
 # TYPE kube_pod_status_unschedulable gauge
+# TYPE kube_pod_tolerations gauge
 kube_pod_annotations{namespace="default",pod="pod0",uid="abc-0"} 1
 kube_pod_container_info{namespace="default",pod="pod0",uid="abc-0",container="pod1_con1",image_spec="k8s.gcr.io/hyperkube2_spec",image="k8s.gcr.io/hyperkube2",image_id="docker://sha256:bbb",container_id="docker://cd456"} 1
 kube_pod_container_info{namespace="default",pod="pod0",uid="abc-0",container="pod1_con2",image_spec="k8s.gcr.io/hyperkube3_spec",image="k8s.gcr.io/hyperkube3",image_id="docker://sha256:ccc",container_id="docker://ef789"} 1