Merge pull request kubernetes#1938 from ryanrolds/rolds/pod_ready_time

Add metrics of kube_pod_status_ready_time and kube_pod_status_containers_ready_time redux

Author: k8s-ci-robot

docs/pod-metrics.md

@@ -24,6 +24,8 @@
 | kube_pod_container_status_last_terminated_reason | Gauge | Describes the last reason the container was in terminated state       | |`container`=&lt;container-name&gt; <br> `pod`=&lt;pod-name&gt; <br> `namespace`=&lt;pod-namespace&gt; <br> `reason`=&lt;last-terminated-reason&gt; <br> `uid`=&lt;pod-uid&gt; | EXPERIMENTAL | - |
 | kube_pod_container_status_last_terminated_exitcode | Gauge | Describes the exit code for the last container in terminated state.   |  | `container`=&lt;container-name&gt; <br> `pod`=&lt;pod-name&gt; <br> `namespace`=&lt;pod-namespace&gt; <br> `uid`=&lt;pod-uid&gt; | EXPERIMENTAL | - |
 | kube_pod_container_status_ready | Gauge | Describes whether the containers readiness check succeeded            | |`container`=&lt;container-name&gt; <br> `pod`=&lt;pod-name&gt; <br> `namespace`=&lt;pod-namespace&gt; <br> `uid`=&lt;pod-uid&gt; | STABLE | - |
+| kube_pod_status_ready_time | Gauge | Time when pod passed readiness probes. | seconds | `pod`=&lt;pod-name&gt; <br> `namespace`=&lt;pod-namespace&gt; <br> `uid`=&lt;pod-uid&gt; | EXPERIMENTAL |
+| kube_pod_status_container_ready_time | Gauge | Time when the container of the pod entered Ready state. | seconds | `pod`=&lt;pod-name&gt; <br> `namespace`=&lt;pod-namespace&gt; <br> `uid`=&lt;pod-uid&gt; | EXPERIMENTAL |
 | kube_pod_container_status_restarts_total | Counter | The number of container restarts per container                        | | `container`=&lt;container-name&gt; <br> `namespace`=&lt;pod-namespace&gt; <br> `pod`=&lt;pod-name&gt; <br> `uid`=&lt;pod-uid&gt; | STABLE | - |
 | kube_pod_container_resource_requests | Gauge | The number of requested request resource by a container. It is recommended to use the `kube_pod_resource_requests` metric exposed by kube-scheduler instead, as it is more precise.               | `cpu`=&lt;core&gt; <br> `memory`=&lt;bytes&gt; |`resource`=&lt;resource-name&gt; <br> `unit`=&lt;resource-unit&gt; <br> `container`=&lt;container-name&gt; <br> `pod`=&lt;pod-name&gt; <br> `namespace`=&lt;pod-namespace&gt; <br> `node`=&lt; node-name&gt; <br> `uid`=&lt;pod-uid&gt; | EXPERIMENTAL | - |
 | kube_pod_container_resource_limits | Gauge | The number of requested limit resource by a container. It is recommended to use the `kube_pod_resource_limits` metric exposed by kube-scheduler instead, as it is more precise.           | `cpu`=&lt;core&gt; <br> `memory`=&lt;bytes&gt; |`resource`=&lt;resource-name&gt; <br> `unit`=&lt;resource-unit&gt; <br> `container`=&lt;container-name&gt; <br> `pod`=&lt;pod-name&gt; <br> `namespace`=&lt;pod-namespace&gt; <br> `node`=&lt; node-name&gt; <br> `uid`=&lt;pod-uid&gt; | EXPERIMENTAL | - |

internal/store/pod.go

@@ -84,6 +84,8 @@ func podMetricFamilies(allowAnnotationsList, allowLabelsList []string) []generat
 		createPodStatusPhaseFamilyGenerator(),
 		createPodStatusQosClassFamilyGenerator(),
 		createPodStatusReadyFamilyGenerator(),
+		createPodStatusReadyTimeFamilyGenerator(),
+		createPodStatusContainerReadyTimeFamilyGenerator(),
 		createPodStatusReasonFamilyGenerator(),
 		createPodStatusScheduledFamilyGenerator(),
 		createPodStatusScheduledTimeFamilyGenerator(),
@@ -1318,6 +1320,60 @@ func createPodStatusPhaseFamilyGenerator() generator.FamilyGenerator {
 	)
 }
 
+func createPodStatusContainerReadyTimeFamilyGenerator() generator.FamilyGenerator {
+	return *generator.NewFamilyGeneratorWithStability(
+		"kube_pod_status_container_ready_time",
+		"Readiness achieved time in unix timestamp for a pod containers.",
+		metric.Gauge,
+		basemetrics.ALPHA,
+		"",
+		wrapPodFunc(func(p *v1.Pod) *metric.Family {
+			ms := []*metric.Metric{}
+
+			for _, c := range p.Status.Conditions {
+				if c.Type == v1.ContainersReady {
+					ms = append(ms, &metric.Metric{
+						LabelKeys:   []string{},
+						LabelValues: []string{},
+						Value:       float64((c.LastTransitionTime).Unix()),
+					})
+				}
+			}
+
+			return &metric.Family{
+				Metrics: ms,
+			}
+		}),
+	)
+}
+
+func createPodStatusReadyTimeFamilyGenerator() generator.FamilyGenerator {
+	return *generator.NewFamilyGeneratorWithStability(
+		"kube_pod_status_ready_time",
+		"Readiness achieved time in unix timestamp for a pod.",
+		metric.Gauge,
+		basemetrics.ALPHA,
+		"",
+		wrapPodFunc(func(p *v1.Pod) *metric.Family {
+			ms := []*metric.Metric{}
+
+			for _, c := range p.Status.Conditions {
+				if c.Type == v1.PodReady {
+					ms = append(ms, &metric.Metric{
+						LabelKeys:   []string{},
+						LabelValues: []string{},
+						Value:       float64((c.LastTransitionTime).Unix()),
+					})
+				}
+			}
+
+			return &metric.Family{
+				Metrics: ms,
+			}
+		}),
+	)
+}
+
 func createPodStatusQosClassFamilyGenerator() generator.FamilyGenerator {
 	return *generator.NewFamilyGeneratorWithStability(
 		"kube_pod_status_qos_class",

internal/store/pod_test.go

@@ -1434,6 +1434,32 @@ func TestPodStore(t *testing.T) {
 `,
 			MetricNames: []string{"kube_pod_status_reason"},
 		},
+		{
+			Obj: &v1.Pod{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "pod1",
+					Namespace: "ns1",
+					UID:       "uid1",
+				},
+				Status: v1.PodStatus{
+					Conditions: []v1.PodCondition{
+						{
+							Type:   v1.ContainersReady,
+							Status: v1.ConditionTrue,
+							LastTransitionTime: metav1.Time{
+								Time: time.Unix(1501666018, 0),
+							},
+						},
+					},
+				},
+			},
+			Want: `
+				# HELP kube_pod_status_container_ready_time Readiness achieved time in unix timestamp for a pod containers.
+				# TYPE kube_pod_status_container_ready_time gauge
+				kube_pod_status_container_ready_time{namespace="ns1",pod="pod1",uid="uid1"} 1.501666018e+09
+			`,
+			MetricNames: []string{"kube_pod_status_container_ready_time"},
+		},
 		{
 			Obj: &v1.Pod{
 				ObjectMeta: metav1.ObjectMeta{
@@ -1446,18 +1472,24 @@ func TestPodStore(t *testing.T) {
 						{
 							Type:   v1.PodReady,
 							Status: v1.ConditionTrue,
+							LastTransitionTime: metav1.Time{
+								Time: time.Unix(1501666018, 0),
+							},
 						},
 					},
 				},
 			},
 			Want: `
 				# HELP kube_pod_status_ready [STABLE] Describes whether the pod is ready to serve requests.
+				# HELP kube_pod_status_ready_time Readiness achieved time in unix timestamp for a pod.
 				# TYPE kube_pod_status_ready gauge
+				# TYPE kube_pod_status_ready_time gauge
+				kube_pod_status_ready_time{namespace="ns1",pod="pod1",uid="uid1"} 1.501666018e+09
 				kube_pod_status_ready{condition="false",namespace="ns1",pod="pod1",uid="uid1"} 0
 				kube_pod_status_ready{condition="true",namespace="ns1",pod="pod1",uid="uid1"} 1
 				kube_pod_status_ready{condition="unknown",namespace="ns1",pod="pod1",uid="uid1"} 0
 			`,
-			MetricNames: []string{"kube_pod_status_ready"},
+			MetricNames: []string{"kube_pod_status_ready_time", "kube_pod_status_ready"},
 		},
 		{
 			Obj: &v1.Pod{
@@ -1471,18 +1503,24 @@ func TestPodStore(t *testing.T) {
 						{
 							Type:   v1.PodReady,
 							Status: v1.ConditionFalse,
+							LastTransitionTime: metav1.Time{
+								Time: time.Unix(1501666018, 0),
+							},
 						},
 					},
 				},
 			},
 			Want: `
 				# HELP kube_pod_status_ready [STABLE] Describes whether the pod is ready to serve requests.
+				# HELP kube_pod_status_ready_time Readiness achieved time in unix timestamp for a pod.
 				# TYPE kube_pod_status_ready gauge
+				# TYPE kube_pod_status_ready_time gauge
+				kube_pod_status_ready_time{namespace="ns2",pod="pod2",uid="uid2"} 1.501666018e+09
 				kube_pod_status_ready{condition="false",namespace="ns2",pod="pod2",uid="uid2"} 1
 				kube_pod_status_ready{condition="true",namespace="ns2",pod="pod2",uid="uid2"} 0
 				kube_pod_status_ready{condition="unknown",namespace="ns2",pod="pod2",uid="uid2"} 0
 			`,
-			MetricNames: []string{"kube_pod_status_ready"},
+			MetricNames: []string{"kube_pod_status_ready_time", "kube_pod_status_ready"},
 		},
 		{
 			Obj: &v1.Pod{
@@ -2099,7 +2137,7 @@ func BenchmarkPodStore(b *testing.B) {
 		},
 	}
 
-	expectedFamilies := 48
+	expectedFamilies := 50
 	for n := 0; n < b.N; n++ {
 		families := f(pod)
 		if len(families) != expectedFamilies {

pkg/app/server_test.go

@@ -238,8 +238,10 @@ func TestFullScrapeCycle(t *testing.T) {
 # HELP kube_pod_spec_volumes_persistentvolumeclaims_info [STABLE] Information about persistentvolumeclaim volumes in a pod.
 # HELP kube_pod_spec_volumes_persistentvolumeclaims_readonly [STABLE] Describes whether a persistentvolumeclaim is mounted read only.
 # HELP kube_pod_start_time [STABLE] Start time in unix timestamp for a pod.
+# HELP kube_pod_status_container_ready_time Readiness achieved time in unix timestamp for a pod containers.
 # HELP kube_pod_status_qos_class The pods current qosClass.
 # HELP kube_pod_status_phase [STABLE] The pods current phase.
+# HELP kube_pod_status_ready_time Readiness achieved time in unix timestamp for a pod.
 # HELP kube_pod_status_ready [STABLE] Describes whether the pod is ready to serve requests.
 # HELP kube_pod_status_reason The pod status reasons
 # HELP kube_pod_status_scheduled [STABLE] Describes the status of the scheduling process for the pod.
@@ -285,9 +287,11 @@ func TestFullScrapeCycle(t *testing.T) {
 # TYPE kube_pod_spec_volumes_persistentvolumeclaims_info gauge
 # TYPE kube_pod_spec_volumes_persistentvolumeclaims_readonly gauge
 # TYPE kube_pod_start_time gauge
+# TYPE kube_pod_status_container_ready_time gauge
 # TYPE kube_pod_status_phase gauge
 # TYPE kube_pod_status_qos_class gauge
 # TYPE kube_pod_status_ready gauge
+# TYPE kube_pod_status_ready_time gauge
 # TYPE kube_pod_status_reason gauge
 # TYPE kube_pod_status_scheduled gauge
 # TYPE kube_pod_status_scheduled_time gauge