feat: add kube_pod_container_status_last_terminated_timestamp metric

Signed-off-by: Tetiana Kravchenko <[email protected]>

Author: tetianakravchenko

docs/pod-metrics.md

@@ -23,6 +23,7 @@
 | kube_pod_container_status_terminated_reason           | Gauge       | Describes the reason the container is currently in terminated state                                                                                                                 |                                                | `container`=&lt;container-name&gt; <br> `pod`=&lt;pod-name&gt; <br> `namespace`=&lt;pod-namespace&gt; <br> `reason`=&lt;container-terminated-reason&gt; <br> `uid`=&lt;pod-uid&gt;                                                                                                                                                                                | EXPERIMENTAL | -      |
 | kube_pod_container_status_last_terminated_reason      | Gauge       | Describes the last reason the container was in terminated state                                                                                                                     |                                                | `container`=&lt;container-name&gt; <br> `pod`=&lt;pod-name&gt; <br> `namespace`=&lt;pod-namespace&gt; <br> `reason`=&lt;last-terminated-reason&gt; <br> `uid`=&lt;pod-uid&gt;                                                                                                                                                                                     | EXPERIMENTAL | -      |
 | kube_pod_container_status_last_terminated_exitcode    | Gauge       | Describes the exit code for the last container in terminated state.                                                                                                                 |                                                | `container`=&lt;container-name&gt; <br> `pod`=&lt;pod-name&gt; <br> `namespace`=&lt;pod-namespace&gt; <br> `uid`=&lt;pod-uid&gt;                                                                                                                                                                                                                                  | EXPERIMENTAL | -      |
+| kube_pod_container_status_last_terminated_timestamp   | Gauge       | Last terminated time for a pod container, when container was terminated with kube_pod_container_status_last_terminated_reason, in unix timestamp.                                                                                                             |                                                | `container`=&lt;container-name&gt; <br> `pod`=&lt;pod-name&gt; <br> `namespace`=&lt;pod-namespace&gt; <br> `uid`=&lt;pod-uid&gt;                                                                                                                                                                                                                                  | EXPERIMENTAL | -      |
 | kube_pod_container_status_ready                       | Gauge       | Describes whether the containers readiness check succeeded                                                                                                                          |                                                | `container`=&lt;container-name&gt; <br> `pod`=&lt;pod-name&gt; <br> `namespace`=&lt;pod-namespace&gt; <br> `uid`=&lt;pod-uid&gt;                                                                                                                                                                                                                                  | STABLE       | -      |
 | kube_pod_status_initialized_time                      | Gauge       | Time when the pod is initialized.                                                                                                                                                   | seconds                                        | `pod`=&lt;pod-name&gt; <br> `namespace`=&lt;pod-namespace&gt; <br> `uid`=&lt;pod-uid&gt;                                                                                                                                                                                                                                                                          | EXPERIMENTAL |
 | kube_pod_status_ready_time                            | Gauge       | Time when pod passed readiness probes.                                                                                                                                              | seconds                                        | `pod`=&lt;pod-name&gt; <br> `namespace`=&lt;pod-namespace&gt; <br> `uid`=&lt;pod-uid&gt;                                                                                                                                                                                                                                                                          | EXPERIMENTAL |

internal/store/pod.go

@@ -49,6 +49,7 @@ func podMetricFamilies(allowAnnotationsList, allowLabelsList []string) []generat
 		createPodContainerStateStartedFamilyGenerator(),
 		createPodContainerStatusLastTerminatedReasonFamilyGenerator(),
 		createPodContainerStatusLastTerminatedExitCodeFamilyGenerator(),
+		createPodContainerStatusLastTerminatedTimestampFamilyGenerator(),
 		createPodContainerStatusReadyFamilyGenerator(),
 		createPodContainerStatusRestartsTotalFamilyGenerator(),
 		createPodContainerStatusRunningFamilyGenerator(),
@@ -375,6 +376,32 @@ func createPodContainerStatusLastTerminatedExitCodeFamilyGenerator() generator.F
 	)
 }
 
+func createPodContainerStatusLastTerminatedTimestampFamilyGenerator() generator.FamilyGenerator {
+	return *generator.NewFamilyGeneratorWithStability(
+		"kube_pod_container_status_last_terminated_timestamp",
+		"Last terminated time for a pod container, when container was terminated with kube_pod_container_status_last_terminated_reason, in unix timestamp.",
+		metric.Gauge,
+		basemetrics.ALPHA,
+		"",
+		wrapPodFunc(func(p *v1.Pod) *metric.Family {
+			ms := make([]*metric.Metric, 0, len(p.Status.ContainerStatuses))
+			for _, cs := range p.Status.ContainerStatuses {
+				if cs.LastTerminationState.Terminated != nil {
+					ms = append(ms, &metric.Metric{
+						LabelKeys:   []string{"container"},
+						LabelValues: []string{cs.Name},
+						Value:       float64(cs.LastTerminationState.Terminated.FinishedAt.Unix()),
+					})
+				}
+			}
+
+			return &metric.Family{
+				Metrics: ms,
+			}
+		}),
+	)
+}
+
 func createPodContainerStatusReadyFamilyGenerator() generator.FamilyGenerator {
 	return *generator.NewFamilyGeneratorWithStability(
 		"kube_pod_container_status_ready",

internal/store/pod_test.go

@@ -720,6 +720,9 @@ func TestPodStore(t *testing.T) {
 								Terminated: &v1.ContainerStateTerminated{
 									Reason:   "OOMKilled",
 									ExitCode: 137,
+									FinishedAt: metav1.Time{
+										Time: time.Unix(1501779547, 0),
+									},
 								},
 							},
 						},
@@ -729,6 +732,7 @@ func TestPodStore(t *testing.T) {
 			Want: `
 				# HELP kube_pod_container_status_last_terminated_reason Describes the last reason the container was in terminated state.
 				# HELP kube_pod_container_status_last_terminated_exitcode Describes the exit code for the last container in terminated state.
+				# HELP kube_pod_container_status_last_terminated_timestamp Last terminated time for a pod container, when container was terminated with kube_pod_container_status_last_terminated_reason, in unix timestamp.
 				# HELP kube_pod_container_status_running [STABLE] Describes whether the container is currently in running state.
 				# HELP kube_pod_container_status_terminated [STABLE] Describes whether the container is currently in terminated state.
 				# HELP kube_pod_container_status_terminated_reason Describes the reason the container is currently in terminated state.
@@ -737,6 +741,7 @@ func TestPodStore(t *testing.T) {
 				# HELP kube_pod_container_state_started [STABLE] Start time in unix timestamp for a pod container.
 				# TYPE kube_pod_container_status_last_terminated_reason gauge
 				# TYPE kube_pod_container_status_last_terminated_exitcode gauge
+				# TYPE kube_pod_container_status_last_terminated_timestamp gauge
 				# TYPE kube_pod_container_status_running gauge
 				# TYPE kube_pod_container_status_terminated gauge
 				# TYPE kube_pod_container_status_terminated_reason gauge
@@ -749,10 +754,12 @@ func TestPodStore(t *testing.T) {
 				kube_pod_container_status_waiting{container="container7",namespace="ns6",pod="pod6",uid="uid6"} 0
 				kube_pod_container_status_last_terminated_reason{container="container7",namespace="ns6",pod="pod6",reason="OOMKilled",uid="uid6"} 1
 				kube_pod_container_status_last_terminated_exitcode{container="container7",namespace="ns6",pod="pod6",uid="uid6"} 137
+				kube_pod_container_status_last_terminated_timestamp{container="container7",namespace="ns6",pod="pod6",uid="uid6"} 1.501779547e+09
 			`,
 			MetricNames: []string{
 				"kube_pod_container_status_last_terminated_reason",
 				"kube_pod_container_status_last_terminated_exitcode",
+				"kube_pod_container_status_last_terminated_timestamp",
 				"kube_pod_container_status_running",
 				"kube_pod_container_state_started",
 				"kube_pod_container_status_terminated",