Skip to content

Commit 20ef8a7

Browse files
k8s-ci-robotk8s-ci-robot
authored
Merge pull request #1752 from ssabo/master
add exit code
2 parents 7343894 + 5bfe006 commit 20ef8a7

File tree

4 files changed

+59
-8
lines changed

4 files changed

+59
-8
lines changed

‎docs/pod-metrics.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
| kube_pod_container_status_terminated | Gauge | Describes whether the container is currently in terminated state | |`container`=&lt;container-name&gt; <br> `pod`=&lt;pod-name&gt; <br> `namespace`=&lt;pod-namespace&gt; <br> `uid`=&lt;pod-uid&gt; | STABLE | - |
2222
| kube_pod_container_status_terminated_reason | Gauge | Describes the reason the container is currently in terminated state | |`container`=&lt;container-name&gt; <br> `pod`=&lt;pod-name&gt; <br> `namespace`=&lt;pod-namespace&gt; <br> `reason`=&lt;container-terminated-reason&gt; <br> `uid`=&lt;pod-uid&gt; | EXPERIMENTAL | - |
2323
| kube_pod_container_status_last_terminated_reason | Gauge | Describes the last reason the container was in terminated state | |`container`=&lt;container-name&gt; <br> `pod`=&lt;pod-name&gt; <br> `namespace`=&lt;pod-namespace&gt; <br> `reason`=&lt;last-terminated-reason&gt; <br> `uid`=&lt;pod-uid&gt; | EXPERIMENTAL | - |
24+
| kube_pod_container_status_last_terminated_exitcode | Gauge | Describes the exit code for the last container in terminated state. | | `container`=&lt;container-name&gt; <br> `pod`=&lt;pod-name&gt; <br> `namespace`=&lt;pod-namespace&gt; <br> `uid`=&lt;pod-uid&gt; | EXPERIMENTAL | - |
2425
| kube_pod_container_status_ready | Gauge | Describes whether the containers readiness check succeeded | |`container`=&lt;container-name&gt; <br> `pod`=&lt;pod-name&gt; <br> `namespace`=&lt;pod-namespace&gt; <br> `uid`=&lt;pod-uid&gt; | STABLE | - |
2526
| kube_pod_container_status_restarts_total | Counter | The number of container restarts per container | | `container`=&lt;container-name&gt; <br> `namespace`=&lt;pod-namespace&gt; <br> `pod`=&lt;pod-name&gt; <br> `uid`=&lt;pod-uid&gt; | STABLE | - |
2627
| kube_pod_container_resource_requests | Gauge | The number of requested request resource by a container | `cpu`=&lt;core&gt; <br> `memory`=&lt;bytes&gt; |`resource`=&lt;resource-name&gt; <br> `unit`=&lt;resource-unit&gt; <br> `container`=&lt;container-name&gt; <br> `pod`=&lt;pod-name&gt; <br> `namespace`=&lt;pod-namespace&gt; <br> `node`=&lt; node-name&gt; <br> `uid`=&lt;pod-uid&gt; | EXPERIMENTAL | - |

‎internal/store/pod.go

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@ func podMetricFamilies(allowAnnotationsList, allowLabelsList []string) []generat
4747
createPodContainerResourceRequestsFamilyGenerator(),
4848
createPodContainerStateStartedFamilyGenerator(),
4949
createPodContainerStatusLastTerminatedReasonFamilyGenerator(),
50+
createPodContainerStatusLastTerminatedExitCodeFamilyGenerator(),
5051
createPodContainerStatusReadyFamilyGenerator(),
5152
createPodContainerStatusRestartsTotalFamilyGenerator(),
5253
createPodContainerStatusRunningFamilyGenerator(),
@@ -335,6 +336,31 @@ func createPodContainerStatusLastTerminatedReasonFamilyGenerator() generator.Fam
335336
)
336337
}
337338

339+
func createPodContainerStatusLastTerminatedExitCodeFamilyGenerator() generator.FamilyGenerator {
340+
return *generator.NewFamilyGenerator(
341+
"kube_pod_container_status_last_terminated_exitcode",
342+
"Describes the exit code for the last container in terminated state.",
343+
metric.Gauge,
344+
"",
345+
wrapPodFunc(func(p *v1.Pod) *metric.Family {
346+
ms := make([]*metric.Metric, 0, len(p.Status.ContainerStatuses))
347+
for _, cs := range p.Status.ContainerStatuses {
348+
if cs.LastTerminationState.Terminated != nil {
349+
ms = append(ms, &metric.Metric{
350+
LabelKeys: []string{"container"},
351+
LabelValues: []string{cs.Name},
352+
Value: float64(cs.LastTerminationState.Terminated.ExitCode),
353+
})
354+
}
355+
}
356+
357+
return &metric.Family{
358+
Metrics: ms,
359+
}
360+
}),
361+
)
362+
}
363+
338364
func createPodContainerStatusReadyFamilyGenerator() generator.FamilyGenerator {
339365
return *generator.NewFamilyGenerator(
340366
"kube_pod_container_status_ready",

‎internal/store/pod_test.go

Lines changed: 27 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -573,7 +573,8 @@ func TestPodStore(t *testing.T) {
573573
StartedAt: metav1.Time{
574574
Time: time.Unix(1501777018, 0),
575575
},
576-
Reason: "OOMKilled",
576+
Reason: "OOMKilled",
577+
ExitCode: 137,
577578
},
578579
},
579580
},
@@ -648,12 +649,14 @@ func TestPodStore(t *testing.T) {
648649
},
649650
},
650651
Want: `
652+
# HELP kube_pod_container_status_last_terminated_exitcode Describes the exit code for the last container in terminated state.
651653
# HELP kube_pod_container_status_last_terminated_reason Describes the last reason the container was in terminated state.
652654
# HELP kube_pod_container_status_running Describes whether the container is currently in running state.
653655
# HELP kube_pod_container_status_terminated Describes whether the container is currently in terminated state.
654656
# HELP kube_pod_container_status_terminated_reason Describes the reason the container is currently in terminated state.
655657
# HELP kube_pod_container_status_waiting Describes whether the container is currently in waiting state.
656658
# HELP kube_pod_container_status_waiting_reason Describes the reason the container is currently in waiting state.
659+
# TYPE kube_pod_container_status_last_terminated_exitcode gauge
657660
# TYPE kube_pod_container_status_last_terminated_reason gauge
658661
# TYPE kube_pod_container_status_running gauge
659662
# TYPE kube_pod_container_status_terminated gauge
@@ -678,6 +681,10 @@ func TestPodStore(t *testing.T) {
678681
"kube_pod_container_status_last_terminated_reason",
679682
"kube_pod_container_status_last_terminated_reason",
680683
"kube_pod_container_status_last_terminated_reason",
684+
"kube_pod_container_status_last_terminated_exitcode",
685+
"kube_pod_container_status_last_terminated_exitcode",
686+
"kube_pod_container_status_last_terminated_exitcode",
687+
"kube_pod_container_status_last_terminated_exitcode",
681688
},
682689
},
683690
{
@@ -709,7 +716,8 @@ func TestPodStore(t *testing.T) {
709716
},
710717
LastTerminationState: v1.ContainerState{
711718
Terminated: &v1.ContainerStateTerminated{
712-
Reason: "OOMKilled",
719+
Reason: "OOMKilled",
720+
ExitCode: 137,
713721
},
714722
},
715723
},
@@ -718,13 +726,15 @@ func TestPodStore(t *testing.T) {
718726
},
719727
Want: `
720728
# HELP kube_pod_container_status_last_terminated_reason Describes the last reason the container was in terminated state.
729+
# HELP kube_pod_container_status_last_terminated_exitcode Describes the exit code for the last container in terminated state.
721730
# HELP kube_pod_container_status_running Describes whether the container is currently in running state.
722731
# HELP kube_pod_container_status_terminated Describes whether the container is currently in terminated state.
723732
# HELP kube_pod_container_status_terminated_reason Describes the reason the container is currently in terminated state.
724733
# HELP kube_pod_container_status_waiting Describes whether the container is currently in waiting state.
725734
# HELP kube_pod_container_status_waiting_reason Describes the reason the container is currently in waiting state.
726735
# HELP kube_pod_container_state_started Start time in unix timestamp for a pod container.
727736
# TYPE kube_pod_container_status_last_terminated_reason gauge
737+
# TYPE kube_pod_container_status_last_terminated_exitcode gauge
728738
# TYPE kube_pod_container_status_running gauge
729739
# TYPE kube_pod_container_status_terminated gauge
730740
# TYPE kube_pod_container_status_terminated_reason gauge
@@ -736,9 +746,11 @@ func TestPodStore(t *testing.T) {
736746
kube_pod_container_status_terminated{container="container7",namespace="ns6",pod="pod6",uid="uid6"} 0
737747
kube_pod_container_status_waiting{container="container7",namespace="ns6",pod="pod6",uid="uid6"} 0
738748
kube_pod_container_status_last_terminated_reason{container="container7",namespace="ns6",pod="pod6",reason="OOMKilled",uid="uid6"} 1
749+
kube_pod_container_status_last_terminated_exitcode{container="container7",namespace="ns6",pod="pod6",uid="uid6"} 137
739750
`,
740751
MetricNames: []string{
741752
"kube_pod_container_status_last_terminated_reason",
753+
"kube_pod_container_status_last_terminated_exitcode",
742754
"kube_pod_container_status_running",
743755
"kube_pod_container_state_started",
744756
"kube_pod_container_status_terminated",
@@ -774,21 +786,24 @@ func TestPodStore(t *testing.T) {
774786
},
775787
LastTerminationState: v1.ContainerState{
776788
Terminated: &v1.ContainerStateTerminated{
777-
Reason: "DeadlineExceeded",
789+
Reason: "DeadlineExceeded",
790+
ExitCode: 143,
778791
},
779792
},
780793
},
781794
},
782795
},
783796
},
784797
Want: `
798+
# HELP kube_pod_container_status_last_terminated_exitcode Describes the exit code for the last container in terminated state.
785799
# HELP kube_pod_container_status_last_terminated_reason Describes the last reason the container was in terminated state.
786800
# HELP kube_pod_container_status_running Describes whether the container is currently in running state.
787801
# HELP kube_pod_container_state_started Start time in unix timestamp for a pod container.
788802
# HELP kube_pod_container_status_terminated Describes whether the container is currently in terminated state.
789803
# HELP kube_pod_container_status_terminated_reason Describes the reason the container is currently in terminated state.
790804
# HELP kube_pod_container_status_waiting Describes whether the container is currently in waiting state.
791805
# HELP kube_pod_container_status_waiting_reason Describes the reason the container is currently in waiting state.
806+
# TYPE kube_pod_container_status_last_terminated_exitcode gauge
792807
# TYPE kube_pod_container_status_last_terminated_reason gauge
793808
# TYPE kube_pod_container_status_running gauge
794809
# TYPE kube_pod_container_state_started gauge
@@ -797,6 +812,7 @@ func TestPodStore(t *testing.T) {
797812
# TYPE kube_pod_container_status_waiting gauge
798813
# TYPE kube_pod_container_status_waiting_reason gauge
799814
kube_pod_container_state_started{container="container7",namespace="ns7",pod="pod7",uid="uid7"} 1.501777018e+09
815+
kube_pod_container_status_last_terminated_exitcode{container="container7",namespace="ns7",pod="pod7",uid="uid7"} 143
800816
kube_pod_container_status_last_terminated_reason{container="container7",namespace="ns7",pod="pod7",reason="DeadlineExceeded",uid="uid7"} 1
801817
kube_pod_container_status_running{container="container7",namespace="ns7",pod="pod7",uid="uid7"} 1
802818
kube_pod_container_status_terminated{container="container7",namespace="ns7",pod="pod7",uid="uid7"} 0
@@ -809,6 +825,7 @@ func TestPodStore(t *testing.T) {
809825
"kube_pod_container_status_terminated_reason",
810826
"kube_pod_container_status_waiting",
811827
"kube_pod_container_status_last_terminated_reason",
828+
"kube_pod_container_status_last_terminated_exitcode",
812829
},
813830
},
814831
{
@@ -2019,7 +2036,8 @@ func BenchmarkPodStore(b *testing.B) {
20192036
},
20202037
LastTerminationState: v1.ContainerState{
20212038
Terminated: &v1.ContainerStateTerminated{
2022-
Reason: "OOMKilled",
2039+
Reason: "OOMKilled",
2040+
ExitCode: 137,
20232041
},
20242042
},
20252043
},
@@ -2035,7 +2053,8 @@ func BenchmarkPodStore(b *testing.B) {
20352053
},
20362054
LastTerminationState: v1.ContainerState{
20372055
Terminated: &v1.ContainerStateTerminated{
2038-
Reason: "OOMKilled",
2056+
Reason: "OOMKilled",
2057+
ExitCode: 137,
20392058
},
20402059
},
20412060
},
@@ -2051,15 +2070,16 @@ func BenchmarkPodStore(b *testing.B) {
20512070
},
20522071
LastTerminationState: v1.ContainerState{
20532072
Terminated: &v1.ContainerStateTerminated{
2054-
Reason: "OOMKilled",
2073+
Reason: "OOMKilled",
2074+
ExitCode: 137,
20552075
},
20562076
},
20572077
},
20582078
},
20592079
},
20602080
}
20612081

2062-
expectedFamilies := 46
2082+
expectedFamilies := 47
20632083
for n := 0; n < b.N; n++ {
20642084
families := f(pod)
20652085
if len(families) != expectedFamilies {

‎pkg/app/server_test.go

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -196,6 +196,7 @@ func TestFullScrapeCycle(t *testing.T) {
196196
# HELP kube_pod_container_resource_limits The number of requested limit resource by a container.
197197
# HELP kube_pod_container_resource_requests The number of requested request resource by a container.
198198
# HELP kube_pod_container_state_started Start time in unix timestamp for a pod container.
199+
# HELP kube_pod_container_status_last_terminated_exitcode Describes the exit code for the last container in terminated state.
199200
# HELP kube_pod_container_status_last_terminated_reason Describes the last reason the container was in terminated state.
200201
# HELP kube_pod_container_status_ready Describes whether the containers readiness check succeeded.
201202
# HELP kube_pod_container_status_restarts_total The number of container restarts per container.
@@ -241,6 +242,7 @@ func TestFullScrapeCycle(t *testing.T) {
241242
# TYPE kube_pod_container_resource_limits gauge
242243
# TYPE kube_pod_container_resource_requests gauge
243244
# TYPE kube_pod_container_state_started gauge
245+
# TYPE kube_pod_container_status_last_terminated_exitcode gauge
244246
# TYPE kube_pod_container_status_last_terminated_reason gauge
245247
# TYPE kube_pod_container_status_ready gauge
246248
# TYPE kube_pod_container_status_restarts_total counter
@@ -297,6 +299,7 @@ kube_pod_container_resource_requests{namespace="default",pod="pod0",uid="abc-0",
297299
kube_pod_container_resource_requests{namespace="default",pod="pod0",uid="abc-0",container="pod1_con1",node="node1",resource="storage",unit="byte"} 4e+08
298300
kube_pod_container_resource_requests{namespace="default",pod="pod0",uid="abc-0",container="pod1_con2",node="node1",resource="cpu",unit="core"} 0.3
299301
kube_pod_container_resource_requests{namespace="default",pod="pod0",uid="abc-0",container="pod1_con2",node="node1",resource="memory",unit="byte"} 2e+08
302+
kube_pod_container_status_last_terminated_exitcode{namespace="default",pod="pod0",uid="abc-0",container="pod1_con1"} 137
300303
kube_pod_container_status_last_terminated_reason{namespace="default",pod="pod0",uid="abc-0",container="pod1_con1",reason="OOMKilled"} 1
301304
kube_pod_container_status_ready{namespace="default",pod="pod0",uid="abc-0",container="pod1_con1"} 0
302305
kube_pod_container_status_ready{namespace="default",pod="pod0",uid="abc-0",container="pod1_con2"} 0
@@ -794,7 +797,8 @@ func pod(client *fake.Clientset, index int) error {
794797
},
795798
LastTerminationState: v1.ContainerState{
796799
Terminated: &v1.ContainerStateTerminated{
797-
Reason: "OOMKilled",
800+
Reason: "OOMKilled",
801+
ExitCode: 137,
798802
},
799803
},
800804
},

0 commit comments

Comments
 (0)