Skip to content

Commit a483701

Browse files
yshnggyshngg
committed
feat(pod): add new metric for pod unscheduled time tracking
The changes introduce a new metric generator `kube_pod_status_unscheduled_time` to record the Unix timestamp when pods transition to unscheduled status. This enhances monitoring capabilities for scheduling issues by capturing exact transition times during `PodScheduled` condition changes.
1 parent 56d3b56 commit a483701

File tree

4 files changed

+32
-1
lines changed

4 files changed

+32
-1
lines changed

‎docs/metrics/workload/pod-metrics.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,7 @@
5353
| kube_pod_status_reason | Gauge | The pod status reasons | | `pod`=&lt;pod-name&gt; <br> `namespace`=&lt;pod-namespace&gt; <br> `reason`=&lt;Evicted\|NodeAffinity\|NodeLost\|Shutdown\|UnexpectedAdmissionError&gt; <br> `uid`=&lt;pod-uid&gt; | EXPERIMENTAL | - |
5454
| kube_pod_status_scheduled_time | Gauge | Unix timestamp when pod moved into scheduled status | seconds | `pod`=&lt;pod-name&gt; <br> `namespace`=&lt;pod-namespace&gt; <br> `uid`=&lt;pod-uid&gt; | STABLE | - |
5555
| kube_pod_status_unschedulable | Gauge | Describes the unschedulable status for the pod | | `pod`=&lt;pod-name&gt; <br> `namespace`=&lt;pod-namespace&gt; <br> `uid`=&lt;pod-uid&gt; | STABLE | - |
56+
| kube_pod_status_unscheduled_time | Gauge | Unix timestamp when pod moved into unscheduled status | seconds | `pod`=&lt;pod-name&gt; <br> `namespace`=&lt;pod-namespace&gt; <br> `uid`=&lt;pod-uid&gt; | EXPERIMENTAL | - |
5657
| kube_pod_tolerations | Gauge | Information about the pod tolerations | | `pod`=&lt;pod-name&gt; <br> `namespace`=&lt;pod-namespace&gt; <br> `uid`=&lt;pod-uid&gt; <br> `key`=&lt;toleration-key&gt; <br> `operator`=&lt;toleration-operator&gt; <br> `value`=&lt;toleration-value&gt; <br> `effect`=&lt;toleration-effect&gt; `toleration_seconds`=&lt;toleration-seconds&gt; | EXPERIMENTAL | - |
5758
| kube_pod_service_account | Gauge | The service account for a pod | | `pod`=&lt;pod-name&gt; <br> `namespace`=&lt;pod-namespace&gt; <br> `uid`=&lt;pod-uid&gt; <br> `service_account`=&lt;service_account&gt; | EXPERIMENTAL | - |
5859
| kube_pod_scheduler | Gauge | The scheduler for a pod | | `pod`=&lt;pod-name&gt; <br> `namespace`=&lt;pod-namespace&gt; <br> `uid`=&lt;pod-uid&gt; <br> `name`=&lt;scheduler-name&gt; | EXPERIMENTAL | - |

‎internal/store/pod.go

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -92,6 +92,7 @@ func podMetricFamilies(allowAnnotationsList, allowLabelsList []string) []generat
9292
createPodStatusScheduledFamilyGenerator(),
9393
createPodStatusScheduledTimeFamilyGenerator(),
9494
createPodStatusUnschedulableFamilyGenerator(),
95+
createPodStatusUnscheduledTimeFamilyGenerator(),
9596
createPodTolerationsFamilyGenerator(),
9697
createPodNodeSelectorsFamilyGenerator(),
9798
createPodServiceAccountFamilyGenerator(),
@@ -1656,6 +1657,33 @@ func createPodStatusUnschedulableFamilyGenerator() generator.FamilyGenerator {
16561657
)
16571658
}
16581659

1660+
func createPodStatusUnscheduledTimeFamilyGenerator() generator.FamilyGenerator {
1661+
return *generator.NewFamilyGeneratorWithStability(
1662+
"kube_pod_status_unscheduled_time",
1663+
"Unix timestamp when pod moved into unscheduled status",
1664+
metric.Gauge,
1665+
basemetrics.ALPHA,
1666+
"",
1667+
wrapPodFunc(func(p *v1.Pod) *metric.Family {
1668+
ms := []*metric.Metric{}
1669+
1670+
for _, c := range p.Status.Conditions {
1671+
if c.Type == v1.PodScheduled && c.Status == v1.ConditionFalse {
1672+
ms = append(ms, &metric.Metric{
1673+
LabelKeys: []string{},
1674+
LabelValues: []string{},
1675+
Value: float64(c.LastTransitionTime.Unix()),
1676+
})
1677+
}
1678+
}
1679+
1680+
return &metric.Family{
1681+
Metrics: ms,
1682+
}
1683+
}),
1684+
)
1685+
}
1686+
16591687
// getUniqueTolerations takes an array
16601688
func getUniqueTolerations(tolerations []v1.Toleration) []v1.Toleration {
16611689
uniqueTolerationsMap := make(map[v1.Toleration]struct{})

‎internal/store/pod_test.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2282,7 +2282,7 @@ func BenchmarkPodStore(b *testing.B) {
22822282
},
22832283
}
22842284

2285-
expectedFamilies := 54
2285+
expectedFamilies := 55
22862286
for n := 0; n < b.N; n++ {
22872287
families := f(pod)
22882288
if len(families) != expectedFamilies {

‎pkg/app/server_test.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -259,6 +259,7 @@ func TestFullScrapeCycle(t *testing.T) {
259259
# HELP kube_pod_status_scheduled [STABLE] Describes the status of the scheduling process for the pod.
260260
# HELP kube_pod_status_scheduled_time [STABLE] Unix timestamp when pod moved into scheduled status
261261
# HELP kube_pod_status_unschedulable [STABLE] Describes the unschedulable status for the pod.
262+
# HELP kube_pod_status_unscheduled_time Unix timestamp when pod moved into unscheduled status
262263
# HELP kube_pod_tolerations Information about the pod tolerations
263264
# TYPE kube_pod_annotations gauge
264265
# TYPE kube_pod_completion_time gauge
@@ -312,6 +313,7 @@ func TestFullScrapeCycle(t *testing.T) {
312313
# TYPE kube_pod_status_scheduled gauge
313314
# TYPE kube_pod_status_scheduled_time gauge
314315
# TYPE kube_pod_status_unschedulable gauge
316+
# TYPE kube_pod_status_unscheduled_time gauge
315317
# TYPE kube_pod_tolerations gauge
316318
kube_pod_container_info{namespace="default",pod="pod0",uid="abc-0",container="pod1_con1",image_spec="k8s.gcr.io/hyperkube2_spec",image="k8s.gcr.io/hyperkube2",image_id="docker://sha256:bbb",container_id="docker://cd456"} 1
317319
kube_pod_container_info{namespace="default",pod="pod0",uid="abc-0",container="pod1_con2",image_spec="k8s.gcr.io/hyperkube3_spec",image="k8s.gcr.io/hyperkube3",image_id="docker://sha256:ccc",container_id="docker://ef789"} 1

0 commit comments

Comments
 (0)