Skip to content

Commit 7533123

Browse files
AxeZhanAxeZhan
committed
consider sidecar containers in maxContainerRestarts
1 parent a310305 commit 7533123

File tree

4 files changed

+201
-39
lines changed

4 files changed

+201
-39
lines changed

‎pkg/controller/controller_utils.go‎

Lines changed: 39 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -713,8 +713,8 @@ func (s ByLogging) Less(i, j int) bool {
713713
}
714714
}
715715
// 5. Pods with containers with higher restart counts < lower restart counts
716-
if maxContainerRestarts(s[i]) != maxContainerRestarts(s[j]) {
717-
return maxContainerRestarts(s[i]) > maxContainerRestarts(s[j])
716+
if res := compareMaxContainerRestarts(s[i], s[j]); res != nil {
717+
return *res
718718
}
719719
// 6. older pods < newer pods < empty timestamp pods
720720
if !s[i].CreationTimestamp.Equal(&s[j].CreationTimestamp) {
@@ -756,8 +756,8 @@ func (s ActivePods) Less(i, j int) bool {
756756
}
757757
}
758758
// 5. Pods with containers with higher restart counts < lower restart counts
759-
if maxContainerRestarts(s[i]) != maxContainerRestarts(s[j]) {
760-
return maxContainerRestarts(s[i]) > maxContainerRestarts(s[j])
759+
if res := compareMaxContainerRestarts(s[i], s[j]); res != nil {
760+
return *res
761761
}
762762
// 6. Empty creation time pods < newer pods < older pods
763763
if !s[i].CreationTimestamp.Equal(&s[j].CreationTimestamp) {
@@ -878,8 +878,8 @@ func (s ActivePodsWithRanks) Less(i, j int) bool {
878878
}
879879
}
880880
// 7. Pods with containers with higher restart counts < lower restart counts
881-
if maxContainerRestarts(s.Pods[i]) != maxContainerRestarts(s.Pods[j]) {
882-
return maxContainerRestarts(s.Pods[i]) > maxContainerRestarts(s.Pods[j])
881+
if res := compareMaxContainerRestarts(s.Pods[i], s.Pods[j]); res != nil {
882+
return *res
883883
}
884884
// 8. Empty creation time pods < newer pods < older pods
885885
if !s.Pods[i].CreationTimestamp.Equal(&s.Pods[j].CreationTimestamp) {
@@ -936,12 +936,41 @@ func podReadyTime(pod *v1.Pod) *metav1.Time {
936936
return &metav1.Time{}
937937
}
938938

939-
func maxContainerRestarts(pod *v1.Pod) int {
940-
maxRestarts := 0
939+
func maxContainerRestarts(pod *v1.Pod) (regularRestarts, sidecarRestarts int) {
941940
for _, c := range pod.Status.ContainerStatuses {
942-
maxRestarts = max(maxRestarts, int(c.RestartCount))
941+
regularRestarts = max(regularRestarts, int(c.RestartCount))
943942
}
944-
return maxRestarts
943+
names := sets.New[string]()
944+
for _, c := range pod.Spec.InitContainers {
945+
if c.RestartPolicy != nil && *c.RestartPolicy == v1.ContainerRestartPolicyAlways {
946+
names.Insert(c.Name)
947+
}
948+
}
949+
for _, c := range pod.Status.InitContainerStatuses {
950+
if names.Has(c.Name) {
951+
sidecarRestarts = max(sidecarRestarts, int(c.RestartCount))
952+
}
953+
}
954+
return
955+
}
956+
957+
// We use *bool here to determine equality:
958+
// true: pi has a higher container restart count.
959+
// false: pj has a higher container restart count.
960+
// nil: Both have the same container restart count.
961+
func compareMaxContainerRestarts(pi *v1.Pod, pj *v1.Pod) *bool {
962+
regularRestartsI, sidecarRestartsI := maxContainerRestarts(pi)
963+
regularRestartsJ, sidecarRestartsJ := maxContainerRestarts(pj)
964+
if regularRestartsI != regularRestartsJ {
965+
res := regularRestartsI > regularRestartsJ
966+
return &res
967+
}
968+
// If pods have the same restart count, an attempt is made to compare the restart counts of sidecar containers.
969+
if sidecarRestartsI != sidecarRestartsJ {
970+
res := sidecarRestartsI > sidecarRestartsJ
971+
return &res
972+
}
973+
return nil
945974
}
946975

947976
// FilterActivePods returns pods that have not terminated.

‎pkg/controller/controller_utils_test.go‎

Lines changed: 48 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -490,6 +490,7 @@ func TestSortingActivePods(t *testing.T) {
490490
now := metav1.Now()
491491
then := metav1.Time{Time: now.AddDate(0, -1, 0)}
492492

493+
restartAlways := v1.ContainerRestartPolicyAlways
493494
tests := []struct {
494495
name string
495496
pods []v1.Pod
@@ -545,6 +546,22 @@ func TestSortingActivePods(t *testing.T) {
545546
ContainerStatuses: []v1.ContainerStatus{{RestartCount: 3}, {RestartCount: 0}},
546547
},
547548
},
549+
{
550+
ObjectMeta: metav1.ObjectMeta{Name: "lowerSidecarContainerRestartCount", CreationTimestamp: now},
551+
Spec: v1.PodSpec{
552+
NodeName: "foo",
553+
InitContainers: []v1.Container{{
554+
Name: "sidecar",
555+
RestartPolicy: &restartAlways,
556+
}},
557+
},
558+
Status: v1.PodStatus{
559+
Phase: v1.PodRunning,
560+
Conditions: []v1.PodCondition{{Type: v1.PodReady, Status: v1.ConditionTrue, LastTransitionTime: then}},
561+
ContainerStatuses: []v1.ContainerStatus{{RestartCount: 2}, {RestartCount: 1}},
562+
InitContainerStatuses: []v1.ContainerStatus{{Name: "sidecar", RestartCount: 2}},
563+
},
564+
},
548565
{
549566
ObjectMeta: metav1.ObjectMeta{Name: "lowerContainerRestartCount", CreationTimestamp: now},
550567
Spec: v1.PodSpec{NodeName: "foo"},
@@ -572,6 +589,7 @@ func TestSortingActivePods(t *testing.T) {
572589
"runningNoLastTransitionTime",
573590
"runningWithLastTransitionTime",
574591
"runningLongerTime",
592+
"lowerSidecarContainerRestartCount",
575593
"lowerContainerRestartCount",
576594
"oldest",
577595
},
@@ -610,44 +628,52 @@ func TestSortingActivePodsWithRanks(t *testing.T) {
610628
then5Hours := metav1.Time{Time: now.Add(-5 * time.Hour)}
611629
then8Hours := metav1.Time{Time: now.Add(-8 * time.Hour)}
612630
zeroTime := metav1.Time{}
613-
pod := func(podName, nodeName string, phase v1.PodPhase, ready bool, restarts int32, readySince metav1.Time, created metav1.Time, annotations map[string]string) *v1.Pod {
631+
restartAlways := v1.ContainerRestartPolicyAlways
632+
pod := func(podName, nodeName string, phase v1.PodPhase, ready bool, restarts int32, sideRestarts int32, readySince metav1.Time, created metav1.Time, annotations map[string]string) *v1.Pod {
614633
var conditions []v1.PodCondition
615634
var containerStatuses []v1.ContainerStatus
635+
var initContainerStatuses []v1.ContainerStatus
616636
if ready {
617637
conditions = []v1.PodCondition{{Type: v1.PodReady, Status: v1.ConditionTrue, LastTransitionTime: readySince}}
618638
containerStatuses = []v1.ContainerStatus{{RestartCount: restarts}}
639+
initContainerStatuses = []v1.ContainerStatus{{Name: "sidecar", RestartCount: sideRestarts}}
619640
}
620641
return &v1.Pod{
621642
ObjectMeta: metav1.ObjectMeta{
622643
CreationTimestamp: created,
623644
Name: podName,
624645
Annotations: annotations,
625646
},
626-
Spec: v1.PodSpec{NodeName: nodeName},
647+
Spec: v1.PodSpec{
648+
NodeName: nodeName,
649+
InitContainers: []v1.Container{{Name: "sidecar", RestartPolicy: &restartAlways}},
650+
},
627651
Status: v1.PodStatus{
628-
Conditions: conditions,
629-
ContainerStatuses: containerStatuses,
630-
Phase: phase,
652+
Conditions: conditions,
653+
ContainerStatuses: containerStatuses,
654+
InitContainerStatuses: initContainerStatuses,
655+
Phase: phase,
631656
},
632657
}
633658
}
634659
var (
635-
unscheduledPod = pod("unscheduled", "", v1.PodPending, false, 0, zeroTime, zeroTime, nil)
636-
scheduledPendingPod = pod("pending", "node", v1.PodPending, false, 0, zeroTime, zeroTime, nil)
637-
unknownPhasePod = pod("unknown-phase", "node", v1.PodUnknown, false, 0, zeroTime, zeroTime, nil)
638-
runningNotReadyPod = pod("not-ready", "node", v1.PodRunning, false, 0, zeroTime, zeroTime, nil)
639-
runningReadyNoLastTransitionTimePod = pod("ready-no-last-transition-time", "node", v1.PodRunning, true, 0, zeroTime, zeroTime, nil)
640-
runningReadyNow = pod("ready-now", "node", v1.PodRunning, true, 0, now, now, nil)
641-
runningReadyThen = pod("ready-then", "node", v1.PodRunning, true, 0, then1Month, then1Month, nil)
642-
runningReadyNowHighRestarts = pod("ready-high-restarts", "node", v1.PodRunning, true, 9001, now, now, nil)
643-
runningReadyNowCreatedThen = pod("ready-now-created-then", "node", v1.PodRunning, true, 0, now, then1Month, nil)
644-
lowPodDeletionCost = pod("low-deletion-cost", "node", v1.PodRunning, true, 0, now, then1Month, map[string]string{core.PodDeletionCost: "10"})
645-
highPodDeletionCost = pod("high-deletion-cost", "node", v1.PodRunning, true, 0, now, then1Month, map[string]string{core.PodDeletionCost: "100"})
646-
unscheduled5Hours = pod("unscheduled-5-hours", "", v1.PodPending, false, 0, then5Hours, then5Hours, nil)
647-
unscheduled8Hours = pod("unscheduled-10-hours", "", v1.PodPending, false, 0, then8Hours, then8Hours, nil)
648-
ready2Hours = pod("ready-2-hours", "", v1.PodRunning, true, 0, then2Hours, then1Month, nil)
649-
ready5Hours = pod("ready-5-hours", "", v1.PodRunning, true, 0, then5Hours, then1Month, nil)
650-
ready10Hours = pod("ready-10-hours", "", v1.PodRunning, true, 0, then8Hours, then1Month, nil)
660+
unscheduledPod = pod("unscheduled", "", v1.PodPending, false, 0, 0, zeroTime, zeroTime, nil)
661+
scheduledPendingPod = pod("pending", "node", v1.PodPending, false, 0, 0, zeroTime, zeroTime, nil)
662+
unknownPhasePod = pod("unknown-phase", "node", v1.PodUnknown, false, 0, 0, zeroTime, zeroTime, nil)
663+
runningNotReadyPod = pod("not-ready", "node", v1.PodRunning, false, 0, 0, zeroTime, zeroTime, nil)
664+
runningReadyNoLastTransitionTimePod = pod("ready-no-last-transition-time", "node", v1.PodRunning, true, 0, 0, zeroTime, zeroTime, nil)
665+
runningReadyNow = pod("ready-now", "node", v1.PodRunning, true, 0, 0, now, now, nil)
666+
runningReadyThen = pod("ready-then", "node", v1.PodRunning, true, 0, 0, then1Month, then1Month, nil)
667+
runningReadyNowHighRestarts = pod("ready-high-restarts", "node", v1.PodRunning, true, 9001, 0, now, now, nil)
668+
runningReadyNowHighSideRestarts = pod("ready-high-restarts", "node", v1.PodRunning, true, 9001, 9001, now, now, nil)
669+
runningReadyNowCreatedThen = pod("ready-now-created-then", "node", v1.PodRunning, true, 0, 0, now, then1Month, nil)
670+
lowPodDeletionCost = pod("low-deletion-cost", "node", v1.PodRunning, true, 0, 0, now, then1Month, map[string]string{core.PodDeletionCost: "10"})
671+
highPodDeletionCost = pod("high-deletion-cost", "node", v1.PodRunning, true, 0, 0, now, then1Month, map[string]string{core.PodDeletionCost: "100"})
672+
unscheduled5Hours = pod("unscheduled-5-hours", "", v1.PodPending, false, 0, 0, then5Hours, then5Hours, nil)
673+
unscheduled8Hours = pod("unscheduled-10-hours", "", v1.PodPending, false, 0, 0, then8Hours, then8Hours, nil)
674+
ready2Hours = pod("ready-2-hours", "", v1.PodRunning, true, 0, 0, then2Hours, then1Month, nil)
675+
ready5Hours = pod("ready-5-hours", "", v1.PodRunning, true, 0, 0, then5Hours, then1Month, nil)
676+
ready10Hours = pod("ready-10-hours", "", v1.PodRunning, true, 0, 0, then8Hours, then1Month, nil)
651677
)
652678
equalityTests := []struct {
653679
p1 *v1.Pod
@@ -702,6 +728,7 @@ func TestSortingActivePodsWithRanks(t *testing.T) {
702728
{lesser: podWithRank{runningReadyNow, 1}, greater: podWithRank{runningReadyThen, 1}},
703729
{lesser: podWithRank{runningReadyNow, 2}, greater: podWithRank{runningReadyThen, 1}},
704730
{lesser: podWithRank{runningReadyNowHighRestarts, 1}, greater: podWithRank{runningReadyNow, 1}},
731+
{lesser: podWithRank{runningReadyNowHighSideRestarts, 1}, greater: podWithRank{runningReadyNowHighRestarts, 1}},
705732
{lesser: podWithRank{runningReadyNow, 2}, greater: podWithRank{runningReadyNowHighRestarts, 1}},
706733
{lesser: podWithRank{runningReadyNow, 1}, greater: podWithRank{runningReadyNowCreatedThen, 1}},
707734
{lesser: podWithRank{runningReadyNowCreatedThen, 2}, greater: podWithRank{runningReadyNow, 1}},

‎staging/src/k8s.io/kubectl/pkg/util/podutils/podutils.go‎

Lines changed: 38 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ import (
2121

2222
corev1 "k8s.io/api/core/v1"
2323
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
24+
"k8s.io/apimachinery/pkg/util/sets"
2425
)
2526

2627
// IsPodAvailable returns true if a pod is available; false otherwise.
@@ -113,8 +114,8 @@ func (s ByLogging) Less(i, j int) bool {
113114
return afterOrZero(podReadyTime(s[j]), podReadyTime(s[i]))
114115
}
115116
// 5. Pods with containers with higher restart counts < lower restart counts
116-
if maxContainerRestarts(s[i]) != maxContainerRestarts(s[j]) {
117-
return maxContainerRestarts(s[i]) > maxContainerRestarts(s[j])
117+
if res := compareMaxContainerRestarts(s[i], s[j]); res != nil {
118+
return *res
118119
}
119120
// 6. older pods < newer pods < empty timestamp pods
120121
if !s[i].CreationTimestamp.Equal(&s[j].CreationTimestamp) {
@@ -161,8 +162,8 @@ func (s ActivePods) Less(i, j int) bool {
161162
return afterOrZero(podReadyTime(s[i]), podReadyTime(s[j]))
162163
}
163164
// 7. Pods with containers with higher restart counts < lower restart counts
164-
if maxContainerRestarts(s[i]) != maxContainerRestarts(s[j]) {
165-
return maxContainerRestarts(s[i]) > maxContainerRestarts(s[j])
165+
if res := compareMaxContainerRestarts(s[i], s[j]); res != nil {
166+
return *res
166167
}
167168
// 8. Empty creation time pods < newer pods < older pods
168169
if !s[i].CreationTimestamp.Equal(&s[j].CreationTimestamp) {
@@ -190,12 +191,41 @@ func podReadyTime(pod *corev1.Pod) *metav1.Time {
190191
return &metav1.Time{}
191192
}
192193

193-
func maxContainerRestarts(pod *corev1.Pod) int {
194-
maxRestarts := 0
194+
func maxContainerRestarts(pod *corev1.Pod) (regularRestarts, sidecarRestarts int) {
195195
for _, c := range pod.Status.ContainerStatuses {
196-
maxRestarts = max(maxRestarts, int(c.RestartCount))
196+
regularRestarts = max(regularRestarts, int(c.RestartCount))
197197
}
198-
return maxRestarts
198+
names := sets.New[string]()
199+
for _, c := range pod.Spec.InitContainers {
200+
if c.RestartPolicy != nil && *c.RestartPolicy == corev1.ContainerRestartPolicyAlways {
201+
names.Insert(c.Name)
202+
}
203+
}
204+
for _, c := range pod.Status.InitContainerStatuses {
205+
if names.Has(c.Name) {
206+
sidecarRestarts = max(sidecarRestarts, int(c.RestartCount))
207+
}
208+
}
209+
return
210+
}
211+
212+
// We use *bool here to determine equality:
213+
// true: pi has a higher container restart count.
214+
// false: pj has a higher container restart count.
215+
// nil: Both have the same container restart count.
216+
func compareMaxContainerRestarts(pi *corev1.Pod, pj *corev1.Pod) *bool {
217+
regularRestartsI, sidecarRestartsI := maxContainerRestarts(pi)
218+
regularRestartsJ, sidecarRestartsJ := maxContainerRestarts(pj)
219+
if regularRestartsI != regularRestartsJ {
220+
res := regularRestartsI > regularRestartsJ
221+
return &res
222+
}
223+
// If pods have the same restart count, an attempt is made to compare the restart counts of sidecar containers.
224+
if sidecarRestartsI != sidecarRestartsJ {
225+
res := sidecarRestartsI > sidecarRestartsJ
226+
return &res
227+
}
228+
return nil
199229
}
200230

201231
// ContainerType and VisitContainers are taken from

‎staging/src/k8s.io/kubectl/pkg/util/podutils/podutils_test.go‎

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ func TestActivePods(t *testing.T) {
2929
time1 := metav1.Now()
3030
time2 := metav1.NewTime(time1.Add(1 * time.Second))
3131
time3 := metav1.NewTime(time1.Add(2 * time.Second))
32+
restartAlways := corev1.ContainerRestartPolicyAlways
3233

3334
tests := []struct {
3435
name string
@@ -364,6 +365,81 @@ func TestActivePods(t *testing.T) {
364365
},
365366
},
366367
},
368+
{
369+
name: "higher sidecar restart count should sort before lower restart count",
370+
pod1: &corev1.Pod{
371+
ObjectMeta: metav1.ObjectMeta{
372+
Name: "podWithMoreRestarts",
373+
Namespace: "default",
374+
},
375+
Spec: corev1.PodSpec{
376+
NodeName: "node1",
377+
InitContainers: []corev1.Container{
378+
{
379+
Name: "sidecar",
380+
RestartPolicy: &restartAlways,
381+
},
382+
},
383+
},
384+
Status: corev1.PodStatus{
385+
Phase: corev1.PodRunning,
386+
Conditions: []corev1.PodCondition{
387+
{
388+
Type: corev1.PodReady,
389+
Status: corev1.ConditionTrue,
390+
LastTransitionTime: time1,
391+
},
392+
},
393+
ContainerStatuses: []corev1.ContainerStatus{
394+
{
395+
RestartCount: 3,
396+
},
397+
},
398+
InitContainerStatuses: []corev1.ContainerStatus{
399+
{
400+
Name: "sidecar",
401+
RestartCount: 3,
402+
},
403+
},
404+
},
405+
},
406+
pod2: &corev1.Pod{
407+
ObjectMeta: metav1.ObjectMeta{
408+
Name: "podWithLessRestarts",
409+
Namespace: "default",
410+
},
411+
Spec: corev1.PodSpec{
412+
NodeName: "node1",
413+
InitContainers: []corev1.Container{
414+
{
415+
Name: "sidecar",
416+
RestartPolicy: &restartAlways,
417+
},
418+
},
419+
},
420+
Status: corev1.PodStatus{
421+
Phase: corev1.PodRunning,
422+
Conditions: []corev1.PodCondition{
423+
{
424+
Type: corev1.PodReady,
425+
Status: corev1.ConditionTrue,
426+
LastTransitionTime: time1,
427+
},
428+
},
429+
ContainerStatuses: []corev1.ContainerStatus{
430+
{
431+
RestartCount: 3,
432+
},
433+
},
434+
InitContainerStatuses: []corev1.ContainerStatus{
435+
{
436+
Name: "sidecar",
437+
RestartCount: 2,
438+
},
439+
},
440+
},
441+
},
442+
},
367443
}
368444
for _, tt := range tests {
369445
t.Run(tt.name, func(t *testing.T) {

0 commit comments

Comments
 (0)