Skip to content

Commit f6b460c

Browse files
Wait for terminating pods to be deleted
To reduce flakiness of Scheduler Predicates E2E tests Signed-off-by: Aldo Culquicondor <[email protected]>
1 parent ce11622 commit f6b460c

File tree

2 files changed

+51
-11
lines changed

2 files changed

+51
-11
lines changed

test/e2e/scheduling/framework.go

Lines changed: 50 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ limitations under the License.
1717
package scheduling
1818

1919
import (
20+
"fmt"
2021
"time"
2122

2223
"github.com/onsi/ginkgo"
@@ -29,20 +30,23 @@ import (
2930
e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
3031
)
3132

33+
var (
34+
timeout = 10 * time.Minute
35+
waitTime = 2 * time.Second
36+
)
37+
3238
// SIGDescribe annotates the test with the SIG label.
3339
func SIGDescribe(text string, body func()) bool {
3440
return ginkgo.Describe("[sig-scheduling] "+text, body)
3541
}
3642

3743
// WaitForStableCluster waits until all existing pods are scheduled and returns their amount.
3844
func WaitForStableCluster(c clientset.Interface, masterNodes sets.String) int {
39-
timeout := 10 * time.Minute
4045
startTime := time.Now()
41-
4246
// Wait for all pods to be scheduled.
43-
allScheduledPods, allNotScheduledPods := getFilteredPods(c, masterNodes, metav1.NamespaceAll)
47+
allScheduledPods, allNotScheduledPods := getScheduledAndUnscheduledPods(c, masterNodes, metav1.NamespaceAll)
4448
for len(allNotScheduledPods) != 0 {
45-
time.Sleep(2 * time.Second)
49+
time.Sleep(waitTime)
4650
if startTime.Add(timeout).Before(time.Now()) {
4751
framework.Logf("Timed out waiting for the following pods to schedule")
4852
for _, p := range allNotScheduledPods {
@@ -51,22 +55,57 @@ func WaitForStableCluster(c clientset.Interface, masterNodes sets.String) int {
5155
framework.Failf("Timed out after %v waiting for stable cluster.", timeout)
5256
break
5357
}
54-
allScheduledPods, allNotScheduledPods = getFilteredPods(c, masterNodes, metav1.NamespaceAll)
58+
allScheduledPods, allNotScheduledPods = getScheduledAndUnscheduledPods(c, masterNodes, metav1.NamespaceAll)
5559
}
5660
return len(allScheduledPods)
5761
}
5862

59-
// getFilteredPods lists scheduled and not scheduled pods in the given namespace, with succeeded and failed pods filtered out.
60-
func getFilteredPods(c clientset.Interface, masterNodes sets.String, ns string) (scheduledPods, notScheduledPods []v1.Pod) {
63+
// WaitForPodsToBeDeleted waits until pods that are terminating to get deleted.
64+
func WaitForPodsToBeDeleted(c clientset.Interface) {
65+
startTime := time.Now()
66+
deleting := getDeletingPods(c, metav1.NamespaceAll)
67+
for len(deleting) != 0 {
68+
if startTime.Add(timeout).Before(time.Now()) {
69+
framework.Logf("Pods still not deleted")
70+
for _, p := range deleting {
71+
framework.Logf("%v/%v", p.Namespace, p.Name)
72+
}
73+
framework.Failf("Timed out after %v waiting for pods to be deleted", timeout)
74+
break
75+
}
76+
time.Sleep(waitTime)
77+
deleting = getDeletingPods(c, metav1.NamespaceAll)
78+
}
79+
}
80+
81+
// getScheduledAndUnscheduledPods lists scheduled and not scheduled pods in the given namespace, with succeeded and failed pods filtered out.
82+
func getScheduledAndUnscheduledPods(c clientset.Interface, masterNodes sets.String, ns string) (scheduledPods, notScheduledPods []v1.Pod) {
6183
pods, err := c.CoreV1().Pods(ns).List(metav1.ListOptions{})
62-
framework.ExpectNoError(err, "listing all pods in kube-system namespace while waiting for stable cluster")
84+
framework.ExpectNoError(err, fmt.Sprintf("listing all pods in namespace %q while waiting for stable cluster", ns))
6385
// API server returns also Pods that succeeded. We need to filter them out.
6486
filteredPods := make([]v1.Pod, 0, len(pods.Items))
65-
for _, pod := range pods.Items {
66-
if pod.Status.Phase != v1.PodSucceeded && pod.Status.Phase != v1.PodFailed {
67-
filteredPods = append(filteredPods, pod)
87+
for _, p := range pods.Items {
88+
if !podTerminated(p) {
89+
filteredPods = append(filteredPods, p)
6890
}
6991
}
7092
pods.Items = filteredPods
7193
return e2epod.GetPodsScheduled(masterNodes, pods)
7294
}
95+
96+
// getDeletingPods returns whether there are any pods marked for deletion.
97+
func getDeletingPods(c clientset.Interface, ns string) []v1.Pod {
98+
pods, err := c.CoreV1().Pods(ns).List(metav1.ListOptions{})
99+
framework.ExpectNoError(err, fmt.Sprintf("listing all pods in namespace %q while waiting for pods to terminate", ns))
100+
var deleting []v1.Pod
101+
for _, p := range pods.Items {
102+
if p.ObjectMeta.DeletionTimestamp != nil && !podTerminated(p) {
103+
deleting = append(deleting, p)
104+
}
105+
}
106+
return deleting
107+
}
108+
109+
func podTerminated(p v1.Pod) bool {
110+
return p.Status.Phase == v1.PodSucceeded || p.Status.Phase == v1.PodFailed
111+
}

test/e2e/scheduling/predicates.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -129,6 +129,7 @@ var _ = SIGDescribe("SchedulerPredicates [Serial]", func() {
129129
totalPodCapacity += podCapacity.Value()
130130
}
131131

132+
WaitForPodsToBeDeleted(cs)
132133
currentlyScheduledPods := WaitForStableCluster(cs, masterNodes)
133134
podsNeededForSaturation := int(totalPodCapacity) - currentlyScheduledPods
134135

0 commit comments

Comments
 (0)