Skip to content

Commit 99eea80

Browse files
kaisozkaisoz
committed
Improve Job integration tests runtime
1 parent 17d7d28 commit 99eea80

File tree

1 file changed

+85
-52
lines changed

1 file changed

+85
-52
lines changed

‎test/integration/job/job_test.go‎

Lines changed: 85 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -424,16 +424,17 @@ func TestJobPodFailurePolicy(t *testing.T) {
424424
},
425425
},
426426
}
427+
428+
closeFn, restConfig, clientSet, ns := setup(t, "pod-failure-policy")
429+
t.Cleanup(closeFn)
427430
for name, test := range testCases {
428431
t.Run(name, func(t *testing.T) {
429432
resetMetrics()
430433

431-
closeFn, restConfig, clientSet, ns := setup(t, "simple")
432-
defer closeFn()
433434
ctx, cancel := startJobControllerAndWaitForCaches(t, restConfig)
434-
defer func() {
435+
t.Cleanup(func() {
435436
cancel()
436-
}()
437+
})
437438

438439
jobObj, err := createJobWithDefaults(ctx, clientSet, ns.Name, &test.job)
439440
if err != nil {
@@ -758,18 +759,18 @@ func TestSuccessPolicy(t *testing.T) {
758759
wantConditionTypes: []batchv1.JobConditionType{batchv1.JobFailed},
759760
},
760761
}
762+
763+
closeFn, restConfig, clientSet, ns := setup(t, "success-policy")
764+
t.Cleanup(closeFn)
761765
for name, tc := range testCases {
762766
t.Run(name, func(t *testing.T) {
763767
resetMetrics()
764768
featuregatetesting.SetFeatureGateDuringTest(t, feature.DefaultFeatureGate, features.JobSuccessPolicy, tc.enableJobSuccessPolicy)
765769
featuregatetesting.SetFeatureGateDuringTest(t, feature.DefaultFeatureGate, features.JobBackoffLimitPerIndex, tc.enableBackoffLimitPerIndex)
766770

767-
closeFn, restConfig, clientSet, ns := setup(t, "simple")
768-
defer closeFn()
769771
ctx, cancel := startJobControllerAndWaitForCaches(t, restConfig)
770-
defer func() {
771-
cancel()
772-
}()
772+
t.Cleanup(cancel)
773+
773774
jobObj, err := createJobWithDefaults(ctx, clientSet, ns.Name, &tc.job)
774775
if err != nil {
775776
t.Fatalf("Error %v while creating the Job %q", err, jobObj.Name)
@@ -1177,11 +1178,12 @@ func TestBackoffLimitPerIndex_JobPodsCreatedWithExponentialBackoff(t *testing.T)
11771178
// are terminal. The fate of the Job is indicated by the interim Job conditions:
11781179
// FailureTarget, or SuccessCriteriaMet.
11791180
func TestDelayTerminalPhaseCondition(t *testing.T) {
1181+
const blockDeletionFinalizerForTest string = "fake.example.com/blockDeletion"
11801182
t.Cleanup(setDurationDuringTest(&jobcontroller.DefaultJobPodFailureBackOff, fastPodFailureBackoff))
11811183

11821184
podTemplateSpec := v1.PodTemplateSpec{
11831185
ObjectMeta: metav1.ObjectMeta{
1184-
Finalizers: []string{"fake.example.com/blockDeletion"},
1186+
Finalizers: []string{blockDeletionFinalizerForTest},
11851187
},
11861188
Spec: v1.PodSpec{
11871189
Containers: []v1.Container{
@@ -1512,6 +1514,9 @@ func TestDelayTerminalPhaseCondition(t *testing.T) {
15121514
},
15131515
},
15141516
}
1517+
1518+
closeFn, restConfig, clientSet, ns := setup(t, "delay-terminal-condition")
1519+
t.Cleanup(closeFn)
15151520
for name, test := range testCases {
15161521
t.Run(name, func(t *testing.T) {
15171522
resetMetrics()
@@ -1520,16 +1525,18 @@ func TestDelayTerminalPhaseCondition(t *testing.T) {
15201525
featuregatetesting.SetFeatureGateDuringTest(t, feature.DefaultFeatureGate, features.ElasticIndexedJob, true)
15211526
featuregatetesting.SetFeatureGateDuringTest(t, feature.DefaultFeatureGate, features.JobSuccessPolicy, test.enableJobSuccessPolicy)
15221527

1523-
closeFn, restConfig, clientSet, ns := setup(t, "delay-terminal-condition")
1524-
t.Cleanup(closeFn)
15251528
ctx, cancel := startJobControllerAndWaitForCaches(t, restConfig)
15261529
t.Cleanup(cancel)
15271530

15281531
jobObj, err := createJobWithDefaults(ctx, clientSet, ns.Name, &test.job)
15291532
if err != nil {
15301533
t.Fatalf("Error %q while creating the job %q", err, jobObj.Name)
15311534
}
1532-
t.Cleanup(func() { removePodsFinalizer(ctx, t, clientSet, ns.Name) })
1535+
t.Cleanup(func() {
1536+
if err := cleanUp(ctx, clientSet, jobObj, []string{blockDeletionFinalizerForTest, batchv1.JobTrackingFinalizer}); err != nil {
1537+
t.Fatalf("Failed cleanup: %v", err)
1538+
}
1539+
})
15331540
jobClient := clientSet.BatchV1().Jobs(jobObj.Namespace)
15341541

15351542
waitForPodsToBeActive(ctx, t, jobClient, *jobObj.Spec.Parallelism, jobObj)
@@ -1898,17 +1905,17 @@ func TestBackoffLimitPerIndex(t *testing.T) {
18981905
},
18991906
},
19001907
}
1908+
1909+
closeFn, restConfig, clientSet, ns := setup(t, "backoff-limit-per-index")
1910+
t.Cleanup(closeFn)
19011911
for name, test := range testCases {
19021912
t.Run(name, func(t *testing.T) {
19031913
resetMetrics()
19041914
featuregatetesting.SetFeatureGateDuringTest(t, feature.DefaultFeatureGate, features.JobBackoffLimitPerIndex, true)
19051915

1906-
closeFn, restConfig, clientSet, ns := setup(t, "simple")
1907-
defer closeFn()
19081916
ctx, cancel := startJobControllerAndWaitForCaches(t, restConfig)
1909-
defer func() {
1910-
cancel()
1911-
}()
1917+
t.Cleanup(cancel)
1918+
19121919
jobObj, err := createJobWithDefaults(ctx, clientSet, ns.Name, &test.job)
19131920
if err != nil {
19141921
t.Fatalf("Error %q while creating the job %q", err, jobObj.Name)
@@ -1969,6 +1976,7 @@ func TestBackoffLimitPerIndex(t *testing.T) {
19691976
// reconcile or skip reconciliation of the Job depending on the Job's managedBy
19701977
// field, and the enablement of the JobManagedBy feature gate.
19711978
func TestManagedBy(t *testing.T) {
1979+
const blockDeletionFinalizerForTest string = "fake.example.com/blockDeletion"
19721980
customControllerName := "example.com/custom-job-controller"
19731981
podTemplateSpec := v1.PodTemplateSpec{
19741982
Spec: v1.PodSpec{
@@ -2061,20 +2069,29 @@ func TestManagedBy(t *testing.T) {
20612069
},
20622070
},
20632071
}
2072+
2073+
closeFn, restConfig, clientSet, ns := setup(t, "managed-by")
2074+
t.Cleanup(closeFn)
20642075
for name, test := range testCases {
20652076
t.Run(name, func(t *testing.T) {
20662077
resetMetrics()
20672078
featuregatetesting.SetFeatureGateDuringTest(t, feature.DefaultFeatureGate, features.JobManagedBy, test.enableJobManagedBy)
20682079

2069-
closeFn, restConfig, clientSet, ns := setup(t, "managed-by")
2070-
defer closeFn()
20712080
ctx, cancel := startJobControllerAndWaitForCaches(t, restConfig)
2072-
defer cancel()
2081+
t.Cleanup(cancel)
2082+
20732083
jobObj, err := createJobWithDefaults(ctx, clientSet, ns.Name, &test.job)
20742084
if err != nil {
20752085
t.Fatalf("Error %v while creating the job %q", err, klog.KObj(jobObj))
20762086
}
20772087

2088+
t.Cleanup(func() {
2089+
// Wait for cleanup to finish to prevent this job from affecting metrics
2090+
if err := cleanUp(ctx, clientSet, jobObj, []string{blockDeletionFinalizerForTest, batchv1.JobTrackingFinalizer}); err != nil {
2091+
t.Fatalf("Failed cleanup: %v", err)
2092+
}
2093+
})
2094+
20782095
if test.wantReconciledByBuiltInController {
20792096
validateJobPodsStatus(ctx, t, clientSet, jobObj, podsByStatus{
20802097
Active: int(*jobObj.Spec.Parallelism),
@@ -2814,6 +2831,7 @@ func TestIndexedJob(t *testing.T) {
28142831
}
28152832

28162833
func TestJobPodReplacementPolicy(t *testing.T) {
2834+
const blockDeletionFinalizerForTest string = "fake.example.com/blockDeletion"
28172835
t.Cleanup(setDurationDuringTest(&jobcontroller.DefaultJobPodFailureBackOff, fastPodFailureBackoff))
28182836
indexedCompletion := batchv1.IndexedCompletion
28192837
nonIndexedCompletion := batchv1.NonIndexedCompletion
@@ -2844,7 +2862,7 @@ func TestJobPodReplacementPolicy(t *testing.T) {
28442862
CompletionMode: &indexedCompletion,
28452863
Template: v1.PodTemplateSpec{
28462864
ObjectMeta: metav1.ObjectMeta{
2847-
Finalizers: []string{"fake.example.com/blockDeletion"},
2865+
Finalizers: []string{blockDeletionFinalizerForTest},
28482866
},
28492867
},
28502868
},
@@ -2869,7 +2887,7 @@ func TestJobPodReplacementPolicy(t *testing.T) {
28692887
PodReplacementPolicy: podReplacementPolicy(batchv1.TerminatingOrFailed),
28702888
Template: v1.PodTemplateSpec{
28712889
ObjectMeta: metav1.ObjectMeta{
2872-
Finalizers: []string{"fake.example.com/blockDeletion"},
2890+
Finalizers: []string{blockDeletionFinalizerForTest},
28732891
},
28742892
},
28752893
},
@@ -2897,7 +2915,7 @@ func TestJobPodReplacementPolicy(t *testing.T) {
28972915
PodReplacementPolicy: podReplacementPolicy(batchv1.TerminatingOrFailed),
28982916
Template: v1.PodTemplateSpec{
28992917
ObjectMeta: metav1.ObjectMeta{
2900-
Finalizers: []string{"fake.example.com/blockDeletion"},
2918+
Finalizers: []string{blockDeletionFinalizerForTest},
29012919
},
29022920
},
29032921
},
@@ -2925,7 +2943,7 @@ func TestJobPodReplacementPolicy(t *testing.T) {
29252943
PodReplacementPolicy: podReplacementPolicy(batchv1.Failed),
29262944
Template: v1.PodTemplateSpec{
29272945
ObjectMeta: metav1.ObjectMeta{
2928-
Finalizers: []string{"fake.example.com/blockDeletion"},
2946+
Finalizers: []string{blockDeletionFinalizerForTest},
29292947
},
29302948
},
29312949
PodFailurePolicy: &batchv1.PodFailurePolicy{
@@ -2959,7 +2977,7 @@ func TestJobPodReplacementPolicy(t *testing.T) {
29592977
PodReplacementPolicy: podReplacementPolicy(batchv1.Failed),
29602978
Template: v1.PodTemplateSpec{
29612979
ObjectMeta: metav1.ObjectMeta{
2962-
Finalizers: []string{"fake.example.com/blockDeletion"},
2980+
Finalizers: []string{blockDeletionFinalizerForTest},
29632981
},
29642982
},
29652983
},
@@ -2987,7 +3005,7 @@ func TestJobPodReplacementPolicy(t *testing.T) {
29873005
PodReplacementPolicy: podReplacementPolicy(batchv1.Failed),
29883006
Template: v1.PodTemplateSpec{
29893007
ObjectMeta: metav1.ObjectMeta{
2990-
Finalizers: []string{"fake.example.com/blockDeletion"},
3008+
Finalizers: []string{blockDeletionFinalizerForTest},
29913009
},
29923010
},
29933011
},
@@ -3007,13 +3025,14 @@ func TestJobPodReplacementPolicy(t *testing.T) {
30073025
},
30083026
},
30093027
}
3028+
3029+
closeFn, restConfig, clientSet, ns := setup(t, "pod-replacement-policy")
3030+
t.Cleanup(closeFn)
30103031
for name, tc := range cases {
30113032
tc := tc
30123033
t.Run(name, func(t *testing.T) {
30133034
featuregatetesting.SetFeatureGateDuringTest(t, feature.DefaultFeatureGate, features.JobPodReplacementPolicy, tc.podReplacementPolicyEnabled)
30143035

3015-
closeFn, restConfig, clientSet, ns := setup(t, "pod-replacement-policy")
3016-
t.Cleanup(closeFn)
30173036
ctx, cancel := startJobControllerAndWaitForCaches(t, restConfig)
30183037
t.Cleanup(cancel)
30193038
resetMetrics()
@@ -3027,7 +3046,11 @@ func TestJobPodReplacementPolicy(t *testing.T) {
30273046
jobClient := clientSet.BatchV1().Jobs(jobObj.Namespace)
30283047

30293048
waitForPodsToBeActive(ctx, t, jobClient, 2, jobObj)
3030-
t.Cleanup(func() { removePodsFinalizer(ctx, t, clientSet, ns.Name) })
3049+
t.Cleanup(func() {
3050+
if err := cleanUp(ctx, clientSet, jobObj, []string{blockDeletionFinalizerForTest, batchv1.JobTrackingFinalizer}); err != nil {
3051+
t.Fatalf("Failed cleanup: %v", err)
3052+
}
3053+
})
30313054

30323055
deletePods(ctx, t, clientSet, ns.Name)
30333056

@@ -3218,11 +3241,11 @@ func TestElasticIndexedJob(t *testing.T) {
32183241
},
32193242
}
32203243

3244+
closeFn, restConfig, clientSet, ns := setup(t, "indexed")
3245+
t.Cleanup(closeFn)
32213246
for name, tc := range cases {
32223247
tc := tc
32233248
t.Run(name, func(t *testing.T) {
3224-
closeFn, restConfig, clientSet, ns := setup(t, "indexed")
3225-
defer closeFn()
32263249
ctx, cancel := startJobControllerAndWaitForCaches(t, restConfig)
32273250
defer cancel()
32283251
resetMetrics()
@@ -3357,7 +3380,7 @@ func BenchmarkLargeIndexedJob(b *testing.B) {
33573380
b.Fatalf("Failed to create Job: %v", err)
33583381
}
33593382
b.Cleanup(func() {
3360-
if err := cleanUp(ctx, clientSet, jobObj); err != nil {
3383+
if err := cleanUp(ctx, clientSet, jobObj, []string{batchv1.JobTrackingFinalizer}); err != nil {
33613384
b.Fatalf("Failed cleanup: %v", err)
33623385
}
33633386
})
@@ -3443,7 +3466,7 @@ func BenchmarkLargeFailureHandling(b *testing.B) {
34433466
b.Fatalf("Failed to create Job: %v", err)
34443467
}
34453468
b.Cleanup(func() {
3446-
if err := cleanUp(ctx, clientSet, jobObj); err != nil {
3469+
if err := cleanUp(ctx, clientSet, jobObj, []string{batchv1.JobTrackingFinalizer}); err != nil {
34473470
b.Fatalf("Failed cleanup: %v", err)
34483471
}
34493472
})
@@ -3477,8 +3500,11 @@ func BenchmarkLargeFailureHandling(b *testing.B) {
34773500
}
34783501
}
34793502

3480-
// cleanUp deletes all pods and the job
3481-
func cleanUp(ctx context.Context, clientSet clientset.Interface, jobObj *batchv1.Job) error {
3503+
// cleanUp removes the specified pod finalizers, then deletes all pods and the job.
3504+
func cleanUp(ctx context.Context, clientSet clientset.Interface, jobObj *batchv1.Job, podFinalizersToRemove []string) error {
3505+
if err := removePodsFinalizers(ctx, clientSet, jobObj.Namespace, podFinalizersToRemove); err != nil {
3506+
return err
3507+
}
34823508
// Clean up pods in pages, because DeleteCollection might timeout.
34833509
// #90743
34843510
for {
@@ -3498,14 +3524,18 @@ func cleanUp(ctx context.Context, clientSet clientset.Interface, jobObj *batchv1
34983524
return err
34993525
}
35003526
}
3501-
return clientSet.BatchV1().Jobs(jobObj.Namespace).Delete(ctx, jobObj.Name, metav1.DeleteOptions{})
3527+
3528+
// Set the propagation policy to background to ensure that the job doesn't receive any finalizer at deletion time
3529+
return clientSet.BatchV1().Jobs(jobObj.Namespace).Delete(ctx, jobObj.Name, metav1.DeleteOptions{
3530+
PropagationPolicy: ptr.To(metav1.DeletePropagationBackground),
3531+
})
35023532
}
35033533

35043534
func TestOrphanPodsFinalizersClearedWithGC(t *testing.T) {
3535+
closeFn, restConfig, clientSet, ns := setup(t, "orphan-pod-finalizers")
3536+
t.Cleanup(closeFn)
35053537
for _, policy := range []metav1.DeletionPropagation{metav1.DeletePropagationOrphan, metav1.DeletePropagationBackground, metav1.DeletePropagationForeground} {
35063538
t.Run(string(policy), func(t *testing.T) {
3507-
closeFn, restConfig, clientSet, ns := setup(t, "simple")
3508-
defer closeFn()
35093539
informerSet := informers.NewSharedInformerFactory(clientset.NewForConfigOrDie(restclient.AddUserAgent(restConfig, "controller-informers")), 0)
35103540
// Make the job controller significantly slower to trigger race condition.
35113541
restConfig.QPS = 1
@@ -3814,11 +3844,11 @@ func TestSuspendJob(t *testing.T) {
38143844
},
38153845
}
38163846

3847+
closeFn, restConfig, clientSet, ns := setup(t, "suspend")
3848+
t.Cleanup(closeFn)
38173849
for _, tc := range testCases {
38183850
name := fmt.Sprintf("feature=%v,create=%v,update=%v", tc.featureGate, tc.create.flag, tc.update.flag)
38193851
t.Run(name, func(t *testing.T) {
3820-
closeFn, restConfig, clientSet, ns := setup(t, "suspend")
3821-
defer closeFn()
38223852
ctx, cancel := startJobControllerAndWaitForCaches(t, restConfig)
38233853
defer cancel()
38243854
events, err := clientSet.EventsV1().Events(ns.Name).Watch(ctx, metav1.ListOptions{})
@@ -4333,7 +4363,7 @@ func getCompletionIndex(p *v1.Pod) (int, error) {
43334363

43344364
func createJobWithDefaults(ctx context.Context, clientSet clientset.Interface, ns string, jobObj *batchv1.Job) (*batchv1.Job, error) {
43354365
if jobObj.Name == "" {
4336-
jobObj.Name = "test-job"
4366+
jobObj.GenerateName = "test-job"
43374367
}
43384368
if len(jobObj.Spec.Template.Spec.Containers) == 0 {
43394369
jobObj.Spec.Template.Spec.Containers = []v1.Container{
@@ -4468,23 +4498,25 @@ func deletePods(ctx context.Context, t *testing.T, clientSet clientset.Interface
44684498
}
44694499
}
44704500

4471-
func removePodsFinalizer(ctx context.Context, t *testing.T, clientSet clientset.Interface, namespace string) {
4472-
t.Helper()
4501+
func removePodsFinalizers(ctx context.Context, clientSet clientset.Interface, namespace string, finalizersNames []string) error {
44734502
pods, err := clientSet.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{})
44744503
if err != nil {
4475-
t.Fatalf("Failed to list pods: %v", err)
4504+
return err
44764505
}
4477-
updatePod(ctx, t, clientSet, pods.Items, func(pod *v1.Pod) {
4478-
for i, finalizer := range pod.Finalizers {
4479-
if finalizer == "fake.example.com/blockDeletion" {
4480-
pod.Finalizers = append(pod.Finalizers[:i], pod.Finalizers[i+1:]...)
4506+
4507+
finalizersSet := sets.New(finalizersNames...)
4508+
return updatePod(ctx, clientSet, pods.Items, func(pod *v1.Pod) {
4509+
podFinalizers := []string{}
4510+
for _, podFinalizer := range pod.Finalizers {
4511+
if _, found := finalizersSet[podFinalizer]; !found {
4512+
podFinalizers = append(podFinalizers, podFinalizer)
44814513
}
44824514
}
4515+
pod.Finalizers = podFinalizers
44834516
})
44844517
}
44854518

4486-
func updatePod(ctx context.Context, t *testing.T, clientSet clientset.Interface, pods []v1.Pod, updateFunc func(*v1.Pod)) {
4487-
t.Helper()
4519+
func updatePod(ctx context.Context, clientSet clientset.Interface, pods []v1.Pod, updateFunc func(*v1.Pod)) error {
44884520
for _, val := range pods {
44894521
if err := retry.RetryOnConflict(retry.DefaultBackoff, func() error {
44904522
newPod, err := clientSet.CoreV1().Pods(val.Namespace).Get(ctx, val.Name, metav1.GetOptions{})
@@ -4495,9 +4527,10 @@ func updatePod(ctx context.Context, t *testing.T, clientSet clientset.Interface,
44954527
_, err = clientSet.CoreV1().Pods(val.Namespace).Update(ctx, newPod, metav1.UpdateOptions{})
44964528
return err
44974529
}); err != nil {
4498-
t.Fatalf("Failed to update pod %s: %v", val.Name, err)
4530+
return err
44994531
}
45004532
}
4533+
return nil
45014534
}
45024535

45034536
func failTerminatingPods(ctx context.Context, t *testing.T, clientSet clientset.Interface, namespace string) {

0 commit comments

Comments
 (0)