Skip to content

Commit ebe3404

Browse files
k8s-ci-robotk8s-ci-robot
authored
Merge pull request kubernetes#126439 from kaisoz/improve-job-test-runtime
Improve Job integration tests runtime by reusing apiserver within each test
2 parents c3ebd95 + 99eea80 commit ebe3404

File tree

1 file changed

+85
-52
lines changed

1 file changed

+85
-52
lines changed

‎test/integration/job/job_test.go‎

Lines changed: 85 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -427,16 +427,17 @@ func TestJobPodFailurePolicy(t *testing.T) {
427427
},
428428
},
429429
}
430+
431+
closeFn, restConfig, clientSet, ns := setup(t, "pod-failure-policy")
432+
t.Cleanup(closeFn)
430433
for name, test := range testCases {
431434
t.Run(name, func(t *testing.T) {
432435
resetMetrics()
433436

434-
closeFn, restConfig, clientSet, ns := setup(t, "simple")
435-
defer closeFn()
436437
ctx, cancel := startJobControllerAndWaitForCaches(t, restConfig)
437-
defer func() {
438+
t.Cleanup(func() {
438439
cancel()
439-
}()
440+
})
440441

441442
jobObj, err := createJobWithDefaults(ctx, clientSet, ns.Name, &test.job)
442443
if err != nil {
@@ -761,18 +762,18 @@ func TestSuccessPolicy(t *testing.T) {
761762
wantConditionTypes: []batchv1.JobConditionType{batchv1.JobFailed},
762763
},
763764
}
765+
766+
closeFn, restConfig, clientSet, ns := setup(t, "success-policy")
767+
t.Cleanup(closeFn)
764768
for name, tc := range testCases {
765769
t.Run(name, func(t *testing.T) {
766770
resetMetrics()
767771
featuregatetesting.SetFeatureGateDuringTest(t, feature.DefaultFeatureGate, features.JobSuccessPolicy, tc.enableJobSuccessPolicy)
768772
featuregatetesting.SetFeatureGateDuringTest(t, feature.DefaultFeatureGate, features.JobBackoffLimitPerIndex, tc.enableBackoffLimitPerIndex)
769773

770-
closeFn, restConfig, clientSet, ns := setup(t, "simple")
771-
defer closeFn()
772774
ctx, cancel := startJobControllerAndWaitForCaches(t, restConfig)
773-
defer func() {
774-
cancel()
775-
}()
775+
t.Cleanup(cancel)
776+
776777
jobObj, err := createJobWithDefaults(ctx, clientSet, ns.Name, &tc.job)
777778
if err != nil {
778779
t.Fatalf("Error %v while creating the Job %q", err, jobObj.Name)
@@ -1180,11 +1181,12 @@ func TestBackoffLimitPerIndex_JobPodsCreatedWithExponentialBackoff(t *testing.T)
11801181
// are terminal. The fate of the Job is indicated by the interim Job conditions:
11811182
// FailureTarget, or SuccessCriteriaMet.
11821183
func TestDelayTerminalPhaseCondition(t *testing.T) {
1184+
const blockDeletionFinalizerForTest string = "fake.example.com/blockDeletion"
11831185
t.Cleanup(setDurationDuringTest(&jobcontroller.DefaultJobPodFailureBackOff, fastPodFailureBackoff))
11841186

11851187
podTemplateSpec := v1.PodTemplateSpec{
11861188
ObjectMeta: metav1.ObjectMeta{
1187-
Finalizers: []string{"fake.example.com/blockDeletion"},
1189+
Finalizers: []string{blockDeletionFinalizerForTest},
11881190
},
11891191
Spec: v1.PodSpec{
11901192
Containers: []v1.Container{
@@ -1515,6 +1517,9 @@ func TestDelayTerminalPhaseCondition(t *testing.T) {
15151517
},
15161518
},
15171519
}
1520+
1521+
closeFn, restConfig, clientSet, ns := setup(t, "delay-terminal-condition")
1522+
t.Cleanup(closeFn)
15181523
for name, test := range testCases {
15191524
t.Run(name, func(t *testing.T) {
15201525
resetMetrics()
@@ -1523,16 +1528,18 @@ func TestDelayTerminalPhaseCondition(t *testing.T) {
15231528
featuregatetesting.SetFeatureGateDuringTest(t, feature.DefaultFeatureGate, features.ElasticIndexedJob, true)
15241529
featuregatetesting.SetFeatureGateDuringTest(t, feature.DefaultFeatureGate, features.JobSuccessPolicy, test.enableJobSuccessPolicy)
15251530

1526-
closeFn, restConfig, clientSet, ns := setup(t, "delay-terminal-condition")
1527-
t.Cleanup(closeFn)
15281531
ctx, cancel := startJobControllerAndWaitForCaches(t, restConfig)
15291532
t.Cleanup(cancel)
15301533

15311534
jobObj, err := createJobWithDefaults(ctx, clientSet, ns.Name, &test.job)
15321535
if err != nil {
15331536
t.Fatalf("Error %q while creating the job %q", err, jobObj.Name)
15341537
}
1535-
t.Cleanup(func() { removePodsFinalizer(ctx, t, clientSet, ns.Name) })
1538+
t.Cleanup(func() {
1539+
if err := cleanUp(ctx, clientSet, jobObj, []string{blockDeletionFinalizerForTest, batchv1.JobTrackingFinalizer}); err != nil {
1540+
t.Fatalf("Failed cleanup: %v", err)
1541+
}
1542+
})
15361543
jobClient := clientSet.BatchV1().Jobs(jobObj.Namespace)
15371544

15381545
waitForPodsToBeActive(ctx, t, jobClient, *jobObj.Spec.Parallelism, jobObj)
@@ -1901,17 +1908,17 @@ func TestBackoffLimitPerIndex(t *testing.T) {
19011908
},
19021909
},
19031910
}
1911+
1912+
closeFn, restConfig, clientSet, ns := setup(t, "backoff-limit-per-index")
1913+
t.Cleanup(closeFn)
19041914
for name, test := range testCases {
19051915
t.Run(name, func(t *testing.T) {
19061916
resetMetrics()
19071917
featuregatetesting.SetFeatureGateDuringTest(t, feature.DefaultFeatureGate, features.JobBackoffLimitPerIndex, true)
19081918

1909-
closeFn, restConfig, clientSet, ns := setup(t, "simple")
1910-
defer closeFn()
19111919
ctx, cancel := startJobControllerAndWaitForCaches(t, restConfig)
1912-
defer func() {
1913-
cancel()
1914-
}()
1920+
t.Cleanup(cancel)
1921+
19151922
jobObj, err := createJobWithDefaults(ctx, clientSet, ns.Name, &test.job)
19161923
if err != nil {
19171924
t.Fatalf("Error %q while creating the job %q", err, jobObj.Name)
@@ -1972,6 +1979,7 @@ func TestBackoffLimitPerIndex(t *testing.T) {
19721979
// reconcile or skip reconciliation of the Job depending on the Job's managedBy
19731980
// field, and the enablement of the JobManagedBy feature gate.
19741981
func TestManagedBy(t *testing.T) {
1982+
const blockDeletionFinalizerForTest string = "fake.example.com/blockDeletion"
19751983
customControllerName := "example.com/custom-job-controller"
19761984
podTemplateSpec := v1.PodTemplateSpec{
19771985
Spec: v1.PodSpec{
@@ -2064,20 +2072,29 @@ func TestManagedBy(t *testing.T) {
20642072
},
20652073
},
20662074
}
2075+
2076+
closeFn, restConfig, clientSet, ns := setup(t, "managed-by")
2077+
t.Cleanup(closeFn)
20672078
for name, test := range testCases {
20682079
t.Run(name, func(t *testing.T) {
20692080
resetMetrics()
20702081
featuregatetesting.SetFeatureGateDuringTest(t, feature.DefaultFeatureGate, features.JobManagedBy, test.enableJobManagedBy)
20712082

2072-
closeFn, restConfig, clientSet, ns := setup(t, "managed-by")
2073-
defer closeFn()
20742083
ctx, cancel := startJobControllerAndWaitForCaches(t, restConfig)
2075-
defer cancel()
2084+
t.Cleanup(cancel)
2085+
20762086
jobObj, err := createJobWithDefaults(ctx, clientSet, ns.Name, &test.job)
20772087
if err != nil {
20782088
t.Fatalf("Error %v while creating the job %q", err, klog.KObj(jobObj))
20792089
}
20802090

2091+
t.Cleanup(func() {
2092+
// Wait for cleanup to finish to prevent this job from affecting metrics
2093+
if err := cleanUp(ctx, clientSet, jobObj, []string{blockDeletionFinalizerForTest, batchv1.JobTrackingFinalizer}); err != nil {
2094+
t.Fatalf("Failed cleanup: %v", err)
2095+
}
2096+
})
2097+
20812098
if test.wantReconciledByBuiltInController {
20822099
validateJobPodsStatus(ctx, t, clientSet, jobObj, podsByStatus{
20832100
Active: int(*jobObj.Spec.Parallelism),
@@ -2817,6 +2834,7 @@ func TestIndexedJob(t *testing.T) {
28172834
}
28182835

28192836
func TestJobPodReplacementPolicy(t *testing.T) {
2837+
const blockDeletionFinalizerForTest string = "fake.example.com/blockDeletion"
28202838
t.Cleanup(setDurationDuringTest(&jobcontroller.DefaultJobPodFailureBackOff, fastPodFailureBackoff))
28212839
indexedCompletion := batchv1.IndexedCompletion
28222840
nonIndexedCompletion := batchv1.NonIndexedCompletion
@@ -2847,7 +2865,7 @@ func TestJobPodReplacementPolicy(t *testing.T) {
28472865
CompletionMode: &indexedCompletion,
28482866
Template: v1.PodTemplateSpec{
28492867
ObjectMeta: metav1.ObjectMeta{
2850-
Finalizers: []string{"fake.example.com/blockDeletion"},
2868+
Finalizers: []string{blockDeletionFinalizerForTest},
28512869
},
28522870
},
28532871
},
@@ -2872,7 +2890,7 @@ func TestJobPodReplacementPolicy(t *testing.T) {
28722890
PodReplacementPolicy: podReplacementPolicy(batchv1.TerminatingOrFailed),
28732891
Template: v1.PodTemplateSpec{
28742892
ObjectMeta: metav1.ObjectMeta{
2875-
Finalizers: []string{"fake.example.com/blockDeletion"},
2893+
Finalizers: []string{blockDeletionFinalizerForTest},
28762894
},
28772895
},
28782896
},
@@ -2900,7 +2918,7 @@ func TestJobPodReplacementPolicy(t *testing.T) {
29002918
PodReplacementPolicy: podReplacementPolicy(batchv1.TerminatingOrFailed),
29012919
Template: v1.PodTemplateSpec{
29022920
ObjectMeta: metav1.ObjectMeta{
2903-
Finalizers: []string{"fake.example.com/blockDeletion"},
2921+
Finalizers: []string{blockDeletionFinalizerForTest},
29042922
},
29052923
},
29062924
},
@@ -2928,7 +2946,7 @@ func TestJobPodReplacementPolicy(t *testing.T) {
29282946
PodReplacementPolicy: podReplacementPolicy(batchv1.Failed),
29292947
Template: v1.PodTemplateSpec{
29302948
ObjectMeta: metav1.ObjectMeta{
2931-
Finalizers: []string{"fake.example.com/blockDeletion"},
2949+
Finalizers: []string{blockDeletionFinalizerForTest},
29322950
},
29332951
},
29342952
PodFailurePolicy: &batchv1.PodFailurePolicy{
@@ -2962,7 +2980,7 @@ func TestJobPodReplacementPolicy(t *testing.T) {
29622980
PodReplacementPolicy: podReplacementPolicy(batchv1.Failed),
29632981
Template: v1.PodTemplateSpec{
29642982
ObjectMeta: metav1.ObjectMeta{
2965-
Finalizers: []string{"fake.example.com/blockDeletion"},
2983+
Finalizers: []string{blockDeletionFinalizerForTest},
29662984
},
29672985
},
29682986
},
@@ -2990,7 +3008,7 @@ func TestJobPodReplacementPolicy(t *testing.T) {
29903008
PodReplacementPolicy: podReplacementPolicy(batchv1.Failed),
29913009
Template: v1.PodTemplateSpec{
29923010
ObjectMeta: metav1.ObjectMeta{
2993-
Finalizers: []string{"fake.example.com/blockDeletion"},
3011+
Finalizers: []string{blockDeletionFinalizerForTest},
29943012
},
29953013
},
29963014
},
@@ -3010,13 +3028,14 @@ func TestJobPodReplacementPolicy(t *testing.T) {
30103028
},
30113029
},
30123030
}
3031+
3032+
closeFn, restConfig, clientSet, ns := setup(t, "pod-replacement-policy")
3033+
t.Cleanup(closeFn)
30133034
for name, tc := range cases {
30143035
tc := tc
30153036
t.Run(name, func(t *testing.T) {
30163037
featuregatetesting.SetFeatureGateDuringTest(t, feature.DefaultFeatureGate, features.JobPodReplacementPolicy, tc.podReplacementPolicyEnabled)
30173038

3018-
closeFn, restConfig, clientSet, ns := setup(t, "pod-replacement-policy")
3019-
t.Cleanup(closeFn)
30203039
ctx, cancel := startJobControllerAndWaitForCaches(t, restConfig)
30213040
t.Cleanup(cancel)
30223041
resetMetrics()
@@ -3030,7 +3049,11 @@ func TestJobPodReplacementPolicy(t *testing.T) {
30303049
jobClient := clientSet.BatchV1().Jobs(jobObj.Namespace)
30313050

30323051
waitForPodsToBeActive(ctx, t, jobClient, 2, jobObj)
3033-
t.Cleanup(func() { removePodsFinalizer(ctx, t, clientSet, ns.Name) })
3052+
t.Cleanup(func() {
3053+
if err := cleanUp(ctx, clientSet, jobObj, []string{blockDeletionFinalizerForTest, batchv1.JobTrackingFinalizer}); err != nil {
3054+
t.Fatalf("Failed cleanup: %v", err)
3055+
}
3056+
})
30343057

30353058
deletePods(ctx, t, clientSet, ns.Name)
30363059

@@ -3221,11 +3244,11 @@ func TestElasticIndexedJob(t *testing.T) {
32213244
},
32223245
}
32233246

3247+
closeFn, restConfig, clientSet, ns := setup(t, "indexed")
3248+
t.Cleanup(closeFn)
32243249
for name, tc := range cases {
32253250
tc := tc
32263251
t.Run(name, func(t *testing.T) {
3227-
closeFn, restConfig, clientSet, ns := setup(t, "indexed")
3228-
defer closeFn()
32293252
ctx, cancel := startJobControllerAndWaitForCaches(t, restConfig)
32303253
defer cancel()
32313254
resetMetrics()
@@ -3362,7 +3385,7 @@ func BenchmarkLargeIndexedJob(b *testing.B) {
33623385
b.Fatalf("Failed to create Job: %v", err)
33633386
}
33643387
b.Cleanup(func() {
3365-
if err := cleanUp(ctx, clientSet, jobObj); err != nil {
3388+
if err := cleanUp(ctx, clientSet, jobObj, []string{batchv1.JobTrackingFinalizer}); err != nil {
33663389
b.Fatalf("Failed cleanup: %v", err)
33673390
}
33683391
})
@@ -3450,7 +3473,7 @@ func BenchmarkLargeFailureHandling(b *testing.B) {
34503473
b.Fatalf("Failed to create Job: %v", err)
34513474
}
34523475
b.Cleanup(func() {
3453-
if err := cleanUp(ctx, clientSet, jobObj); err != nil {
3476+
if err := cleanUp(ctx, clientSet, jobObj, []string{batchv1.JobTrackingFinalizer}); err != nil {
34543477
b.Fatalf("Failed cleanup: %v", err)
34553478
}
34563479
})
@@ -3484,8 +3507,11 @@ func BenchmarkLargeFailureHandling(b *testing.B) {
34843507
}
34853508
}
34863509

3487-
// cleanUp deletes all pods and the job
3488-
func cleanUp(ctx context.Context, clientSet clientset.Interface, jobObj *batchv1.Job) error {
3510+
// cleanUp removes the specified pod finalizers, then deletes all pods and the job.
3511+
func cleanUp(ctx context.Context, clientSet clientset.Interface, jobObj *batchv1.Job, podFinalizersToRemove []string) error {
3512+
if err := removePodsFinalizers(ctx, clientSet, jobObj.Namespace, podFinalizersToRemove); err != nil {
3513+
return err
3514+
}
34893515
// Clean up pods in pages, because DeleteCollection might timeout.
34903516
// #90743
34913517
for {
@@ -3505,14 +3531,18 @@ func cleanUp(ctx context.Context, clientSet clientset.Interface, jobObj *batchv1
35053531
return err
35063532
}
35073533
}
3508-
return clientSet.BatchV1().Jobs(jobObj.Namespace).Delete(ctx, jobObj.Name, metav1.DeleteOptions{})
3534+
3535+
// Set the propagation policy to background to ensure that the job doesn't receive any finalizer at deletion time
3536+
return clientSet.BatchV1().Jobs(jobObj.Namespace).Delete(ctx, jobObj.Name, metav1.DeleteOptions{
3537+
PropagationPolicy: ptr.To(metav1.DeletePropagationBackground),
3538+
})
35093539
}
35103540

35113541
func TestOrphanPodsFinalizersClearedWithGC(t *testing.T) {
3542+
closeFn, restConfig, clientSet, ns := setup(t, "orphan-pod-finalizers")
3543+
t.Cleanup(closeFn)
35123544
for _, policy := range []metav1.DeletionPropagation{metav1.DeletePropagationOrphan, metav1.DeletePropagationBackground, metav1.DeletePropagationForeground} {
35133545
t.Run(string(policy), func(t *testing.T) {
3514-
closeFn, restConfig, clientSet, ns := setup(t, "simple")
3515-
defer closeFn()
35163546
informerSet := informers.NewSharedInformerFactory(clientset.NewForConfigOrDie(restclient.AddUserAgent(restConfig, "controller-informers")), 0)
35173547
// Make the job controller significantly slower to trigger race condition.
35183548
restConfig.QPS = 1
@@ -3821,11 +3851,11 @@ func TestSuspendJob(t *testing.T) {
38213851
},
38223852
}
38233853

3854+
closeFn, restConfig, clientSet, ns := setup(t, "suspend")
3855+
t.Cleanup(closeFn)
38243856
for _, tc := range testCases {
38253857
name := fmt.Sprintf("feature=%v,create=%v,update=%v", tc.featureGate, tc.create.flag, tc.update.flag)
38263858
t.Run(name, func(t *testing.T) {
3827-
closeFn, restConfig, clientSet, ns := setup(t, "suspend")
3828-
defer closeFn()
38293859
ctx, cancel := startJobControllerAndWaitForCaches(t, restConfig)
38303860
defer cancel()
38313861
events, err := clientSet.EventsV1().Events(ns.Name).Watch(ctx, metav1.ListOptions{})
@@ -4341,7 +4371,7 @@ func getCompletionIndex(p *v1.Pod) (int, error) {
43414371

43424372
func createJobWithDefaults(ctx context.Context, clientSet clientset.Interface, ns string, jobObj *batchv1.Job) (*batchv1.Job, error) {
43434373
if jobObj.Name == "" {
4344-
jobObj.Name = "test-job"
4374+
jobObj.GenerateName = "test-job"
43454375
}
43464376
if len(jobObj.Spec.Template.Spec.Containers) == 0 {
43474377
jobObj.Spec.Template.Spec.Containers = []v1.Container{
@@ -4476,23 +4506,25 @@ func deletePods(ctx context.Context, t *testing.T, clientSet clientset.Interface
44764506
}
44774507
}
44784508

4479-
func removePodsFinalizer(ctx context.Context, t *testing.T, clientSet clientset.Interface, namespace string) {
4480-
t.Helper()
4509+
func removePodsFinalizers(ctx context.Context, clientSet clientset.Interface, namespace string, finalizersNames []string) error {
44814510
pods, err := clientSet.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{})
44824511
if err != nil {
4483-
t.Fatalf("Failed to list pods: %v", err)
4512+
return err
44844513
}
4485-
updatePod(ctx, t, clientSet, pods.Items, func(pod *v1.Pod) {
4486-
for i, finalizer := range pod.Finalizers {
4487-
if finalizer == "fake.example.com/blockDeletion" {
4488-
pod.Finalizers = append(pod.Finalizers[:i], pod.Finalizers[i+1:]...)
4514+
4515+
finalizersSet := sets.New(finalizersNames...)
4516+
return updatePod(ctx, clientSet, pods.Items, func(pod *v1.Pod) {
4517+
podFinalizers := []string{}
4518+
for _, podFinalizer := range pod.Finalizers {
4519+
if _, found := finalizersSet[podFinalizer]; !found {
4520+
podFinalizers = append(podFinalizers, podFinalizer)
44894521
}
44904522
}
4523+
pod.Finalizers = podFinalizers
44914524
})
44924525
}
44934526

4494-
func updatePod(ctx context.Context, t *testing.T, clientSet clientset.Interface, pods []v1.Pod, updateFunc func(*v1.Pod)) {
4495-
t.Helper()
4527+
func updatePod(ctx context.Context, clientSet clientset.Interface, pods []v1.Pod, updateFunc func(*v1.Pod)) error {
44964528
for _, val := range pods {
44974529
if err := retry.RetryOnConflict(retry.DefaultBackoff, func() error {
44984530
newPod, err := clientSet.CoreV1().Pods(val.Namespace).Get(ctx, val.Name, metav1.GetOptions{})
@@ -4503,9 +4535,10 @@ func updatePod(ctx context.Context, t *testing.T, clientSet clientset.Interface,
45034535
_, err = clientSet.CoreV1().Pods(val.Namespace).Update(ctx, newPod, metav1.UpdateOptions{})
45044536
return err
45054537
}); err != nil {
4506-
t.Fatalf("Failed to update pod %s: %v", val.Name, err)
4538+
return err
45074539
}
45084540
}
4541+
return nil
45094542
}
45104543

45114544
func failTerminatingPods(ctx context.Context, t *testing.T, clientSet clientset.Interface, namespace string) {

0 commit comments

Comments
 (0)