Skip to content

Commit a4da72d

Browse files
carretercarreter
authored
Add FailurePolicyAction unit tests (#967)
* Add FailurePolicyAction unit tests * Rebase on top of #958 * Typo * Remove unecessary test parameter
1 parent f20ff9f commit a4da72d

File tree

2 files changed

+208
-0
lines changed

2 files changed

+208
-0
lines changed

‎pkg/controllers/failure_policy_test.go‎

Lines changed: 202 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,12 +19,14 @@ import (
1919
"time"
2020

2121
"github.com/google/go-cmp/cmp"
22+
"github.com/google/go-cmp/cmp/cmpopts"
2223
"github.com/stretchr/testify/assert"
2324
batchv1 "k8s.io/api/batch/v1"
2425
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
2526
"k8s.io/utils/ptr"
2627

2728
jobset "sigs.k8s.io/jobset/api/jobset/v1alpha2"
29+
"sigs.k8s.io/jobset/pkg/constants"
2830
testutils "sigs.k8s.io/jobset/pkg/util/testing"
2931
)
3032

@@ -371,3 +373,203 @@ func TestFindFirstFailedPolicyRuleAndJob(t *testing.T) {
371373
})
372374
}
373375
}
376+
377+
func TestApplyFailurePolicyRuleAction(t *testing.T) {
378+
matchingFailedJob := jobWithFailedCondition("failed-job", time.Now())
379+
380+
testCases := []struct {
381+
name string
382+
jobSet *jobset.JobSet
383+
matchingFailedJob *batchv1.Job
384+
failurePolicyAction jobset.FailurePolicyAction
385+
expectedJobSetStatus jobset.JobSetStatus
386+
}{
387+
{
388+
name: "FailJobSet action",
389+
jobSet: testutils.MakeJobSet("test-js", "default").FailurePolicy(&jobset.FailurePolicy{}).Obj(),
390+
matchingFailedJob: matchingFailedJob,
391+
failurePolicyAction: jobset.FailJobSet,
392+
expectedJobSetStatus: jobset.JobSetStatus{
393+
TerminalState: string(jobset.JobSetFailed),
394+
Conditions: []metav1.Condition{
395+
{
396+
Type: string(jobset.JobSetFailed),
397+
Status: metav1.ConditionTrue,
398+
Reason: constants.FailJobSetActionReason,
399+
},
400+
},
401+
},
402+
},
403+
{
404+
name: "RestartJobSet when restarts < maxRestarts increments restarts count, counts towards max, and resets individualJobRecreates",
405+
jobSet: testutils.MakeJobSet("test-js", "default").FailurePolicy(&jobset.FailurePolicy{MaxRestarts: 5}).
406+
SetStatus(jobset.JobSetStatus{
407+
Restarts: 1,
408+
RestartsCountTowardsMax: 1,
409+
IndividualJobsStatus: []jobset.IndividualJobStatus{
410+
{
411+
Name: "other-failed-job",
412+
Recreates: 1,
413+
},
414+
},
415+
}).
416+
Obj(),
417+
matchingFailedJob: matchingFailedJob,
418+
failurePolicyAction: jobset.RestartJobSet,
419+
expectedJobSetStatus: jobset.JobSetStatus{
420+
Restarts: 2,
421+
RestartsCountTowardsMax: 2,
422+
IndividualJobsStatus: []jobset.IndividualJobStatus{
423+
{
424+
Name: "other-failed-job",
425+
Recreates: 0,
426+
},
427+
},
428+
},
429+
},
430+
{
431+
name: "RestartJobSet action when restarts >= maxRestarts fails the jobset",
432+
jobSet: testutils.MakeJobSet("test-js", "default").
433+
FailurePolicy(&jobset.FailurePolicy{MaxRestarts: 2}).
434+
SetStatus(jobset.JobSetStatus{
435+
Restarts: 2,
436+
RestartsCountTowardsMax: 2,
437+
}).
438+
Obj(),
439+
matchingFailedJob: matchingFailedJob,
440+
failurePolicyAction: jobset.RestartJobSet,
441+
expectedJobSetStatus: jobset.JobSetStatus{
442+
Restarts: 2,
443+
RestartsCountTowardsMax: 2,
444+
TerminalState: string(jobset.JobSetFailed),
445+
Conditions: []metav1.Condition{
446+
{
447+
Type: string(jobset.JobSetFailed),
448+
Status: metav1.ConditionTrue,
449+
Reason: constants.ReachedMaxRestartsReason,
450+
},
451+
},
452+
},
453+
},
454+
{
455+
name: "RestartJobSetAndIgnoreMaxRestarts action does not count toward max restarts and resets IndividualJobStatus.Recreates",
456+
jobSet: testutils.MakeJobSet("test-js", "default").
457+
FailurePolicy(&jobset.FailurePolicy{MaxRestarts: 1}).
458+
SetStatus(jobset.JobSetStatus{
459+
Restarts: 1,
460+
RestartsCountTowardsMax: 1,
461+
IndividualJobsStatus: []jobset.IndividualJobStatus{
462+
{
463+
Name: "other-failed-job",
464+
Recreates: 1,
465+
},
466+
},
467+
}).
468+
Obj(),
469+
matchingFailedJob: matchingFailedJob,
470+
failurePolicyAction: jobset.RestartJobSetAndIgnoreMaxRestarts,
471+
expectedJobSetStatus: jobset.JobSetStatus{
472+
Restarts: 2,
473+
RestartsCountTowardsMax: 1, // not incremented
474+
IndividualJobsStatus: []jobset.IndividualJobStatus{
475+
{
476+
Name: "other-failed-job",
477+
Recreates: 0,
478+
},
479+
},
480+
},
481+
},
482+
{
483+
name: "RecreateJob action when restarts < maxRestarts increments individualJobRecreates and counts toward max restarts",
484+
jobSet: testutils.MakeJobSet("test-js", "default").
485+
FailurePolicy(&jobset.FailurePolicy{MaxRestarts: 5}).
486+
SetStatus(jobset.JobSetStatus{
487+
RestartsCountTowardsMax: 1,
488+
IndividualJobsStatus: []jobset.IndividualJobStatus{
489+
{
490+
Name: "failed-job",
491+
Recreates: 1,
492+
},
493+
},
494+
}).
495+
Obj(),
496+
matchingFailedJob: matchingFailedJob,
497+
failurePolicyAction: jobset.RecreateJob,
498+
expectedJobSetStatus: jobset.JobSetStatus{
499+
RestartsCountTowardsMax: 2,
500+
IndividualJobsStatus: []jobset.IndividualJobStatus{
501+
{
502+
Name: "failed-job",
503+
Recreates: 2,
504+
},
505+
},
506+
},
507+
},
508+
{
509+
name: "RecreateJob action assumes individualJobStatus.Recreates is 0 when entry does not exist",
510+
jobSet: testutils.MakeJobSet("test-js", "default").
511+
FailurePolicy(&jobset.FailurePolicy{MaxRestarts: 5}).
512+
SetStatus(jobset.JobSetStatus{
513+
RestartsCountTowardsMax: 1,
514+
}).
515+
Obj(),
516+
matchingFailedJob: matchingFailedJob,
517+
failurePolicyAction: jobset.RecreateJob,
518+
expectedJobSetStatus: jobset.JobSetStatus{
519+
RestartsCountTowardsMax: 2,
520+
IndividualJobsStatus: []jobset.IndividualJobStatus{
521+
{
522+
Name: "failed-job",
523+
Recreates: 1,
524+
},
525+
},
526+
},
527+
},
528+
{
529+
name: "RecreateJob action when restarts >= maxRestarts fails jobset",
530+
jobSet: testutils.MakeJobSet("test-js", "default").
531+
FailurePolicy(&jobset.FailurePolicy{MaxRestarts: 2}).
532+
SetStatus(jobset.JobSetStatus{
533+
RestartsCountTowardsMax: 2,
534+
}).
535+
Obj(),
536+
matchingFailedJob: matchingFailedJob,
537+
failurePolicyAction: jobset.RecreateJob,
538+
expectedJobSetStatus: jobset.JobSetStatus{
539+
RestartsCountTowardsMax: 2,
540+
TerminalState: string(jobset.JobSetFailed),
541+
Conditions: []metav1.Condition{
542+
{
543+
Type: string(jobset.JobSetFailed),
544+
Status: metav1.ConditionTrue,
545+
Reason: constants.ReachedMaxRestartsReason,
546+
},
547+
},
548+
},
549+
},
550+
}
551+
552+
for _, tc := range testCases {
553+
t.Run(tc.name, func(t *testing.T) {
554+
updateStatusOpts := &statusUpdateOpts{}
555+
jobSetCopy := tc.jobSet.DeepCopy()
556+
err := applyFailurePolicyRuleAction(context.TODO(), jobSetCopy, tc.matchingFailedJob, updateStatusOpts, tc.failurePolicyAction)
557+
if err != nil {
558+
t.Fatalf("unexpected error: %v", err)
559+
}
560+
561+
if !updateStatusOpts.shouldUpdate {
562+
t.Fatalf("unexpected updateStatusOpts.shouldUpdate value: got %v, want true", updateStatusOpts.shouldUpdate)
563+
}
564+
565+
opts := []cmp.Option{
566+
cmpopts.IgnoreFields(metav1.Condition{}, "LastTransitionTime", "Message"),
567+
cmpopts.SortSlices(func(a, b metav1.Condition) bool { return a.Type < b.Type }),
568+
}
569+
570+
if diff := cmp.Diff(tc.expectedJobSetStatus, jobSetCopy.Status, opts...); diff != "" {
571+
t.Errorf("unexpected JobSetStatus value after applying failure policy rule action (+got/-want): %s", diff)
572+
}
573+
})
574+
}
575+
}

‎pkg/util/testing/wrappers.go‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,12 @@ func (j *JobSetWrapper) Conditions(conditions []metav1.Condition) *JobSetWrapper
6161
return j
6262
}
6363

64+
// SetStatus sets the value of jobSet.status
65+
func (j *JobSetWrapper) SetStatus(status jobset.JobSetStatus) *JobSetWrapper {
66+
j.Status = status
67+
return j
68+
}
69+
6470
// ManagedBy sets the value of jobSet.spec.managedBy
6571
func (j *JobSetWrapper) ManagedBy(managedBy string) *JobSetWrapper {
6672
j.Spec.ManagedBy = ptr.To(managedBy)

0 commit comments

Comments
 (0)