Skip to content

Commit 1cdb40b

Browse files
k8s-ci-robotk8s-ci-robot
authored
Merge pull request #545 from faganihajizada/fix/skip-failfast-472
fix: exclude skipped tests from fail-fast
2 parents f641d15 + c2d895c commit 1cdb40b

File tree

3 files changed

+165
-2
lines changed

3 files changed

+165
-2
lines changed

‎docs/fail-fast-mode.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,8 @@ There are certain caveats to how this feature works.
2626
1. The `fail-fast` mode doesn't work in conjunction with the `parallel` test mode
2727
2. Test developers have to explicitly invoke the `t.Fail()` or `t.FailNow()` handlers in the assessment to inform
2828
the framework that the fail-fast mode needs to be triggered
29+
3. Skipped tests (via `t.Skip()` or `--skip-assessment` regex) do **not** trigger fail-fast behavior -
30+
subsequent assessments and features will continue to execute normally
2931

3032
## Example Assessment
3133
Below section shows a simple example of how the feature can be leveraged in the assessment. This should be combined with `--fail-fast` argument while invoking the test to leverage the full feature.

‎pkg/env/env.go‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -507,8 +507,12 @@ func (e *testEnv) execFeature(ctx context.Context, t *testing.T, f types.Feature
507507
// shouldFailNow catches whether t.FailNow() is called in the assessment.
508508
// If it is, we won't proceed with the next assessment.
509509
var shouldFailNow bool
510+
var wasSkipped bool
510511
t.Run(assessName, func(internalT *testing.T) {
511512
internalT.Helper()
513+
defer func() {
514+
wasSkipped = internalT.Skipped()
515+
}()
512516
skipped, message := e.requireAssessmentProcessing(assess, i+1)
513517
if skipped {
514518
internalT.Skip(message)
@@ -524,8 +528,8 @@ func (e *testEnv) execFeature(ctx context.Context, t *testing.T, f types.Feature
524528
// - a t.FailNow() invocation
525529
// - a `t.Fail()` or `t.Failed()` invocation
526530
// In one of those cases, we need to track that and stop the next set of assessment in the feature
527-
// under test from getting executed.
528-
if shouldFailNow || (e.cfg.FailFast() && t.Failed()) {
531+
// under test from getting executed. Skipped tests should not trigger this.
532+
if !wasSkipped && (shouldFailNow || (e.cfg.FailFast() && t.Failed())) {
529533
failed = true
530534
break
531535
}

‎pkg/env/env_test.go‎

Lines changed: 157 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -942,3 +942,160 @@ func getFeaturesForTest() []features.Feature {
942942
}).Feature()
943943
return []features.Feature{f1, f2}
944944
}
945+
946+
// TestEnv_FailFast_WithSkip tests that skip does not trigger fail-fast behavior.
947+
func TestEnv_FailFast_WithSkip(t *testing.T) {
948+
tests := []struct {
949+
name string
950+
setup func(*testing.T) []string
951+
expected []string
952+
}{
953+
{
954+
name: "skip via skip-assessment flag with fail-fast should continue",
955+
expected: []string{
956+
"assess-2",
957+
},
958+
setup: func(t *testing.T) (val []string) {
959+
// Configure fail-fast and skip first assessment via regex
960+
env := NewWithConfig(envconf.New().WithFailFast().WithSkipAssessmentRegex("assess-1"))
961+
f := features.New("test-feat").
962+
Assess("assess-1", func(ctx context.Context, t *testing.T, _ *envconf.Config) context.Context {
963+
val = append(val, "assess-1")
964+
return ctx
965+
}).
966+
Assess("assess-2", func(ctx context.Context, t *testing.T, _ *envconf.Config) context.Context {
967+
val = append(val, "assess-2")
968+
return ctx
969+
})
970+
_ = env.Test(t, f.Feature())
971+
return
972+
},
973+
},
974+
{
975+
name: "skip via t.Skip in assessment with fail-fast should continue",
976+
expected: []string{
977+
"assess-2",
978+
},
979+
setup: func(t *testing.T) (val []string) {
980+
// Configure fail-fast, user calls t.Skip() in first assessment
981+
env := NewWithConfig(envconf.New().WithFailFast())
982+
f := features.New("test-feat").
983+
Assess("assess-1", func(ctx context.Context, t *testing.T, _ *envconf.Config) context.Context {
984+
t.Skip("skipping this assessment")
985+
val = append(val, "assess-1")
986+
return ctx
987+
}).
988+
Assess("assess-2", func(ctx context.Context, t *testing.T, _ *envconf.Config) context.Context {
989+
val = append(val, "assess-2")
990+
return ctx
991+
})
992+
_ = env.Test(t, f.Feature())
993+
return
994+
},
995+
},
996+
{
997+
name: "multiple skips with fail-fast should continue to all non-skipped",
998+
expected: []string{
999+
"assess-2",
1000+
"assess-4",
1001+
},
1002+
setup: func(t *testing.T) (val []string) {
1003+
// Configure fail-fast, skip assessments 1 and 3
1004+
env := NewWithConfig(envconf.New().WithFailFast().WithSkipAssessmentRegex("assess-[13]"))
1005+
f := features.New("test-feat").
1006+
Assess("assess-1", func(ctx context.Context, t *testing.T, _ *envconf.Config) context.Context {
1007+
val = append(val, "assess-1")
1008+
return ctx
1009+
}).
1010+
Assess("assess-2", func(ctx context.Context, t *testing.T, _ *envconf.Config) context.Context {
1011+
val = append(val, "assess-2")
1012+
return ctx
1013+
}).
1014+
Assess("assess-3", func(ctx context.Context, t *testing.T, _ *envconf.Config) context.Context {
1015+
val = append(val, "assess-3")
1016+
return ctx
1017+
}).
1018+
Assess("assess-4", func(ctx context.Context, t *testing.T, _ *envconf.Config) context.Context {
1019+
val = append(val, "assess-4")
1020+
return ctx
1021+
})
1022+
_ = env.Test(t, f.Feature())
1023+
return
1024+
},
1025+
},
1026+
{
1027+
name: "pass without fail-fast continues normally",
1028+
expected: []string{
1029+
"assess-1",
1030+
"assess-2",
1031+
},
1032+
setup: func(t *testing.T) (val []string) {
1033+
// No fail-fast, both assessments should run
1034+
env := newTestEnv()
1035+
f := features.New("test-feat").
1036+
Assess("assess-1", func(ctx context.Context, t *testing.T, _ *envconf.Config) context.Context {
1037+
val = append(val, "assess-1")
1038+
return ctx
1039+
}).
1040+
Assess("assess-2", func(ctx context.Context, t *testing.T, _ *envconf.Config) context.Context {
1041+
val = append(val, "assess-2")
1042+
return ctx
1043+
})
1044+
_ = env.Test(t, f.Feature())
1045+
return
1046+
},
1047+
},
1048+
{
1049+
name: "skip in first feature with fail-fast should continue to second feature",
1050+
expected: []string{
1051+
"feat1-assess-2",
1052+
"feat2-assess-1",
1053+
"feat2-assess-2",
1054+
},
1055+
setup: func(t *testing.T) (val []string) {
1056+
// Configure fail-fast, first feature has skipped assessment
1057+
env := NewWithConfig(envconf.New().WithFailFast())
1058+
feat1 := features.New("skip").
1059+
Assess("Assess 1", func(ctx context.Context, t *testing.T, _ *envconf.Config) context.Context {
1060+
t.Skipf("skipping Assess 1")
1061+
val = append(val, "feat1-assess-1")
1062+
return ctx
1063+
}).
1064+
Assess("Assess 2", func(ctx context.Context, t *testing.T, _ *envconf.Config) context.Context {
1065+
val = append(val, "feat1-assess-2")
1066+
return ctx
1067+
}).
1068+
Feature()
1069+
1070+
feat2 := features.New("succeed").
1071+
Assess("Assess 1", func(ctx context.Context, t *testing.T, _ *envconf.Config) context.Context {
1072+
val = append(val, "feat2-assess-1")
1073+
return ctx
1074+
}).
1075+
Assess("Assess 2", func(ctx context.Context, t *testing.T, _ *envconf.Config) context.Context {
1076+
val = append(val, "feat2-assess-2")
1077+
return ctx
1078+
}).
1079+
Feature()
1080+
1081+
_ = env.Test(t, feat1, feat2)
1082+
return
1083+
},
1084+
},
1085+
}
1086+
1087+
for _, test := range tests {
1088+
t.Run(test.name, func(t *testing.T) {
1089+
result := test.setup(t)
1090+
if len(test.expected) != len(result) {
1091+
t.Fatalf("Expected:\n%v but got result:\n%v", test.expected, result)
1092+
}
1093+
for i := range test.expected {
1094+
if result[i] != test.expected[i] {
1095+
t.Errorf("Expected:\n%v but got result:\n%v", test.expected, result)
1096+
break
1097+
}
1098+
}
1099+
})
1100+
}
1101+
}

0 commit comments

Comments
 (0)