Skip to content

Commit e23466e

Browse files
authored
flakeguard: Do not fail when test skipped and fail fast if threshold=100% (#1282)
1 parent 97b9e51 commit e23466e

File tree

3 files changed

+45
-17
lines changed

3 files changed

+45
-17
lines changed

tools/flakeguard/cmd/run.go

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,6 @@ var RunTestsCmd = &cobra.Command{
1919
testPackagesArg, _ := cmd.Flags().GetStringSlice("test-packages")
2020
runCount, _ := cmd.Flags().GetInt("run-count")
2121
useRace, _ := cmd.Flags().GetBool("race")
22-
failFast, _ := cmd.Flags().GetBool("fail-fast")
2322
outputPath, _ := cmd.Flags().GetString("output-json")
2423
threshold, _ := cmd.Flags().GetFloat64("threshold")
2524

@@ -38,7 +37,7 @@ var RunTestsCmd = &cobra.Command{
3837
Verbose: true,
3938
RunCount: runCount,
4039
UseRace: useRace,
41-
FailFast: failFast,
40+
FailFast: threshold == 1.0, // Fail test on first test run if threshold is 1.0
4241
}
4342

4443
testResults, err := runner.RunTests(testPackages)
@@ -47,16 +46,20 @@ var RunTestsCmd = &cobra.Command{
4746
os.Exit(1)
4847
}
4948

50-
// Filter out failed tests based on the threshold
49+
passedTests := reports.FilterPassedTests(testResults, threshold)
5150
failedTests := reports.FilterFailedTests(testResults, threshold)
51+
skippedTests := reports.FilterSkippedTests(testResults)
52+
5253
if len(failedTests) > 0 {
5354
jsonData, err := json.MarshalIndent(failedTests, "", " ")
5455
if err != nil {
5556
log.Fatalf("Error marshaling test results to JSON: %v", err)
5657
}
57-
fmt.Printf("Threshold for flaky tests: %.2f\n%d failed tests:\n%s\n", threshold, len(failedTests), string(jsonData))
58+
fmt.Printf("PassRatio threshold for flaky tests: %.2f\n%d failed tests:\n%s\n", threshold, len(failedTests), string(jsonData))
5859
}
5960

61+
fmt.Printf("Summary: %d passed, %d skipped, %d failed\n", len(passedTests), len(skippedTests), len(failedTests))
62+
6063
// Save the test results in JSON format
6164
if outputPath != "" && len(testResults) > 0 {
6265
jsonData, err := json.MarshalIndent(testResults, "", " ")
@@ -70,11 +73,10 @@ var RunTestsCmd = &cobra.Command{
7073
}
7174

7275
if len(failedTests) > 0 {
76+
// Fail if any tests failed
7377
os.Exit(1)
7478
} else if len(testResults) == 0 {
7579
fmt.Printf("No tests were run for the specified packages.\n")
76-
} else {
77-
fmt.Printf("All %d tests passed.\n", len(testResults))
7880
}
7981
},
8082
}

tools/flakeguard/reports/reports.go

Lines changed: 24 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,15 +6,38 @@ type TestResult struct {
66
PassRatio float64
77
Runs int
88
Outputs []string // Stores outputs for a test
9+
Skipped bool // Indicates if the test was skipped
910
}
1011

1112
// FilterFailedTests returns a slice of TestResult where the pass ratio is below the specified threshold.
1213
func FilterFailedTests(results []TestResult, threshold float64) []TestResult {
1314
var failedTests []TestResult
1415
for _, result := range results {
15-
if result.PassRatio < threshold {
16+
if !result.Skipped && result.PassRatio < threshold {
1617
failedTests = append(failedTests, result)
1718
}
1819
}
1920
return failedTests
2021
}
22+
23+
// FilterPassedTests returns a slice of TestResult where the tests passed and were not skipped.
24+
func FilterPassedTests(results []TestResult, threshold float64) []TestResult {
25+
var passedTests []TestResult
26+
for _, result := range results {
27+
if !result.Skipped && result.PassRatio >= threshold {
28+
passedTests = append(passedTests, result)
29+
}
30+
}
31+
return passedTests
32+
}
33+
34+
// FilterSkippedTests returns a slice of TestResult where the tests were skipped.
35+
func FilterSkippedTests(results []TestResult) []TestResult {
36+
var skippedTests []TestResult
37+
for _, result := range results {
38+
if result.Skipped {
39+
skippedTests = append(skippedTests, result)
40+
}
41+
}
42+
return skippedTests
43+
}

tools/flakeguard/runner/runner.go

Lines changed: 13 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -27,11 +27,14 @@ func (r *Runner) RunTests(packages []string) ([]reports.TestResult, error) {
2727

2828
for _, p := range packages {
2929
for i := 0; i < r.RunCount; i++ {
30-
jsonOutput, err := r.runTestPackage(p)
30+
jsonOutput, passed, err := r.runTestPackage(p)
3131
if err != nil {
3232
return nil, fmt.Errorf("failed to run tests in package %s: %w", p, err)
3333
}
3434
jsonOutputs = append(jsonOutputs, jsonOutput)
35+
if !passed && r.FailFast {
36+
break
37+
}
3538
}
3639
}
3740

@@ -43,17 +46,12 @@ type exitCoder interface {
4346
}
4447

4548
// runTestPackage executes the test command for a single test package.
46-
func (r *Runner) runTestPackage(testPackage string) ([]byte, error) {
49+
// It returns the command output, a boolean indicating success, and any error encountered.
50+
func (r *Runner) runTestPackage(testPackage string) ([]byte, bool, error) {
4751
args := []string{"test", "-json", "-count=1"} // Enable JSON output
48-
// if r.Count > 0 {
49-
// args = append(args, fmt.Sprintf("-count=%d", r.Count))
50-
// }
5152
if r.UseRace {
5253
args = append(args, "-race")
5354
}
54-
if r.FailFast {
55-
args = append(args, "-failfast")
56-
}
5755
args = append(args, testPackage)
5856

5957
if r.Verbose {
@@ -70,12 +68,14 @@ func (r *Runner) runTestPackage(testPackage string) ([]byte, error) {
7068
err := cmd.Run()
7169
if err != nil {
7270
var exErr exitCoder
71+
// Check if the error is due to a non-zero exit code
7372
if errors.As(err, &exErr) && exErr.ExitCode() == 0 {
74-
return nil, fmt.Errorf("test command failed at %s: %w", testPackage, err)
73+
return nil, false, fmt.Errorf("test command failed at %s: %w", testPackage, err)
7574
}
75+
return out.Bytes(), false, nil // Test failed
7676
}
7777

78-
return out.Bytes(), nil
78+
return out.Bytes(), true, nil // Test succeeded
7979
}
8080

8181
// parseTestResults analyzes multiple JSON outputs from 'go test -json' commands to determine test results.
@@ -124,6 +124,9 @@ func parseTestResults(datas [][]byte) ([]reports.TestResult, error) {
124124
result.Outputs = append(result.Outputs, entry.Output)
125125
case "fail":
126126
result.PassRatio = (result.PassRatio * float64(result.Runs-1)) / float64(result.Runs)
127+
case "skip":
128+
result.Skipped = true
129+
result.Runs++
127130
}
128131
}
129132

0 commit comments

Comments
 (0)