flakeguard: Do not fail when test skipped and fail fast if threshold=100% (#1282)

Author: lukaszcl

tools/flakeguard/cmd/run.go

@@ -19,7 +19,6 @@ var RunTestsCmd = &cobra.Command{
 		testPackagesArg, _ := cmd.Flags().GetStringSlice("test-packages")
 		runCount, _ := cmd.Flags().GetInt("run-count")
 		useRace, _ := cmd.Flags().GetBool("race")
-		failFast, _ := cmd.Flags().GetBool("fail-fast")
 		outputPath, _ := cmd.Flags().GetString("output-json")
 		threshold, _ := cmd.Flags().GetFloat64("threshold")
 
@@ -38,7 +37,7 @@ var RunTestsCmd = &cobra.Command{
 			Verbose:  true,
 			RunCount: runCount,
 			UseRace:  useRace,
-			FailFast: failFast,
+			FailFast: threshold == 1.0, // Fail test on first test run if threshold is 1.0
 		}
 
 		testResults, err := runner.RunTests(testPackages)
@@ -47,16 +46,20 @@ var RunTestsCmd = &cobra.Command{
 			os.Exit(1)
 		}
 
-		// Filter out failed tests based on the threshold
+		passedTests := reports.FilterPassedTests(testResults, threshold)
 		failedTests := reports.FilterFailedTests(testResults, threshold)
+		skippedTests := reports.FilterSkippedTests(testResults)
+
 		if len(failedTests) > 0 {
 			jsonData, err := json.MarshalIndent(failedTests, "", "  ")
 			if err != nil {
 				log.Fatalf("Error marshaling test results to JSON: %v", err)
 			}
-			fmt.Printf("Threshold for flaky tests: %.2f\n%d failed tests:\n%s\n", threshold, len(failedTests), string(jsonData))
+			fmt.Printf("PassRatio threshold for flaky tests: %.2f\n%d failed tests:\n%s\n", threshold, len(failedTests), string(jsonData))
 		}
 
+		fmt.Printf("Summary: %d passed, %d skipped, %d failed\n", len(passedTests), len(skippedTests), len(failedTests))
+
 		// Save the test results in JSON format
 		if outputPath != "" && len(testResults) > 0 {
 			jsonData, err := json.MarshalIndent(testResults, "", "  ")
@@ -70,11 +73,10 @@ var RunTestsCmd = &cobra.Command{
 		}
 
 		if len(failedTests) > 0 {
+			// Fail if any tests failed
 			os.Exit(1)
 		} else if len(testResults) == 0 {
 			fmt.Printf("No tests were run for the specified packages.\n")
-		} else {
-			fmt.Printf("All %d tests passed.\n", len(testResults))
 		}
 	},
 }

tools/flakeguard/reports/reports.go

@@ -6,15 +6,38 @@ type TestResult struct {
 	PassRatio   float64
 	Runs        int
 	Outputs     []string // Stores outputs for a test
+	Skipped     bool     // Indicates if the test was skipped
 }
 
 // FilterFailedTests returns a slice of TestResult where the pass ratio is below the specified threshold.
 func FilterFailedTests(results []TestResult, threshold float64) []TestResult {
 	var failedTests []TestResult
 	for _, result := range results {
-		if result.PassRatio < threshold {
+		if !result.Skipped && result.PassRatio < threshold {
 			failedTests = append(failedTests, result)
 		}
 	}
 	return failedTests
 }
+
+// FilterPassedTests returns a slice of TestResult where the tests passed and were not skipped.
+func FilterPassedTests(results []TestResult, threshold float64) []TestResult {
+	var passedTests []TestResult
+	for _, result := range results {
+		if !result.Skipped && result.PassRatio >= threshold {
+			passedTests = append(passedTests, result)
+		}
+	}
+	return passedTests
+}
+
+// FilterSkippedTests returns a slice of TestResult where the tests were skipped.
+func FilterSkippedTests(results []TestResult) []TestResult {
+	var skippedTests []TestResult
+	for _, result := range results {
+		if result.Skipped {
+			skippedTests = append(skippedTests, result)
+		}
+	}
+	return skippedTests
+}

tools/flakeguard/runner/runner.go

@@ -27,11 +27,14 @@ func (r *Runner) RunTests(packages []string) ([]reports.TestResult, error) {
 
 	for _, p := range packages {
 		for i := 0; i < r.RunCount; i++ {
-			jsonOutput, err := r.runTestPackage(p)
+			jsonOutput, passed, err := r.runTestPackage(p)
 			if err != nil {
 				return nil, fmt.Errorf("failed to run tests in package %s: %w", p, err)
 			}
 			jsonOutputs = append(jsonOutputs, jsonOutput)
+			if !passed && r.FailFast {
+				break
+			}
 		}
 	}
 
@@ -43,17 +46,12 @@ type exitCoder interface {
 }
 
 // runTestPackage executes the test command for a single test package.
-func (r *Runner) runTestPackage(testPackage string) ([]byte, error) {
+// It returns the command output, a boolean indicating success, and any error encountered.
+func (r *Runner) runTestPackage(testPackage string) ([]byte, bool, error) {
 	args := []string{"test", "-json", "-count=1"} // Enable JSON output
-	// if r.Count > 0 {
-	// 	args = append(args, fmt.Sprintf("-count=%d", r.Count))
-	// }
 	if r.UseRace {
 		args = append(args, "-race")
 	}
-	if r.FailFast {
-		args = append(args, "-failfast")
-	}
 	args = append(args, testPackage)
 
 	if r.Verbose {
@@ -70,12 +68,14 @@ func (r *Runner) runTestPackage(testPackage string) ([]byte, error) {
 	err := cmd.Run()
 	if err != nil {
 		var exErr exitCoder
+		// Check if the error is due to a non-zero exit code
 		if errors.As(err, &exErr) && exErr.ExitCode() == 0 {
-			return nil, fmt.Errorf("test command failed at %s: %w", testPackage, err)
+			return nil, false, fmt.Errorf("test command failed at %s: %w", testPackage, err)
 		}
+		return out.Bytes(), false, nil // Test failed
 	}
 
-	return out.Bytes(), nil
+	return out.Bytes(), true, nil // Test succeeded
 }
 
 // parseTestResults analyzes multiple JSON outputs from 'go test -json' commands to determine test results.
@@ -124,6 +124,9 @@ func parseTestResults(datas [][]byte) ([]reports.TestResult, error) {
 				result.Outputs = append(result.Outputs, entry.Output)
 			case "fail":
 				result.PassRatio = (result.PassRatio * float64(result.Runs-1)) / float64(result.Runs)
+			case "skip":
+				result.Skipped = true
+				result.Runs++
 			}
 		}