Adds Tests and Fixes Panic Detection for Flakeguard (#1367)

Author: kalverra

tools/flakeguard/.gitignore

@@ -0,0 +1,3 @@
+tmp_test_flaky_state
+test_results_*.json
+debug_outputs/

tools/flakeguard/Makefile

@@ -1,2 +1,9 @@
+.PHONY: test_unit
 test_unit:
-	go test -timeout 5m -json -cover -covermode=count -coverprofile=unit-test-coverage.out ./... 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci
+	go list ./... | grep -v 'example_test_package' | xargs go test -timeout 5m -json -cover -covermode=count -coverprofile=unit-test-coverage.out 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci
+
+.PHONY: test
+test:
+	go install github.com/gotesttools/gotestfmt/v2/cmd/gotestfmt@latest
+	set -euo pipefail
+	go list ./... | grep -v 'example_test_package' | xargs go test -json -cover -coverprofile unit-test-coverage.out -v 2>&1 | tee /tmp/gotest.log | gotestfmt

tools/flakeguard/README.md

@@ -20,7 +20,8 @@ go install github.com/smartcontractkit/chainlink-testing-framework/tools/flakegu
 
 ## Usage
 
-Flakeguard offers two main commands: 
+Flakeguard offers two main commands:
+
 - `find` identifies test packages affected by recent changes.
 - `run` executes tests multiple times to identify flaky tests
 

tools/flakeguard/cmd/aggregate_results.go

@@ -7,17 +7,18 @@ import (
 	"github.com/spf13/cobra"
 )
 
+var (
+	resultsFolderPath string
+	outputResultsPath string
+	outputLogsPath    string
+	maxPassRatio      float64
+	filterFailed      bool
+)
+
 var AggregateResultsCmd = &cobra.Command{
 	Use:   "aggregate-results",
 	Short: "Aggregate test results and optionally filter failed tests based on a threshold",
 	Run: func(cmd *cobra.Command, args []string) {
-		resultsFolderPath, _ := cmd.Flags().GetString("results-path")
-		outputResultsPath, _ := cmd.Flags().GetString("output-results")
-		outputLogsPath, _ := cmd.Flags().GetString("output-logs")
-		threshold, _ := cmd.Flags().GetFloat64("threshold")
-		minPassRatio, _ := cmd.Flags().GetFloat64("min-pass-ratio")
-		filterFailed, _ := cmd.Flags().GetBool("filter-failed")
-
 		// Aggregate all test results
 		allResults, err := reports.AggregateTestResults(resultsFolderPath)
 		if err != nil {
@@ -27,9 +28,9 @@ var AggregateResultsCmd = &cobra.Command{
 		var resultsToSave []reports.TestResult
 
 		if filterFailed {
-			// Filter to only include failed tests based on threshold and minPassRatio
+			// Filter to only include tests that failed below the threshold
 			for _, result := range allResults {
-				if result.PassRatio < threshold && result.PassRatio > minPassRatio && !result.Skipped {
+				if result.PassRatio < maxPassRatio && !result.Skipped {
 					resultsToSave = append(resultsToSave, result)
 				}
 			}
@@ -45,10 +46,9 @@ var AggregateResultsCmd = &cobra.Command{
 }
 
 func init() {
-	AggregateResultsCmd.Flags().String("results-path", "", "Path to the folder containing JSON test result files")
-	AggregateResultsCmd.Flags().String("output-results", "./results.json", "Path to output the aggregated or filtered test results in JSON format")
-	AggregateResultsCmd.Flags().String("output-logs", "", "Path to output the filtered test logs in JSON format")
-	AggregateResultsCmd.Flags().Float64("threshold", 0.8, "Threshold for considering a test as failed (used with --filter-failed)")
-	AggregateResultsCmd.Flags().Float64("min-pass-ratio", 0.001, "Minimum pass ratio for considering a test as flaky (used with --filter-failed)")
-	AggregateResultsCmd.Flags().Bool("filter-failed", false, "If true, filter and output only failed tests based on the threshold")
+	AggregateResultsCmd.Flags().StringVarP(&resultsFolderPath, "results-path", "p", "", "Path to the folder containing JSON test result files")
+	AggregateResultsCmd.Flags().StringVarP(&outputResultsPath, "output-results", "o", "./results.json", "Path to output the aggregated or filtered test results in JSON format")
+	AggregateResultsCmd.Flags().StringVarP(&outputLogsPath, "output-logs", "l", "", "Path to output the filtered test logs in JSON format")
+	AggregateResultsCmd.Flags().Float64VarP(&maxPassRatio, "max-pass-ratio", "m", 1.0, "The maximum (non-inclusive) pass ratio threshold for a test to be considered a failure. Any tests below this pass rate will be considered flaky.")
+	AggregateResultsCmd.Flags().BoolVarP(&filterFailed, "filter-failed", "f", false, "If true, filter and output only failed tests based on the max-pass-ratio threshold")
 }

tools/flakeguard/cmd/run.go

@@ -23,10 +23,9 @@ var RunTestsCmd = &cobra.Command{
 		runCount, _ := cmd.Flags().GetInt("run-count")
 		useRace, _ := cmd.Flags().GetBool("race")
 		outputPath, _ := cmd.Flags().GetString("output-json")
-		threshold, _ := cmd.Flags().GetFloat64("threshold")
+		maxPassRatio, _ := cmd.Flags().GetFloat64("max-pass-ratio")
 		skipTests, _ := cmd.Flags().GetStringSlice("skip-tests")
 		printFailedTests, _ := cmd.Flags().GetBool("print-failed-tests")
-		minPassRatio, _ := cmd.Flags().GetFloat64("min-pass-ratio")
 
 		// Check if project dependencies are correctly set up
 		if err := checkDependencies(projectPath); err != nil {
@@ -59,15 +58,15 @@ var RunTestsCmd = &cobra.Command{
 			os.Exit(1)
 		}
 
-		passedTests := reports.FilterPassedTests(testResults, threshold)
-		failedTests := reports.FilterFailedTests(testResults, threshold)
+		passedTests := reports.FilterPassedTests(testResults, maxPassRatio)
+		failedTests := reports.FilterFailedTests(testResults, maxPassRatio)
 		skippedTests := reports.FilterSkippedTests(testResults)
-		flakyTests := reports.FilterFlakyTests(testResults, minPassRatio, threshold)
+		flakyTests := reports.FilterFlakyTests(testResults, maxPassRatio)
 
 		// Print all failed tests including flaky tests
 		if len(failedTests) > 0 && printFailedTests {
-			fmt.Printf("MinPassRatio threshold for flaky tests: %.2f\n", minPassRatio)
-			fmt.Printf("PassRatio threshold for flaky tests: %.2f\n", threshold)
+			fmt.Printf("Maximum threshold for flaky tests: %.2f\n", maxPassRatio)
+			fmt.Printf("PassRatio threshold for flaky tests: %.2f\n", maxPassRatio)
 			fmt.Printf("%d failed tests:\n", len(failedTests))
 			reports.PrintTests(failedTests, os.Stdout)
 		}
@@ -80,7 +79,7 @@ var RunTestsCmd = &cobra.Command{
 			if err != nil {
 				log.Fatalf("Error marshaling test results to JSON: %v", err)
 			}
-			if err := os.WriteFile(outputPath, jsonData, 0644); err != nil {
+			if err := os.WriteFile(outputPath, jsonData, 0644); err != nil { //nolint:gosec
 				log.Fatalf("Error writing test results to file: %v", err)
 			}
 			fmt.Printf("All test results saved to %s\n", outputPath)
@@ -104,10 +103,9 @@ func init() {
 	RunTestsCmd.Flags().Bool("race", false, "Enable the race detector")
 	RunTestsCmd.Flags().Bool("fail-fast", false, "Stop on the first test failure")
 	RunTestsCmd.Flags().String("output-json", "", "Path to output the test results in JSON format")
-	RunTestsCmd.Flags().Float64("threshold", 0.8, "Threshold for considering a test as flaky")
 	RunTestsCmd.Flags().StringSlice("skip-tests", nil, "Comma-separated list of test names to skip from running")
 	RunTestsCmd.Flags().Bool("print-failed-tests", true, "Print failed test results to the console")
-	RunTestsCmd.Flags().Float64("min-pass-ratio", 0.001, "Minimum pass ratio for considering a test as flaky. Used to distinguish between tests that are truly flaky (with inconsistent results) and those that are consistently failing.")
+	RunTestsCmd.Flags().Float64("max-pass-ratio", 1.0, "The maximum (non-inclusive) pass ratio threshold for a test to be considered a failure. Any tests below this pass rate will be considered flaky.")
 }
 
 func checkDependencies(projectPath string) error {

tools/flakeguard/git/git_test.go

@@ -8,6 +8,8 @@ import (
 )
 
 func TestGetChangedGoPackagesFromDiff(t *testing.T) {
+	t.Parallel()
+
 	tests := []struct {
 		name        string
 		out         string

tools/flakeguard/reports/reports.go

@@ -9,49 +9,56 @@ import (
 	"path/filepath"
 	"sort"
 	"strings"
+	"time"
 )
 
+// TestResult represents the result of a single test being run through flakeguard.
 type TestResult struct {
 	TestName            string
 	TestPackage         string
-	Panicked            bool    // Indicates a test-level panic
-	PackagePanicked     bool    // Indicates a package-level panic
-	PassRatio           float64 // Pass ratio in decimal format like 0.5
-	PassRatioPercentage string  // Pass ratio in percentage format like "50%"
-	Skipped             bool    // Indicates if the test was skipped
-	Runs                int
-	Outputs             []string  // Stores outputs for a test
-	Durations           []float64 // Stores elapsed time in seconds for each run of the test
-	PackageOutputs      []string  // Stores package-level outputs
+	Panicked            bool            // Indicates a test-level panic
+	Skipped             bool            // Indicates if the test was skipped
+	PackagePanicked     bool            // Indicates a package-level panic
+	PassRatio           float64         // Pass ratio in decimal format like 0.5
+	PassRatioPercentage string          // Pass ratio in percentage format like "50%"
+	Runs                int             // Count of how many times the test was run
+	Failures            int             // Count of how many times the test failed
+	Successes           int             // Count of how many times the test passed
+	Panics              int             // Count of how many times the test panicked
+	Races               int             // Count of how many times the test encountered a data race
+	Skips               int             // Count of how many times the test was skipped
+	Outputs             []string        `json:"outputs,omitempty"` // Stores outputs for a test
+	Durations           []time.Duration // Stores elapsed time for each run of the test
+	PackageOutputs      []string        `json:"package_outputs,omitempty"` // Stores package-level outputs
 }
 
 // FilterFailedTests returns a slice of TestResult where the pass ratio is below the specified threshold.
-func FilterFailedTests(results []TestResult, threshold float64) []TestResult {
+func FilterFailedTests(results []TestResult, maxPassRatio float64) []TestResult {
 	var failedTests []TestResult
 	for _, result := range results {
-		if !result.Skipped && result.PassRatio < threshold {
+		if !result.Skipped && result.PassRatio < maxPassRatio {
 			failedTests = append(failedTests, result)
 		}
 	}
 	return failedTests
 }
 
 // FilterFlakyTests returns a slice of TestResult where the pass ratio is between the min pass ratio and the threshold.
-func FilterFlakyTests(testResults []TestResult, minPassRatio, threshold float64) []TestResult {
+func FilterFlakyTests(testResults []TestResult, maxPassRatio float64) []TestResult {
 	var flakyTests []TestResult
 	for _, test := range testResults {
-		if test.PassRatio >= minPassRatio && test.PassRatio < threshold && !test.Skipped {
+		if test.PassRatio < maxPassRatio && !test.Skipped {
 			flakyTests = append(flakyTests, test)
 		}
 	}
 	return flakyTests
 }
 
 // FilterPassedTests returns a slice of TestResult where the tests passed and were not skipped.
-func FilterPassedTests(results []TestResult, threshold float64) []TestResult {
+func FilterPassedTests(results []TestResult, maxPassRatio float64) []TestResult {
 	var passedTests []TestResult
 	for _, result := range results {
-		if !result.Skipped && result.PassRatio >= threshold {
+		if !result.Skipped && result.PassRatio >= maxPassRatio {
 			passedTests = append(passedTests, result)
 		}
 	}
@@ -143,9 +150,14 @@ func PrintTests(tests []TestResult, w io.Writer) {
 		fmt.Fprintf(w, "PassRatio: %.2f\n", test.PassRatio)
 		fmt.Fprintf(w, "Skipped: %v\n", test.Skipped)
 		fmt.Fprintf(w, "Runs: %d\n", test.Runs)
+		fmt.Fprintf(w, "Successes: %d\n", test.Successes)
+		fmt.Fprintf(w, "Failures: %d\n", test.Failures)
+		fmt.Fprintf(w, "Panics: %d\n", test.Panics)
+		fmt.Fprintf(w, "Races: %d\n", test.Races)
+		fmt.Fprintf(w, "Skips: %d\n", test.Skips)
 		durationsStr := make([]string, len(test.Durations))
 		for i, duration := range test.Durations {
-			durationsStr[i] = fmt.Sprintf("%.2fs", duration)
+			durationsStr[i] = duration.String()
 		}
 		fmt.Fprintf(w, "Durations: %s\n", strings.Join(durationsStr, ", "))
 		fmt.Fprintf(w, "Outputs:\n%s\n", strings.Join(test.Outputs, ""))
@@ -171,41 +183,21 @@ func SaveFilteredResultsAndLogs(outputResultsPath, outputLogsPath string, failed
 	}
 }
 
-// Helper function to save results to JSON file
+// Helper function to save results to JSON file without outputs
 func saveResults(filePath string, results []TestResult) error {
-	// Define a struct type without Outputs and PackageOutputs
-	type filteredTestResult struct {
-		TestName            string
-		TestPackage         string
-		Panicked            bool
-		PackagePanicked     bool
-		PassRatio           float64
-		PassRatioPercentage string
-		Skipped             bool
-		Runs                int
-		Durations           []float64
-	}
-
-	var filteredResults []filteredTestResult
+	var filteredResults []TestResult
 	for _, r := range results {
-		filteredResults = append(filteredResults, filteredTestResult{
-			TestName:            r.TestName,
-			TestPackage:         r.TestPackage,
-			Panicked:            r.Panicked,
-			PackagePanicked:     r.PackagePanicked,
-			PassRatio:           r.PassRatio,
-			PassRatioPercentage: r.PassRatioPercentage,
-			Skipped:             r.Skipped,
-			Runs:                r.Runs,
-			Durations:           r.Durations,
-		})
+		filteredResult := r
+		filteredResult.Outputs = nil
+		filteredResult.PackageOutputs = nil
+		filteredResults = append(filteredResults, filteredResult)
 	}
 
 	data, err := json.MarshalIndent(filteredResults, "", "  ")
 	if err != nil {
 		return fmt.Errorf("error marshaling results: %v", err)
 	}
-	return os.WriteFile(filePath, data, 0644)
+	return os.WriteFile(filePath, data, 0644) //nolint:gosec
 }
 
 // Helper function to save test names, packages, and outputs to JSON file
@@ -233,5 +225,5 @@ func saveTestOutputs(filePath string, results []TestResult) error {
 	if err != nil {
 		return fmt.Errorf("error marshaling outputs: %v", err)
 	}
-	return os.WriteFile(filePath, data, 0644)
+	return os.WriteFile(filePath, data, 0644) //nolint:gosec
 }