Merge branch 'main' into tt-1806-labelling-readme

Author: Tofel

tools/flakeguard/.gitignore

@@ -1,3 +1,6 @@
-tmp_test_flaky_state
+tmp_test_flaky*
 test_results_*.json
-debug_outputs/
+debug_outputs
+example_results/
+example*.json
+example*.md

tools/flakeguard/Makefile

@@ -7,3 +7,30 @@ test:
 	go install github.com/gotesttools/gotestfmt/v2/cmd/gotestfmt@latest
 	set -euo pipefail
 	go list ./... | grep -v 'example_test_package' | xargs go test -json -cover -coverprofile unit-test-coverage.out -v 2>&1 | tee /tmp/gotest.log | gotestfmt
+
+.PHONY: example
+example:
+	rm -rf example_results
+	mkdir -p example_results
+	- go run . run --project-path=./runner --test-packages=./example_test_package --run-count=5 --skip-tests=Panic,Timeout --max-pass-ratio=1 --race=false --output-json=example_results/example_run_1.json
+	- go run . run --project-path=./runner --test-packages=./example_test_package --run-count=5 --skip-tests=Panic,Timeout --max-pass-ratio=1 --race=false --output-json=example_results/example_run_2.json
+	- go run . run --project-path=./runner --test-packages=./example_test_package --run-count=5 --skip-tests=Panic,Timeout --max-pass-ratio=1 --race=false --output-json=example_results/example_run_3.json
+	go run . aggregate-results --results-path ./example_results --output-results ./example_results/all_tests_example.json
+
+.PHONY: example_flaky_panic
+example_flaky_panic:
+	rm -rf example_results
+	mkdir -p example_results
+	- go run . run --project-path=./runner --test-packages=./example_test_package --run-count=5 --skip-tests=TestPanic --max-pass-ratio=1 --race=false --output-json=example_results/example_run_1.json
+	- go run . run --project-path=./runner --test-packages=./example_test_package --run-count=5 --skip-tests=TestPanic --max-pass-ratio=1 --race=false --output-json=example_results/example_run_2.json
+	- go run . run --project-path=./runner --test-packages=./example_test_package --run-count=5 --skip-tests=TestPanic --max-pass-ratio=1 --race=false --output-json=example_results/example_run_3.json
+	go run . aggregate-results --results-path ./example_results --output-results ./example_results/all_tests_example.json
+
+.PHONY: example_timeout
+example_timeout:
+	rm -rf example_results
+	mkdir -p example_results
+	- go run . run --project-path=./runner --test-packages=./example_test_package --run-count=5 --select-tests=TestTimeout --timeout=1s --max-pass-ratio=1 --race=false --output-json=example_results/example_run_1.json
+	- go run . run --project-path=./runner --test-packages=./example_test_package --run-count=5 --select-tests=TestTimeout --timeout=1s --max-pass-ratio=1 --race=false --output-json=example_results/example_run_2.json
+	- go run . run --project-path=./runner --test-packages=./example_test_package --run-count=5 --select-tests=TestTimeout --timeout=1s --max-pass-ratio=1 --race=false --output-json=example_results/example_run_3.json
+	go run . aggregate-results --results-path ./example_results --output-results ./example_results/all_tests_example.json

tools/flakeguard/README.md

@@ -30,3 +30,13 @@ Run with `--help` to see all flags for the commands.
 ### JSON Output
 
 Both `find` and `run` commands support JSON output `--json`, making it easy to integrate Flakeguard with CI/CD pipelines and reporting tools.
+
+### Example Run
+
+You can find example usage and see outputs with:
+
+```sh
+make example       # Run an example flow of running tests and aggregating results
+make example_panic # Run example flow with panicking tests
+ls example_results # See results of each run and aggregation
+```

tools/flakeguard/cmd/aggregate_results.go

@@ -18,9 +18,8 @@ var (
 var AggregateResultsCmd = &cobra.Command{
 	Use:   "aggregate-results",
 	Short: "Aggregate test results and optionally filter failed tests based on a threshold",
-	Run: func(cmd *cobra.Command, args []string) {
-		// Aggregate all test results
-		allResults, err := reports.AggregateTestResults(resultsFolderPath)
+	RunE: func(cmd *cobra.Command, args []string) error {
+		allReport, err := reports.AggregateTestResults(resultsFolderPath)
 		if err != nil {
 			log.Fatalf("Error aggregating results: %v", err)
 		}
@@ -29,25 +28,27 @@ var AggregateResultsCmd = &cobra.Command{
 
 		if filterFailed {
 			// Filter to only include tests that failed below the threshold
-			for _, result := range allResults {
+			for _, result := range allReport.Results {
 				if result.PassRatio < maxPassRatio && !result.Skipped {
 					resultsToSave = append(resultsToSave, result)
 				}
 			}
 		} else {
-			resultsToSave = allResults
+			resultsToSave = allReport.Results
 		}
+		allReport.Results = resultsToSave
 
 		// Output results to JSON files
 		if len(resultsToSave) > 0 {
-			reports.SaveFilteredResultsAndLogs(outputResultsPath, outputLogsPath, resultsToSave)
+			return reports.SaveFilteredResultsAndLogs(outputResultsPath, outputLogsPath, allReport)
 		}
+		return nil
 	},
 }
 
 func init() {
 	AggregateResultsCmd.Flags().StringVarP(&resultsFolderPath, "results-path", "p", "", "Path to the folder containing JSON test result files")
-	AggregateResultsCmd.Flags().StringVarP(&outputResultsPath, "output-results", "o", "./results.json", "Path to output the aggregated or filtered test results in JSON format")
+	AggregateResultsCmd.Flags().StringVarP(&outputResultsPath, "output-results", "o", "./results", "Path to output the aggregated or filtered test results in JSON and markdown format")
 	AggregateResultsCmd.Flags().StringVarP(&outputLogsPath, "output-logs", "l", "", "Path to output the filtered test logs in JSON format")
 	AggregateResultsCmd.Flags().Float64VarP(&maxPassRatio, "max-pass-ratio", "m", 1.0, "The maximum (non-inclusive) pass ratio threshold for a test to be considered a failure. Any tests below this pass rate will be considered flaky.")
 	AggregateResultsCmd.Flags().BoolVarP(&filterFailed, "filter-failed", "f", false, "If true, filter and output only failed tests based on the max-pass-ratio threshold")

tools/flakeguard/cmd/run.go

@@ -21,10 +21,13 @@ var RunTestsCmd = &cobra.Command{
 		testPackagesJson, _ := cmd.Flags().GetString("test-packages-json")
 		testPackagesArg, _ := cmd.Flags().GetStringSlice("test-packages")
 		runCount, _ := cmd.Flags().GetInt("run-count")
+		timeout, _ := cmd.Flags().GetDuration("timeout")
+		tags, _ := cmd.Flags().GetStringArray("tags")
 		useRace, _ := cmd.Flags().GetBool("race")
 		outputPath, _ := cmd.Flags().GetString("output-json")
 		maxPassRatio, _ := cmd.Flags().GetFloat64("max-pass-ratio")
 		skipTests, _ := cmd.Flags().GetStringSlice("skip-tests")
+		selectTests, _ := cmd.Flags().GetStringSlice("select-tests")
 		printFailedTests, _ := cmd.Flags().GetBool("print-failed-tests")
 		useShuffle, _ := cmd.Flags().GetBool("shuffle")
 		shuffleSeed, _ := cmd.Flags().GetString("shuffle-seed")
@@ -49,37 +52,31 @@ var RunTestsCmd = &cobra.Command{
 			ProjectPath:          projectPath,
 			Verbose:              true,
 			RunCount:             runCount,
+			Timeout:              timeout,
+			Tags:                 tags,
 			UseRace:              useRace,
 			SkipTests:            skipTests,
+			SelectTests:          selectTests,
 			SelectedTestPackages: testPackages,
 			UseShuffle:           useShuffle,
 			ShuffleSeed:          shuffleSeed,
 		}
 
-		testResults, err := runner.RunTests()
+		testReport, err := runner.RunTests()
 		if err != nil {
 			fmt.Printf("Error running tests: %v\n", err)
 			os.Exit(1)
 		}
 
-		passedTests := reports.FilterPassedTests(testResults, maxPassRatio)
-		failedTests := reports.FilterFailedTests(testResults, maxPassRatio)
-		skippedTests := reports.FilterSkippedTests(testResults)
-		flakyTests := reports.FilterFlakyTests(testResults, maxPassRatio)
-
 		// Print all failed tests including flaky tests
-		if len(failedTests) > 0 && printFailedTests {
-			fmt.Printf("Maximum threshold for flaky tests: %.2f\n", maxPassRatio)
+		if printFailedTests {
 			fmt.Printf("PassRatio threshold for flaky tests: %.2f\n", maxPassRatio)
-			fmt.Printf("%d failed tests:\n", len(failedTests))
-			reports.PrintTests(failedTests, os.Stdout)
+			reports.PrintTests(os.Stdout, testReport.Results, maxPassRatio)
 		}
 
-		fmt.Printf("Summary: %d passed, %d skipped, %d failed, %d flaky\n", len(passedTests), len(skippedTests), len(failedTests), len(flakyTests))
-
 		// Save the test results in JSON format
-		if outputPath != "" && len(testResults) > 0 {
-			jsonData, err := json.MarshalIndent(testResults, "", "  ")
+		if outputPath != "" && len(testReport.Results) > 0 {
+			jsonData, err := json.MarshalIndent(testReport, "", "  ")
 			if err != nil {
 				log.Fatalf("Error marshaling test results to JSON: %v", err)
 			}
@@ -89,10 +86,11 @@ var RunTestsCmd = &cobra.Command{
 			fmt.Printf("All test results saved to %s\n", outputPath)
 		}
 
+		flakyTests := reports.FilterFlakyTests(testReport.Results, maxPassRatio)
 		if len(flakyTests) > 0 {
 			// Exit with error code if there are flaky tests
 			os.Exit(1)
-		} else if len(testResults) == 0 {
+		} else if len(testReport.Results) == 0 {
 			fmt.Printf("No tests were run for the specified packages.\n")
 		}
 	},
@@ -104,12 +102,15 @@ func init() {
 	RunTestsCmd.Flags().StringSlice("test-packages", nil, "Comma-separated list of test packages to run")
 	RunTestsCmd.Flags().Bool("run-all-packages", false, "Run all test packages in the project. This flag overrides --test-packages and --test-packages-json")
 	RunTestsCmd.Flags().IntP("run-count", "c", 1, "Number of times to run the tests")
+	RunTestsCmd.Flags().Duration("timeout", 0, "Passed on to the 'go test' command as the -timeout flag")
+	RunTestsCmd.Flags().StringArray("tags", nil, "Passed on to the 'go test' command as the -tags flag")
 	RunTestsCmd.Flags().Bool("race", false, "Enable the race detector")
 	RunTestsCmd.Flags().Bool("shuffle", false, "Enable test shuffling")
 	RunTestsCmd.Flags().String("shuffle-seed", "", "Set seed for test shuffling. Must be used with --shuffle")
 	RunTestsCmd.Flags().Bool("fail-fast", false, "Stop on the first test failure")
 	RunTestsCmd.Flags().String("output-json", "", "Path to output the test results in JSON format")
 	RunTestsCmd.Flags().StringSlice("skip-tests", nil, "Comma-separated list of test names to skip from running")
+	RunTestsCmd.Flags().StringSlice("select-tests", nil, "Comma-separated list of test names to specifically run")
 	RunTestsCmd.Flags().Bool("print-failed-tests", true, "Print failed test results to the console")
 	RunTestsCmd.Flags().Float64("max-pass-ratio", 1.0, "The maximum (non-inclusive) pass ratio threshold for a test to be considered a failure. Any tests below this pass rate will be considered flaky.")
 }

tools/flakeguard/go.mod

@@ -2,7 +2,10 @@ module github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard
 
 go 1.21.9
 
-require github.com/spf13/cobra v1.8.1
+require (
+	github.com/spf13/cobra v1.8.1
+	golang.org/x/text v0.20.0
+)
 
 require (
 	github.com/davecgh/go-spew v1.1.1 // indirect

tools/flakeguard/go.sum

@@ -12,6 +12,8 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
 github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+golang.org/x/text v0.20.0 h1:gK/Kv2otX8gz+wn7Rmb3vT96ZwuoxnQlY+HlJVj7Qug=
+golang.org/x/text v0.20.0/go.mod h1:D4IsuqiFMhST5bX19pQ9ikHC2GsaKyk/oF+pn3ducp4=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=