Improves and standardizes Flakeguard summary data

Author: kalverra

tools/flakeguard/cmd/aggregate_results.go

@@ -47,6 +47,7 @@ var AggregateResultsCmd = &cobra.Command{
 		s := spinner.New(spinner.CharSets[11], 100*time.Millisecond)
 		s.Suffix = " Aggregating test reports..."
 		s.Start()
+		fmt.Println()
 
 		// Load test reports from JSON files and aggregate them
 		aggregatedReport, err := reports.LoadAndAggregate(
@@ -62,12 +63,10 @@ var AggregateResultsCmd = &cobra.Command{
 		)
 		if err != nil {
 			s.Stop()
-			fmt.Println()
-			log.Error().Err(err).Msg("Error aggregating test reports")
+			log.Error().Err(err).Stack().Msg("Error aggregating test reports")
 			os.Exit(ErrorExitCode)
 		}
 		s.Stop()
-		fmt.Println()
 		log.Debug().Msg("Successfully loaded and aggregated test reports")
 
 		// Start spinner for mapping test results to paths
@@ -79,37 +78,33 @@ var AggregateResultsCmd = &cobra.Command{
 		err = reports.MapTestResultsToPaths(aggregatedReport, repoPath)
 		if err != nil {
 			s.Stop()
-			fmt.Println()
-			log.Error().Err(err).Msg("Error mapping test results to paths")
+			log.Error().Stack().Err(err).Msg("Error mapping test results to paths")
 			os.Exit(ErrorExitCode)
 		}
 		s.Stop()
-		fmt.Println()
 		log.Debug().Msg("Successfully mapped paths to test results")
 
 		// Map test results to code owners if codeOwnersPath is provided
 		if codeOwnersPath != "" {
 			s = spinner.New(spinner.CharSets[11], 100*time.Millisecond)
 			s.Suffix = " Mapping test results to code owners..."
 			s.Start()
+			fmt.Println()
 
 			err = reports.MapTestResultsToOwners(aggregatedReport, codeOwnersPath)
 			if err != nil {
 				s.Stop()
-				fmt.Println()
-				log.Error().Err(err).Msg("Error mapping test results to code owners")
+				log.Error().Stack().Err(err).Msg("Error mapping test results to code owners")
 				os.Exit(ErrorExitCode)
 			}
 			s.Stop()
-			fmt.Println()
 			log.Debug().Msg("Successfully mapped code owners to test results")
 		}
 
 		failedTests := reports.FilterTests(aggregatedReport.Results, func(tr reports.TestResult) bool {
 			return !tr.Skipped && tr.PassRatio < maxPassRatio
 		})
 		s.Stop()
-		fmt.Println()
 
 		// Check if there are any failed tests
 		if len(failedTests) > 0 {
@@ -118,7 +113,7 @@ var AggregateResultsCmd = &cobra.Command{
 			// Create a new report for failed tests with logs
 			failedReportWithLogs := &reports.TestReport{
 				GoProject:          aggregatedReport.GoProject,
-				TestRunCount:       aggregatedReport.TestRunCount,
+				SummaryData:        aggregatedReport.SummaryData,
 				RaceDetection:      aggregatedReport.RaceDetection,
 				ExcludedTests:      aggregatedReport.ExcludedTests,
 				SelectedTests:      aggregatedReport.SelectedTests,
@@ -131,7 +126,7 @@ var AggregateResultsCmd = &cobra.Command{
 			// Save the failed tests report with logs
 			failedTestsReportWithLogsPath := filepath.Join(outputDir, "failed-test-results-with-logs.json")
 			if err := reports.SaveReport(fs, failedTestsReportWithLogsPath, *failedReportWithLogs); err != nil {
-				log.Error().Err(err).Msg("Error saving failed tests report with logs")
+				log.Error().Stack().Err(err).Msg("Error saving failed tests report with logs")
 				os.Exit(ErrorExitCode)
 			}
 			log.Debug().Str("path", failedTestsReportWithLogsPath).Msg("Failed tests report with logs saved")
@@ -146,7 +141,7 @@ var AggregateResultsCmd = &cobra.Command{
 			// Save the failed tests report without logs
 			failedTestsReportNoLogsPath := filepath.Join(outputDir, "failed-test-results.json")
 			if err := reports.SaveReport(fs, failedTestsReportNoLogsPath, *failedReportWithLogs); err != nil {
-				log.Error().Err(err).Msg("Error saving failed tests report without logs")
+				log.Error().Stack().Err(err).Msg("Error saving failed tests report without logs")
 				os.Exit(ErrorExitCode)
 			}
 			log.Debug().Str("path", failedTestsReportNoLogsPath).Msg("Failed tests report without logs saved")
@@ -164,7 +159,7 @@ var AggregateResultsCmd = &cobra.Command{
 		// Save the aggregated report to the output directory
 		aggregatedReportPath := filepath.Join(outputDir, "all-test-results.json")
 		if err := reports.SaveReport(fs, aggregatedReportPath, *aggregatedReport); err != nil {
-			log.Error().Err(err).Msg("Error saving aggregated test report")
+			log.Error().Stack().Err(err).Msg("Error saving aggregated test report")
 			os.Exit(ErrorExitCode)
 		}
 		log.Debug().Str("path", aggregatedReportPath).Msg("Aggregated test report saved")
@@ -175,17 +170,17 @@ var AggregateResultsCmd = &cobra.Command{
 			s = spinner.New(spinner.CharSets[11], 100*time.Millisecond)
 			s.Suffix = " Generating summary json..."
 			s.Start()
+			fmt.Println()
 
 			summaryFilePath = filepath.Join(outputDir, summaryFileName)
 			err = generateAllTestsSummaryJSON(aggregatedReport, summaryFilePath, maxPassRatio)
 			if err != nil {
 				s.Stop()
 				fmt.Println()
-				log.Error().Err(err).Msg("Error generating summary json")
+				log.Error().Stack().Err(err).Msg("Error generating summary json")
 				os.Exit(ErrorExitCode)
 			}
 			s.Stop()
-			fmt.Println()
 			log.Debug().Str("path", summaryFilePath).Msg("Summary generated")
 		}
 

tools/flakeguard/cmd/generate_report.go

@@ -17,23 +17,9 @@ import (
 	"golang.org/x/oauth2"
 )
 
-type SummaryData struct {
-	TotalTests     int     `json:"total_tests"`
-	PanickedTests  int     `json:"panicked_tests"`
-	RacedTests     int     `json:"raced_tests"`
-	FlakyTests     int     `json:"flaky_tests"`
-	FlakyTestRatio string  `json:"flaky_test_ratio"`
-	TotalRuns      int     `json:"total_runs"`
-	PassedRuns     int     `json:"passed_runs"`
-	FailedRuns     int     `json:"failed_runs"`
-	SkippedRuns    int     `json:"skipped_runs"`
-	PassRatio      string  `json:"pass_ratio"`
-	MaxPassRatio   float64 `json:"max_pass_ratio"`
-}
-
 var GenerateReportCmd = &cobra.Command{
 	Use:   "generate-report",
-	Short: "Generate reports from an aggregated test results",
+	Short: "Generate test reports from aggregated results that can be posted to GitHub",
 	Run: func(cmd *cobra.Command, args []string) {
 		fs := reports.OSFileSystem{}
 
@@ -80,7 +66,7 @@ var GenerateReportCmd = &cobra.Command{
 		log.Info().Msg("Successfully loaded aggregated test report")
 
 		// Load the summary data to check for failed tests
-		var summaryData SummaryData
+		var summaryData reports.SummaryData
 
 		if summaryPath == "" {
 			log.Error().Msg("Summary path is required")

tools/flakeguard/cmd/run.go

@@ -40,6 +40,11 @@ var RunTestsCmd = &cobra.Command{
 		shuffleSeed, _ := cmd.Flags().GetString("shuffle-seed")
 		omitOutputsOnSuccess, _ := cmd.Flags().GetBool("omit-test-outputs-on-success")
 
+		if maxPassRatio < 0 || maxPassRatio > 1 {
+			log.Error().Float64("max pass ratio", maxPassRatio).Msg("Error: max pass ratio must be between 0 and 1")
+			os.Exit(ErrorExitCode)
+		}
+
 		// Check if project dependencies are correctly set up
 		if err := checkDependencies(projectPath); err != nil {
 			log.Error().Err(err).Msg("Error checking project dependencies")
@@ -74,6 +79,7 @@ var RunTestsCmd = &cobra.Command{
 			UseShuffle:           useShuffle,
 			ShuffleSeed:          shuffleSeed,
 			OmitOutputsOnSuccess: omitOutputsOnSuccess,
+			MaxPassRatio:         maxPassRatio,
 		}
 
 		// Run the tests
@@ -107,10 +113,10 @@ var RunTestsCmd = &cobra.Command{
 			return !tr.Skipped && tr.PassRatio < maxPassRatio
 		})
 
+		fmt.Printf("\nFlakeguard Summary\n")
+		reports.RenderResults(os.Stdout, flakyTests, maxPassRatio, false, false)
 		if len(flakyTests) > 0 {
 			log.Info().Int("count", len(flakyTests)).Str("pass ratio threshold", fmt.Sprintf("%.2f%%", maxPassRatio*100)).Msg("Found flaky tests")
-			fmt.Printf("\nFlakeguard Summary\n")
-			reports.RenderResults(os.Stdout, flakyTests, maxPassRatio, false, false)
 			// Exit with error code if there are flaky tests
 			os.Exit(FlakyTestsExitCode)
 		}

tools/flakeguard/go.mod

@@ -1,6 +1,6 @@
 module github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard
 
-go 1.23.4
+go 1.23.6
 
 require (
 	github.com/briandowns/spinner v1.23.1
@@ -21,6 +21,7 @@ require (
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.19 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	golang.org/x/net v0.27.0 // indirect

tools/flakeguard/go.sum

@@ -25,6 +25,7 @@ github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovk
 github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
 github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA=
 github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=

tools/flakeguard/main.go

@@ -6,6 +6,7 @@ import (
 
 	"github.com/rs/zerolog"
 	"github.com/rs/zerolog/log"
+	"github.com/rs/zerolog/pkgerrors"
 	"github.com/spf13/cobra"
 
 	"github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard/cmd"
@@ -32,6 +33,7 @@ func init() {
 		Out:        os.Stderr,
 		TimeFormat: "15:04:05.00", // hh:mm:ss.ss format
 	})
+	zerolog.ErrorStackMarshaler = pkgerrors.MarshalStack
 	rootCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle")
 
 	rootCmd.AddCommand(cmd.FindTestsCmd)