flakeguard: TT-1853 Create json file with metadata for all test results (#1361)

Author: lukaszcl

tools/flakeguard/cmd/aggregate_all.go

@@ -0,0 +1,54 @@
+package cmd
+
+import (
+	"log"
+
+	"github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard/reports"
+	"github.com/spf13/cobra"
+)
+
+var AggregateResultsCmd = &cobra.Command{
+	Use:   "aggregate-results",
+	Short: "Aggregate test results and optionally filter failed tests based on a threshold",
+	Run: func(cmd *cobra.Command, args []string) {
+		resultsFolderPath, _ := cmd.Flags().GetString("results-path")
+		outputResultsPath, _ := cmd.Flags().GetString("output-results")
+		outputLogsPath, _ := cmd.Flags().GetString("output-logs")
+		threshold, _ := cmd.Flags().GetFloat64("threshold")
+		minPassRatio, _ := cmd.Flags().GetFloat64("min-pass-ratio")
+		filterFailed, _ := cmd.Flags().GetBool("filter-failed")
+
+		// Aggregate all test results
+		allResults, err := reports.AggregateTestResults(resultsFolderPath)
+		if err != nil {
+			log.Fatalf("Error aggregating results: %v", err)
+		}
+
+		var resultsToSave []reports.TestResult
+
+		if filterFailed {
+			// Filter to only include failed tests based on threshold and minPassRatio
+			for _, result := range allResults {
+				if result.PassRatio < threshold && result.PassRatio > minPassRatio && !result.Skipped {
+					resultsToSave = append(resultsToSave, result)
+				}
+			}
+		} else {
+			resultsToSave = allResults
+		}
+
+		// Output results to JSON files
+		if len(resultsToSave) > 0 {
+			reports.SaveFilteredResultsAndLogs(outputResultsPath, outputLogsPath, resultsToSave)
+		}
+	},
+}
+
+func init() {
+	AggregateResultsCmd.Flags().String("results-path", "", "Path to the folder containing JSON test result files")
+	AggregateResultsCmd.Flags().String("output-results", "./results.json", "Path to output the aggregated or filtered test results in JSON format")
+	AggregateResultsCmd.Flags().String("output-logs", "", "Path to output the filtered test logs in JSON format")
+	AggregateResultsCmd.Flags().Float64("threshold", 0.8, "Threshold for considering a test as failed (used with --filter-failed)")
+	AggregateResultsCmd.Flags().Float64("min-pass-ratio", 0.001, "Minimum pass ratio for considering a test as flaky (used with --filter-failed)")
+	AggregateResultsCmd.Flags().Bool("filter-failed", false, "If true, filter and output only failed tests based on the threshold")
+}

tools/flakeguard/cmd/aggregate_failed.go

@@ -28,8 +28,7 @@ func init() {
 
 	rootCmd.AddCommand(cmd.FindTestsCmd)
 	rootCmd.AddCommand(cmd.RunTestsCmd)
-	rootCmd.AddCommand(cmd.AggregateAllCmd)
-	rootCmd.AddCommand(cmd.AggregateFailedCmd)
+	rootCmd.AddCommand(cmd.AggregateResultsCmd)
 }
 
 func main() {

tools/flakeguard/cmd/aggregate_results.go

@@ -55,14 +55,15 @@ type exitCoder interface {
 // runTests runs the tests for a given package and returns the path to the output file.
 func (r *Runner) runTests(packageName string) (string, bool, error) {
 	args := []string{"test", packageName, "-json", "-count=1"} // Enable JSON output
-	args = append(args, "2>/dev/null")                         // Redirect stderr to null
 	if r.UseRace {
 		args = append(args, "-race")
 	}
 	if len(r.SkipTests) > 0 {
 		skipPattern := strings.Join(r.SkipTests, "|")
 		args = append(args, fmt.Sprintf("-skip=%s", skipPattern))
 	}
+	// The last arg redirects stderr to null
+	args = append(args, "2>/dev/null")
 
 	if r.Verbose {
 		log.Printf("Running command: go %s\n", strings.Join(args, " "))