Refactor run functions to return test results array

Author: lukaszcl

tools/flakeguard/cmd/run.go

@@ -10,6 +10,7 @@ import (
 	"github.com/rs/zerolog/log"
 	"github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard/reports"
 	"github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard/runner"
+	"github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard/utils"
 	"github.com/spf13/cobra"
 )
 
@@ -35,15 +36,16 @@ var RunTestsCmd = &cobra.Command{
 
 		// Retrieve flags
 		projectPath, _ := cmd.Flags().GetString("project-path")
+		codeownersPath, _ := cmd.Flags().GetString("codeowners-path")
 		testPackagesJson, _ := cmd.Flags().GetString("test-packages-json")
 		testPackagesArg, _ := cmd.Flags().GetStringSlice("test-packages")
 		testCmdStrings, _ := cmd.Flags().GetStringArray("test-cmd")
 		runCount, _ := cmd.Flags().GetInt("run-count")
 		rerunFailedCount, _ := cmd.Flags().GetInt("rerun-failed-count")
 		tags, _ := cmd.Flags().GetStringArray("tags")
 		useRace, _ := cmd.Flags().GetBool("race")
-		mainReportPath, _ := cmd.Flags().GetString("main-report-path")
-		rerunReportPath, _ := cmd.Flags().GetString("rerun-report-path")
+		mainResultsPath, _ := cmd.Flags().GetString("main-results-path")
+		rerunResultsPath, _ := cmd.Flags().GetString("rerun-results-path")
 		minPassRatio, _ := cmd.Flags().GetFloat64("min-pass-ratio")
 		// For backward compatibility, check if max-pass-ratio was used
 		maxPassRatio, _ := cmd.Flags().GetFloat64("max-pass-ratio")
@@ -57,6 +59,11 @@ var RunTestsCmd = &cobra.Command{
 		failFast, _ := cmd.Flags().GetBool("fail-fast")
 		goTestTimeoutFlag, _ := cmd.Flags().GetString("go-test-timeout")
 
+		goProject, err := utils.GetGoProjectName(projectPath)
+		if err != nil {
+			log.Warn().Err(err).Str("projectPath", goProject).Msg("Failed to get pretty project path")
+		}
+
 		// Retrieve go-test-count flag as a pointer if explicitly provided.
 		var goTestCountFlag *int
 		if cmd.Flags().Changed("go-test-count") {
@@ -124,45 +131,53 @@ var RunTestsCmd = &cobra.Command{
 			FailFast:                       failFast,
 		}
 
-		var (
-			mainReport  *reports.TestReport // Main test report
-			rerunReport *reports.TestReport // Test report after rerunning failed tests
-		)
-
 		// Run the tests
-		var err error
+		var mainResults []reports.TestResult
 		if len(testCmdStrings) > 0 {
-			mainReport, err = testRunner.RunTestCmd(testCmdStrings)
+			mainResults, err = testRunner.RunTestCmd(testCmdStrings)
 			if err != nil {
 				log.Fatal().Err(err).Msg("Error running custom test command")
 				flushSummaryAndExit(ErrorExitCode)
 			}
 		} else {
-			mainReport, err = testRunner.RunTestPackages(testPackages)
+			mainResults, err = testRunner.RunTestPackages(testPackages)
 			if err != nil {
 				log.Fatal().Err(err).Msg("Error running test packages")
 				flushSummaryAndExit(ErrorExitCode)
 			}
 		}
 
-		// Save the main test report to file
-		if mainReportPath != "" && len(mainReport.Results) > 0 {
-			if err := mainReport.SaveToFile(mainReportPath); err != nil {
+		if len(mainResults) == 0 {
+			log.Warn().Msg("No tests were run for the specified packages")
+			flushSummaryAndExit(0)
+		}
+
+		// Save the main test results to file
+		if mainResultsPath != "" && len(mainResults) > 0 {
+			if err := reports.SaveTestResultsToFile(mainResults, mainResultsPath); err != nil {
 				log.Error().Err(err).Msg("Error saving test results to file")
 				flushSummaryAndExit(ErrorExitCode)
 			}
-			log.Info().Str("path", mainReportPath).Msg("Main test report saved")
+			log.Info().Str("path", mainResultsPath).Msg("Main test report saved")
 		}
 
-		if len(mainReport.Results) == 0 {
-			log.Warn().Msg("No tests were run for the specified packages")
-			flushSummaryAndExit(0)
+		mainReport, err := reports.NewTestReport(mainResults,
+			reports.WithGoProject(goProject),
+			reports.WithCodeOwnersPath(codeownersPath),
+			reports.WithMaxPassRatio(passRatioThreshold),
+			reports.WithGoRaceDetection(useRace),
+			reports.WithExcludedTests(skipTests),
+			reports.WithSelectedTests(selectTests),
+		)
+		if err != nil {
+			log.Error().Err(err).Msg("Error creating main test report")
+			flushSummaryAndExit(ErrorExitCode)
 		}
 
 		// Accumulate main summary into summaryBuffer
 		fmt.Fprint(&summaryBuffer, "\nFlakeguard Main Summary:\n")
 		fmt.Fprintf(&summaryBuffer, "-------------------------\n")
-		reports.RenderResults(&summaryBuffer, *mainReport, false, false)
+		reports.RenderTestReport(&summaryBuffer, mainReport, false, false)
 		fmt.Fprintln(&summaryBuffer)
 
 		// Rerun failed tests
@@ -176,28 +191,41 @@ var RunTestsCmd = &cobra.Command{
 				flushSummaryAndExit(0)
 			}
 
-			rerunReport, err = testRunner.RerunFailedTests(failedTests)
+			rerunResults, rerunJsonOutputPaths, err := testRunner.RerunFailedTests(failedTests)
 			if err != nil {
 				log.Fatal().Err(err).Msg("Error rerunning failed tests")
 				flushSummaryAndExit(ErrorExitCode)
 			}
 
+			rerunReport, err := reports.NewTestReport(rerunResults,
+				reports.WithGoProject(goProject),
+				reports.WithCodeOwnersPath(codeownersPath),
+				reports.WithMaxPassRatio(1),
+				reports.WithExcludedTests(skipTests),
+				reports.WithSelectedTests(selectTests),
+				reports.WithJSONOutputPaths(rerunJsonOutputPaths),
+			)
+			if err != nil {
+				log.Error().Err(err).Msg("Error creating rerun test report")
+				flushSummaryAndExit(ErrorExitCode)
+			}
+
 			fmt.Fprint(&summaryBuffer, "\nAll Tests That Were Rerun:\n")
 			fmt.Fprintf(&summaryBuffer, "---------------------------\n")
 			reports.PrintTestResultsTable(&summaryBuffer, rerunReport.Results, false, false)
 			fmt.Fprintln(&summaryBuffer)
 
 			// Save the rerun test report to file
-			if rerunReportPath != "" && len(rerunReport.Results) > 0 {
-				if err := rerunReport.SaveToFile(rerunReportPath); err != nil {
+			if rerunResultsPath != "" && len(rerunResults) > 0 {
+				if err := reports.SaveTestResultsToFile(rerunResults, rerunResultsPath); err != nil {
 					log.Error().Err(err).Msg("Error saving test results to file")
 					flushSummaryAndExit(ErrorExitCode)
 				}
-				log.Info().Str("path", rerunReportPath).Msg("Rerun test report saved")
+				log.Info().Str("path", rerunResultsPath).Msg("Rerun test report saved")
 			}
 
 			// Filter tests that failed after reruns
-			failedAfterRerun := reports.FilterTests(rerunReport.Results, func(tr reports.TestResult) bool {
+			failedAfterRerun := reports.FilterTests(rerunResults, func(tr reports.TestResult) bool {
 				return !tr.Skipped && tr.Successes == 0
 			})
 
@@ -246,6 +274,7 @@ var RunTestsCmd = &cobra.Command{
 
 func init() {
 	RunTestsCmd.Flags().StringP("project-path", "r", ".", "The path to the Go project. Default is the current directory. Useful for subprojects")
+	RunTestsCmd.Flags().StringP("codeowners-path", "", "", "Path to the CODEOWNERS file")
 	RunTestsCmd.Flags().String("test-packages-json", "", "JSON-encoded string of test packages")
 	RunTestsCmd.Flags().StringSlice("test-packages", nil, "Comma-separated list of test packages to run")
 	RunTestsCmd.Flags().StringArray("test-cmd", nil,
@@ -260,8 +289,8 @@ func init() {
 	RunTestsCmd.Flags().Bool("shuffle", false, "Enable test shuffling")
 	RunTestsCmd.Flags().String("shuffle-seed", "", "Set seed for test shuffling. Must be used with --shuffle")
 	RunTestsCmd.Flags().Bool("fail-fast", false, "Stop on the first test failure")
-	RunTestsCmd.Flags().String("main-report-path", "", "Path to the main test report in JSON format")
-	RunTestsCmd.Flags().String("rerun-report-path", "", "Path to the rerun test report in JSON format")
+	RunTestsCmd.Flags().String("main-results-path", "", "Path to the main test results in JSON format")
+	RunTestsCmd.Flags().String("rerun-results-path", "", "Path to the rerun test results in JSON format")
 	RunTestsCmd.Flags().StringSlice("skip-tests", nil, "Comma-separated list of test names to skip from running")
 	RunTestsCmd.Flags().StringSlice("select-tests", nil, "Comma-separated list of test names to specifically run")
 

tools/flakeguard/reports/data.go

@@ -1,161 +1,19 @@
 package reports
 
 import (
-	"bytes"
 	"encoding/json"
 	"fmt"
-	"io"
 	"os"
-	"os/exec"
 	"sort"
 	"strings"
 	"time"
-
-	"github.com/google/uuid"
 )
 
-// TestReport reports on the parameters and results of one to many test runs
-type TestReport struct {
-	ID                   string       `json:"id,omitempty"`
-	GoProject            string       `json:"go_project"`
-	BranchName           string       `json:"branch_name,omitempty"`
-	HeadSHA              string       `json:"head_sha,omitempty"`
-	BaseSHA              string       `json:"base_sha,omitempty"`
-	RepoURL              string       `json:"repo_url,omitempty"`
-	GitHubWorkflowName   string       `json:"github_workflow_name,omitempty"`
-	GitHubWorkflowRunURL string       `json:"github_workflow_run_url,omitempty"`
-	SummaryData          *SummaryData `json:"summary_data"`
-	RaceDetection        bool         `json:"race_detection"`
-	ExcludedTests        []string     `json:"excluded_tests,omitempty"`
-	SelectedTests        []string     `json:"selected_tests,omitempty"`
-	Results              []TestResult `json:"results,omitempty"`
-	FailedLogsURL        string       `json:"failed_logs_url,omitempty"`
-	JSONOutputPaths      []string     `json:"-"` // go test -json outputs from runs
-	// MaxPassRatio is the maximum flakiness ratio allowed for a test to be considered not flaky
-	MaxPassRatio float64 `json:"max_pass_ratio,omitempty"`
-}
-
-func (r *TestReport) SetRandomReportID() {
-	uuid, err := uuid.NewRandom()
-	if err != nil {
-		panic(fmt.Errorf("error generating random report id: %w", err))
-	}
-	r.SetReportID(uuid.String())
-}
-
-func (r *TestReport) SetReportID(reportID string) {
-	r.ID = reportID
-	// Set the report ID in all test results
-	for i := range r.Results {
-		r.Results[i].ReportID = r.ID
-	}
-}
-
-// SaveToFile saves the test report to a JSON file at the given path.
-// It returns an error if there's any issue with marshaling the report or writing to the file.
-func (testReport *TestReport) SaveToFile(outputPath string) error {
-	jsonData, err := json.MarshalIndent(testReport, "", "  ")
-	if err != nil {
-		return fmt.Errorf("error marshaling test results to JSON: %w", err)
-	}
-
-	if err := os.WriteFile(outputPath, jsonData, 0600); err != nil {
-		return fmt.Errorf("error writing test results to file: %w", err)
-	}
-
-	return nil
-}
-
-func (tr *TestReport) PrintGotestsumOutput(w io.Writer, format string) error {
-	if len(tr.JSONOutputPaths) == 0 {
-		fmt.Fprintf(w, "No JSON output paths found in test report\n")
-		return nil
-	}
-
-	for _, path := range tr.JSONOutputPaths {
-		cmdStr := fmt.Sprintf("cat %q | gotestsum --raw-command --format %q -- cat", path, format)
-		cmd := exec.Command("bash", "-c", cmdStr)
-
-		var outBuf bytes.Buffer
-		cmd.Stdout = &outBuf
-		cmd.Stderr = &outBuf
-
-		if err := cmd.Run(); err != nil {
-			return fmt.Errorf("gotestsum command failed for file %s: %w\nOutput: %s", path, err, outBuf.String())
-		}
-
-		fmt.Fprint(w, outBuf.String())
-		fmt.Fprintln(w, "---------------------")
-	}
-	return nil
-}
-
-// GenerateSummaryData generates a summary of a report's test results
-func (testReport *TestReport) GenerateSummaryData() {
-	var runs, testRunCount, passes, fails, skips, panickedTests, racedTests, flakyTests int
-
-	// Map to hold unique test names that were entirely skipped
-	uniqueSkippedTestsMap := make(map[string]struct{})
-
-	for _, result := range testReport.Results {
-		runs += result.Runs
-		if result.Runs > testRunCount {
-			testRunCount = result.Runs
-		}
-		passes += result.Successes
-		fails += result.Failures
-		skips += result.Skips
-
-		if result.Runs == 0 && result.Skipped {
-			uniqueSkippedTestsMap[result.TestName] = struct{}{}
-		}
-
-		if result.Panic {
-			panickedTests++
-			flakyTests++
-		} else if result.Race {
-			racedTests++
-			flakyTests++
-		} else if !result.Skipped && result.Runs > 0 && result.PassRatio < testReport.MaxPassRatio {
-			flakyTests++
-		}
-	}
-
-	// Calculate the unique count of skipped tests
-	uniqueSkippedTestCount := len(uniqueSkippedTestsMap)
-
-	// Calculate the raw pass ratio
-	passRatio := passRatio(passes, runs)
-
-	// Calculate the raw flake ratio
-	totalTests := len(testReport.Results)
-	flakeRatio := flakeRatio(flakyTests, totalTests)
-
-	passRatioStr := formatRatio(passRatio)
-	flakeTestRatioStr := formatRatio(flakeRatio)
-
-	testReport.SummaryData = &SummaryData{
-		UniqueTestsRun:         totalTests,
-		UniqueSkippedTestCount: uniqueSkippedTestCount,
-		TestRunCount:           testRunCount,
-		PanickedTests:          panickedTests,
-		RacedTests:             racedTests,
-		FlakyTests:             flakyTests,
-		FlakyTestPercent:       flakeTestRatioStr,
-
-		TotalRuns:   runs,
-		PassedRuns:  passes,
-		FailedRuns:  fails,
-		SkippedRuns: skips,
-		PassPercent: passRatioStr,
-	}
-}
-
 // TestResult contains the results and outputs of a single test
 type TestResult struct {
 	// ReportID is the ID of the report this test result belongs to
 	// used mostly for Splunk logging
-	ReportID       string              `json:"report_id"`
+	ReportID       string              `json:"report_id,omitempty"`
 	TestName       string              `json:"test_name"`
 	TestPackage    string              `json:"test_package"`
 	PackagePanic   bool                `json:"package_panic"`
@@ -173,8 +31,21 @@ type TestResult struct {
 	FailedOutputs  map[string][]string `json:"failed_outputs,omitempty"` // Outputs for failed runs
 	Durations      []time.Duration     `json:"durations"`
 	PackageOutputs []string            `json:"package_outputs,omitempty"`
-	TestPath       string              `json:"test_path"`
-	CodeOwners     []string            `json:"code_owners"`
+	TestPath       string              `json:"test_path,omitempty"`
+	CodeOwners     []string            `json:"code_owners,omitempty"`
+}
+
+func SaveTestResultsToFile(results []TestResult, filePath string) error {
+	jsonData, err := json.MarshalIndent(results, "", "  ")
+	if err != nil {
+		return fmt.Errorf("error marshaling test results to JSON: %w", err)
+	}
+
+	if err := os.WriteFile(filePath, jsonData, 0o600); err != nil {
+		return fmt.Errorf("error writing test results to file: %w", err)
+	}
+
+	return nil
 }
 
 // SummaryData contains aggregated data from a set of test results

tools/flakeguard/reports/data_test.go

@@ -379,7 +379,7 @@ func TestAggregate(t *testing.T) {
 		},
 	}
 
-	aggregateOptions := &aggregateOptions{reportID: "123"}
+	aggregateOptions := &reportOptions{reportID: "123"}
 	aggregatedReport, err := aggregateReports(aggregateOptions, report1, report2)
 	if err != nil {
 		t.Fatalf("Error aggregating reports: %v", err)
@@ -468,7 +468,7 @@ func TestAggregateOutputs(t *testing.T) {
 		},
 	}
 
-	aggregateOptions := &aggregateOptions{reportID: "123"}
+	aggregateOptions := &reportOptions{reportID: "123"}
 	aggregatedReport, err := aggregateReports(aggregateOptions, report1, report2)
 	if err != nil {
 		t.Fatalf("Error aggregating reports: %v", err)
@@ -538,7 +538,7 @@ func TestAggregateIdenticalOutputs(t *testing.T) {
 		},
 	}
 
-	aggregateOptions := &aggregateOptions{reportID: "123"}
+	aggregateOptions := &reportOptions{reportID: "123"}
 	aggregatedReport, err := aggregateReports(aggregateOptions, report1, report2)
 	if err != nil {
 		t.Fatalf("Error aggregating reports: %v", err)
@@ -620,7 +620,7 @@ func TestAggregate_AllSkippedTests(t *testing.T) {
 		},
 	}
 
-	aggregateOptions := &aggregateOptions{reportID: "123"}
+	aggregateOptions := &reportOptions{reportID: "123"}
 	aggregatedReport, err := aggregateReports(aggregateOptions, report1, report2)
 	if err != nil {
 		t.Fatalf("Error aggregating reports: %v", err)