Print flay test logs in the console when no reruns

Author: lukaszcl

tools/flakeguard/cmd/run.go

@@ -6,6 +6,8 @@ import (
 	"fmt"
 	"os"
 	"os/exec"
+	"path/filepath"
+	"strings"
 	"time"
 
 	"github.com/briandowns/spinner"
@@ -21,6 +23,7 @@ import (
 const (
 	FlakyTestsExitCode = 1
 	ErrorExitCode      = 2
+	RawOutputDir       = "./flakeguard_raw_output"
 )
 
 type runConfig struct {
@@ -252,27 +255,54 @@ func initializeRunner(cfg *runConfig) *runner.Runner {
 		cfg.SelectTests,
 		cfg.IgnoreParentFailuresOnSubtests,
 		cfg.OmitOutputsOnSuccess,
+		RawOutputDir,
 		nil, // exec
 		nil, // parser
 	)
 }
 
 // generateMainReport creates the initial test report from the main run results.
 func generateMainReport(results []reports.TestResult, cfg *runConfig, goProject string) (*reports.TestReport, error) {
+	// Get the JSON output paths from the raw output directory
+	jsonOutputPaths, err := getJSONOutputPaths(RawOutputDir)
+	if err != nil {
+		log.Warn().Err(err).Msg("Failed to get JSON output paths")
+	}
+
 	reportVal, err := reports.NewTestReport(results,
 		reports.WithGoProject(goProject),
 		reports.WithCodeOwnersPath(cfg.CodeownersPath),
 		reports.WithMaxPassRatio(cfg.MinPassRatio),
 		reports.WithGoRaceDetection(cfg.UseRace),
 		reports.WithExcludedTests(cfg.SkipTests),
 		reports.WithSelectedTests(cfg.SelectTests),
+		reports.WithJSONOutputPaths(jsonOutputPaths),
 	)
 	if err != nil {
 		return nil, err
 	}
 	return &reportVal, nil
 }
 
+// getJSONOutputPaths returns a list of JSON output files from the given directory
+func getJSONOutputPaths(dir string) ([]string, error) {
+	files, err := os.ReadDir(dir)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil, nil
+		}
+		return nil, fmt.Errorf("failed to read directory %s: %w", dir, err)
+	}
+
+	var paths []string
+	for _, file := range files {
+		if !file.IsDir() && strings.HasSuffix(file.Name(), ".json") {
+			paths = append(paths, filepath.Join(dir, file.Name()))
+		}
+	}
+	return paths, nil
+}
+
 // handleReruns manages the process of rerunning failed tests and reporting results.
 func handleReruns(exitHandler *summaryAndExit, testRunner *runner.Runner, mainReport *reports.TestReport, cfg *runConfig, goProject string) {
 	failedTests := reports.FilterTests(mainReport.Results, func(tr reports.TestResult) bool {
@@ -384,6 +414,20 @@ func handleNoReruns(exitHandler *summaryAndExit, mainReport *reports.TestReport,
 	reports.RenderTestReport(&exitHandler.buffer, *mainReport, false, false)
 
 	if len(flakyTests) > 0 {
+		// Create a new report with only flaky tests to get their gotestsum output
+		flakyReport, err := reports.NewTestReport(flakyTests,
+			reports.WithJSONOutputPaths(mainReport.JSONOutputPaths),
+		)
+		if err != nil {
+			log.Error().Err(err).Msg("Error creating flaky tests report")
+		} else {
+			fmt.Fprint(&exitHandler.buffer, "\nFlaky Test Logs:\n\n")
+			err := flakyReport.PrintGotestsumOutput(&exitHandler.buffer, "pkgname")
+			if err != nil {
+				log.Error().Err(err).Msg("Error printing gotestsum output for flaky tests")
+			}
+		}
+
 		exitHandler.logMsgAndExit(zerolog.InfoLevel, "Found flaky tests.", FlakyTestsExitCode, map[string]interface{}{
 			"flaky_count":         len(flakyTests),
 			"stability_threshold": fmt.Sprintf("%.0f%%", cfg.MinPassRatio*100),

tools/flakeguard/runner/runner.go

@@ -13,10 +13,6 @@ import (
 	"github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard/runner/parser"
 )
 
-const (
-	RawOutputDir = "./flakeguard_raw_output"
-)
-
 // Runner describes the test run parameters and manages test execution and result parsing.
 // It delegates command execution to an Executor and result parsing to a Parser.
 type Runner struct {
@@ -33,8 +29,8 @@ type Runner struct {
 	FailFast          bool
 	SkipTests         []string
 	SelectTests       []string
+	RawOutputDir      string
 
-	// Configuration passed down to the parser
 	IgnoreParentFailuresOnSubtests bool
 	OmitOutputsOnSuccess           bool
 
@@ -48,7 +44,6 @@ type Runner struct {
 func NewRunner(
 	projectPath string,
 	verbose bool,
-	// Runner specific config
 	runCount int,
 	goTestCountFlag *int,
 	goTestRaceFlag bool,
@@ -59,12 +54,11 @@ func NewRunner(
 	failFast bool,
 	skipTests []string,
 	selectTests []string,
-	// Parser specific config (passed during initialization)
 	ignoreParentFailuresOnSubtests bool,
 	omitOutputsOnSuccess bool,
-	// Dependencies (allow injection for testing)
+	rawOutputDir string,
 	exec executor.Executor,
-	p parser.Parser, // Use interface type directly
+	p parser.Parser,
 ) *Runner {
 	if exec == nil {
 		exec = executor.NewCommandExecutor()
@@ -87,6 +81,7 @@ func NewRunner(
 		SelectTests:                    selectTests,
 		IgnoreParentFailuresOnSubtests: ignoreParentFailuresOnSubtests,
 		OmitOutputsOnSuccess:           omitOutputsOnSuccess,
+		RawOutputDir:                   rawOutputDir,
 		exec:                           exec,
 		parser:                         p,
 	}
@@ -105,7 +100,7 @@ func (r *Runner) getExecutorConfig() executor.Config {
 		ShuffleSeed:       r.ShuffleSeed,
 		SkipTests:         r.SkipTests,
 		SelectTests:       r.SelectTests,
-		RawOutputDir:      RawOutputDir, // Use the constant defined in this package
+		RawOutputDir:      r.RawOutputDir,
 	}
 }