fix

Author: lukaszcl

tools/flakeguard/cmd/run.go

@@ -6,7 +6,6 @@ import (
 	"fmt"
 	"os"
 	"os/exec"
-	"path/filepath"
 
 	"github.com/rs/zerolog/log"
 	"github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard/reports"
@@ -62,13 +61,6 @@ var RunTestsCmd = &cobra.Command{
 			os.Exit(ErrorExitCode)
 		}
 
-		outputDir := filepath.Dir(outputPath)
-		initialDirSize, err := getDirSize(outputDir)
-		if err != nil {
-			log.Error().Err(err).Str("path", outputDir).Msg("Error getting initial directory size")
-			// intentionally don't exit here, as we can still proceed with the run
-		}
-
 		// Check if project dependencies are correctly set up
 		if err := checkDependencies(projectPath); err != nil {
 			log.Error().Err(err).Msg("Error checking project dependencies")
@@ -112,26 +104,27 @@ var RunTestsCmd = &cobra.Command{
 
 		// Run the tests
 		var (
-			testReport *reports.TestReport
+			firstReport *reports.TestReport
 		)
 
+		var err error
 		if len(testCmdStrings) > 0 {
-			testReport, err = testRunner.RunTestCmd(testCmdStrings)
+			firstReport, err = testRunner.RunTestCmd(testCmdStrings)
 			if err != nil {
 				log.Fatal().Err(err).Msg("Error running custom test command")
 				os.Exit(ErrorExitCode)
 			}
 		} else {
-			testReport, err = testRunner.RunTestPackages(testPackages)
+			firstReport, err = testRunner.RunTestPackages(testPackages)
 			if err != nil {
 				log.Fatal().Err(err).Msg("Error running test packages")
 				os.Exit(ErrorExitCode)
 			}
 		}
 
 		// Save the test results in JSON format
-		if outputPath != "" && len(testReport.Results) > 0 {
-			jsonData, err := json.MarshalIndent(testReport, "", "  ")
+		if outputPath != "" && len(firstReport.Results) > 0 {
+			jsonData, err := json.MarshalIndent(firstReport, "", "  ")
 			if err != nil {
 				log.Error().Err(err).Msg("Error marshaling test results to JSON")
 				os.Exit(ErrorExitCode)
@@ -143,14 +136,17 @@ var RunTestsCmd = &cobra.Command{
 			log.Info().Str("path", outputPath).Msg("Test results saved")
 		}
 
-		if len(testReport.Results) == 0 {
+		if len(firstReport.Results) == 0 {
 			log.Warn().Msg("No tests were run for the specified packages")
 			return
 		}
 
+		fmt.Printf("\nFlakeguard Initial Summary:\n")
+		reports.RenderResults(os.Stdout, firstReport, false, false)
+
 		// Rerun failed tests
 		if rerunFailed > 0 {
-			rerunReport, err := testRunner.RerunFailedTests(testReport.Results)
+			rerunReport, err := testRunner.RerunFailedTests(firstReport.Results)
 			if err != nil {
 				log.Fatal().Err(err).Msg("Error rerunning failed tests")
 				os.Exit(ErrorExitCode)
@@ -160,59 +156,37 @@ var RunTestsCmd = &cobra.Command{
 			failedAfterRerun := reports.FilterTests(rerunReport.Results, func(tr reports.TestResult) bool {
 				return !tr.Skipped && tr.Successes == 0
 			})
+			fmt.Printf("\nFlakeguard Rerun Summary:\n")
+			reports.RenderResults(os.Stdout, rerunReport, false, false)
+
 			if len(failedAfterRerun) > 0 {
 				log.Error().
 					Int("tests", len(failedAfterRerun)).
 					Int("reruns", rerunFailed).
 					Msg("Tests still failing after reruns with 0 successes")
 				os.Exit(ErrorExitCode)
+			} else {
+				log.Info().Msg("Failed tests passed at least once after reruns")
+				os.Exit(0)
 			}
 
 			// TODO: save rerun test report to JSON file
 		}
 
 		// Filter flaky tests using FilterTests
-		flakyTests := reports.FilterTests(testReport.Results, func(tr reports.TestResult) bool {
+		flakyTests := reports.FilterTests(firstReport.Results, func(tr reports.TestResult) bool {
 			return !tr.Skipped && tr.PassRatio < passRatioThreshold
 		})
 
-		finalDirSize, err := getDirSize(outputDir)
-		if err != nil {
-			log.Error().Err(err).Str("path", outputDir).Msg("Error getting final directory size")
-			// intentionally don't exit here, as we can still proceed with the run
-		}
-		diskSpaceUsed := byteCountSI(finalDirSize - initialDirSize)
-
-		// Report with more detailed information about reruns
-		if rerunFailed > 0 {
-			log.Info().
-				Int("initial runs", runCount).
-				Int("reruns for failed tests", rerunFailed).
-				Str("disk space used", diskSpaceUsed).
-				Msg("Test execution completed")
-		} else {
-			log.Info().
-				Int("runs", runCount).
-				Str("disk space used", diskSpaceUsed).
-				Msg("Test execution completed")
-		}
-
 		if len(flakyTests) > 0 {
 			log.Info().
 				Int("count", len(flakyTests)).
 				Str("stability threshold", fmt.Sprintf("%.0f%%", passRatioThreshold*100)).
 				Msg("Found flaky tests")
+			os.Exit(FlakyTestsExitCode)
 		} else {
 			log.Info().Msg("All tests passed stability requirements")
 		}
-
-		fmt.Printf("\nFlakeguard Summary\n")
-		reports.RenderResults(os.Stdout, testReport, false, false)
-
-		if len(flakyTests) > 0 {
-			// Exit with error code if there are flaky tests
-			os.Exit(FlakyTestsExitCode)
-		}
 	},
 }
 

tools/flakeguard/reports/presentation.go

@@ -231,7 +231,6 @@ func renderSummaryTable(w io.Writer, summary *SummaryData, markdown bool, collap
 
 func renderTestResultsTable(w io.Writer, table [][]string, collapsible bool) {
 	if len(table) <= 1 {
-		fmt.Fprintln(w, "No tests found under the specified pass ratio threshold.")
 		return
 	}
 	printTable(w, table, collapsible)

tools/flakeguard/runner/example_test_package/example_tests_test.go

@@ -218,18 +218,21 @@ func TestTimeout(t *testing.T) {
 func TestRandomFlaky(t *testing.T) {
 	t.Parallel()
 
+	time.Sleep(100 * time.Millisecond)
+
 	// Seed random number generator with current time
-	r := rand.New(rand.NewSource(time.Now().UnixNano()))
+	seed := time.Now().UnixNano()
+	t.Logf("Using seed: %d", seed)
+	r := rand.New(rand.NewSource(seed))
 
 	// Generate a random number between 0 and 1
 	randomValue := r.Float64()
 
 	t.Logf("Random value generated: %f", randomValue)
 
-	// Fail the test approximately 90% of the time
-	if randomValue < 0.9 {
-		t.Fatal("This test randomly failed (90% probability)")
+	if randomValue < 0.5 {
+		t.Fatal("This test randomly failed")
 	}
 
-	t.Log("This test randomly passed (90% probability)")
+	t.Log("This test randomly passed")
 }

tools/flakeguard/runner/runner.go

@@ -739,7 +739,8 @@ func (r *Runner) RerunFailedTests(results []reports.TestResult) (*reports.TestRe
 	// Group failing tests by package for more efficient reruns
 	failingTestsByPackage := make(map[string][]string)
 	for _, tr := range results {
-		if !tr.Skipped && tr.PassRatio < 1 {
+		if !tr.Skipped {
+			// if !tr.Skipped && tr.PassRatio < 1 { //TODO uncomment
 			if _, exists := failingTestsByPackage[tr.TestPackage]; !exists {
 				failingTestsByPackage[tr.TestPackage] = []string{}
 			}
@@ -754,7 +755,7 @@ func (r *Runner) RerunFailedTests(results []reports.TestResult) (*reports.TestRe
 	var rerunJsonFilePaths []string
 
 	// Rerun each failing test package up to RerunFailed times
-	for i := 0; i < r.RerunFailed; i++ {
+	for i := range r.RerunFailed {
 		for pkg, tests := range failingTestsByPackage {
 			// Build regex pattern to match all failing tests in this package
 			testPattern := fmt.Sprintf("^(%s)$", strings.Join(tests, "|"))
@@ -805,26 +806,7 @@ func (r *Runner) RerunFailedTests(results []reports.TestResult) (*reports.TestRe
 		Results:       rerunResults,
 		MaxPassRatio:  r.MaxPassRatio,
 	}
+	report.GenerateSummaryData()
 
 	return report, nil
 }
-
-// buildGoTestCommandForTest builds a `go test` command specifically
-// for one failing test, using TestPackage and TestName in the -run argument.
-func (r *Runner) buildGoTestCommandForTest(t reports.TestResult) []string {
-	cmd := []string{
-		"go", "test",
-		t.TestPackage,
-		"-run", fmt.Sprintf("^%s$", t.TestName), // Run exactly this test
-		"-json", // Example flag, adjust as needed
-	}
-
-	// Add verbosity if requested
-	if r.Verbose {
-		cmd = append(cmd, "-v")
-	}
-
-	// Add any additional flags or args required by your setup here
-
-	return cmd
-}