Integrate go test transformer into Flakeguard

Author: lukaszcl

tools/flakeguard/cmd/run.go

@@ -41,6 +41,7 @@ var RunTestsCmd = &cobra.Command{
 		useShuffle, _ := cmd.Flags().GetBool("shuffle")
 		shuffleSeed, _ := cmd.Flags().GetString("shuffle-seed")
 		omitOutputsOnSuccess, _ := cmd.Flags().GetBool("omit-test-outputs-on-success")
+		ignoreParentFailuresOnSubtests, _ := cmd.Flags().GetBool("ignore-parent-failures-on-subtests")
 
 		outputDir := filepath.Dir(outputPath)
 		initialDirSize, err := getDirSize(outputDir)
@@ -63,7 +64,6 @@ var RunTestsCmd = &cobra.Command{
 		// Determine test packages
 		var testPackages []string
 		if len(testCmdStrings) == 0 {
-			// No custom command -> parse packages
 			if testPackagesJson != "" {
 				if err := json.Unmarshal([]byte(testPackagesJson), &testPackages); err != nil {
 					log.Error().Err(err).Msg("Error decoding test packages JSON")
@@ -79,18 +79,19 @@ var RunTestsCmd = &cobra.Command{
 
 		// Initialize the runner
 		testRunner := runner.Runner{
-			ProjectPath:          projectPath,
-			Verbose:              true,
-			RunCount:             runCount,
-			Timeout:              timeout,
-			Tags:                 tags,
-			UseRace:              useRace,
-			SkipTests:            skipTests,
-			SelectTests:          selectTests,
-			UseShuffle:           useShuffle,
-			ShuffleSeed:          shuffleSeed,
-			OmitOutputsOnSuccess: omitOutputsOnSuccess,
-			MaxPassRatio:         maxPassRatio,
+			ProjectPath:                    projectPath,
+			Verbose:                        true,
+			RunCount:                       runCount,
+			Timeout:                        timeout,
+			Tags:                           tags,
+			UseRace:                        useRace,
+			SkipTests:                      skipTests,
+			SelectTests:                    selectTests,
+			UseShuffle:                     useShuffle,
+			ShuffleSeed:                    shuffleSeed,
+			OmitOutputsOnSuccess:           omitOutputsOnSuccess,
+			MaxPassRatio:                   maxPassRatio,
+			IgnoreParentFailuresOnSubtests: ignoreParentFailuresOnSubtests,
 		}
 
 		// Run the tests
@@ -105,7 +106,6 @@ var RunTestsCmd = &cobra.Command{
 				os.Exit(ErrorExitCode)
 			}
 		} else {
-			// Otherwise, use the normal go test approach
 			testReport, err = testRunner.RunTestPackages(testPackages)
 			if err != nil {
 				log.Fatal().Err(err).Msg("Error running test packages")
@@ -180,6 +180,7 @@ func init() {
 	RunTestsCmd.Flags().StringSlice("select-tests", nil, "Comma-separated list of test names to specifically run")
 	RunTestsCmd.Flags().Float64("max-pass-ratio", 1.0, "The maximum pass ratio threshold for a test to be considered flaky. Any tests below this pass rate will be considered flaky.")
 	RunTestsCmd.Flags().Bool("omit-test-outputs-on-success", true, "Omit test outputs and package outputs for tests that pass")
+	RunTestsCmd.Flags().Bool("ignore-parent-failures-on-subtests", false, "Ignore failures in parent tests when only subtests fail")
 }
 
 func checkDependencies(projectPath string) error {

tools/flakeguard/go-test-transform/pkg/transformer/transformer.go

@@ -37,7 +37,7 @@ type TestNode struct {
 }
 
 // TransformJSON transforms go test -json output according to the options
-func TransformJSON(input io.Reader, output io.Writer, opts *Options) (int, error) {
+func TransformJSON(input io.Reader, output io.Writer, opts *Options) error {
 	// Create scanner for JSON input
 	scanner := bufio.NewScanner(input)
 	scanner.Buffer(make([]byte, 1024*1024), 10*1024*1024) // 10MB max buffer
@@ -47,13 +47,13 @@ func TransformJSON(input io.Reader, output io.Writer, opts *Options) (int, error
 	for scanner.Scan() {
 		var event TestEvent
 		if err := json.Unmarshal(scanner.Bytes(), &event); err != nil {
-			return 1, fmt.Errorf("failed to parse JSON: %v", err)
+			return fmt.Errorf("failed to parse JSON: %v", err)
 		}
 		events = append(events, event)
 	}
 
 	if err := scanner.Err(); err != nil {
-		return 1, fmt.Errorf("error reading input: %v", err)
+		return fmt.Errorf("error reading input: %v", err)
 	}
 
 	// Build test tree
@@ -69,22 +69,18 @@ func TransformJSON(input io.Reader, output io.Writer, opts *Options) (int, error
 	propagateIgnoreStatus(testTree, opts)
 
 	// Transform events
-	transformedEvents, anyRemainingFailures := transformEvents(events, testTree)
+	transformedEvents, _ := transformEvents(events, testTree)
 
 	// Output transformed events
 	for _, event := range transformedEvents {
 		eventJSON, err := json.Marshal(event)
 		if err != nil {
-			return 1, fmt.Errorf("failed to marshal JSON: %v", err)
+			return fmt.Errorf("failed to marshal JSON: %v", err)
 		}
 		fmt.Fprintln(output, string(eventJSON))
 	}
 
-	// Return appropriate exit code
-	if anyRemainingFailures {
-		return 1, nil
-	}
-	return 0, nil
+	return nil
 }
 
 // buildTestTree builds a tree of tests from the events

tools/flakeguard/runner/runner.go

@@ -16,6 +16,7 @@ import (
 	"time"
 
 	"github.com/rs/zerolog/log"
+	"github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard/go-test-transform/pkg/transformer"
 	"github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard/reports"
 )
 
@@ -26,22 +27,23 @@ var (
 
 // Runner describes the test run parameters and raw test outputs
 type Runner struct {
-	ProjectPath          string        // Path to the Go project directory.
-	prettyProjectPath    string        // Go project package path, formatted for pretty printing.
-	Verbose              bool          // If true, provides detailed logging.
-	RunCount             int           // Number of times to run the tests.
-	UseRace              bool          // Enable race detector.
-	Timeout              time.Duration // Test timeout
-	Tags                 []string      // Build tags.
-	UseShuffle           bool          // Enable test shuffling. -shuffle=on flag.
-	ShuffleSeed          string        // Set seed for test shuffling -shuffle={seed} flag. Must be used with UseShuffle.
-	FailFast             bool          // Stop on first test failure.
-	SkipTests            []string      // Test names to exclude.
-	SelectTests          []string      // Test names to include.
-	CollectRawOutput     bool          // Set to true to collect test output for later inspection.
-	OmitOutputsOnSuccess bool          // Set to true to omit test outputs on success.
-	MaxPassRatio         float64       // Maximum pass ratio threshold for a test to be considered flaky.
-	rawOutputs           map[string]*bytes.Buffer
+	ProjectPath                    string        // Path to the Go project directory.
+	prettyProjectPath              string        // Go project package path, formatted for pretty printing.
+	Verbose                        bool          // If true, provides detailed logging.
+	RunCount                       int           // Number of times to run the tests.
+	UseRace                        bool          // Enable race detector.
+	Timeout                        time.Duration // Test timeout
+	Tags                           []string      // Build tags.
+	UseShuffle                     bool          // Enable test shuffling. -shuffle=on flag.
+	ShuffleSeed                    string        // Set seed for test shuffling -shuffle={seed} flag. Must be used with UseShuffle.
+	FailFast                       bool          // Stop on first test failure.
+	SkipTests                      []string      // Test names to exclude.
+	SelectTests                    []string      // Test names to include.
+	CollectRawOutput               bool          // Set to true to collect test output for later inspection.
+	OmitOutputsOnSuccess           bool          // Set to true to omit test outputs on success.
+	MaxPassRatio                   float64       // Maximum pass ratio threshold for a test to be considered flaky.
+	IgnoreParentFailuresOnSubtests bool          // Ignore failures in parent tests when only subtests fail.
+	rawOutputs                     map[string]*bytes.Buffer
 }
 
 // RunTestPackages executes the tests for each provided package and aggregates all results.
@@ -283,6 +285,15 @@ func (e entry) String() string {
 // Subtests add more complexity, as panics in subtests are only reported in their parent's output,
 // and cannot be accurately attributed to the subtest that caused them.
 func (r *Runner) parseTestResults(filePaths []string) ([]reports.TestResult, error) {
+	// If the option is enabled, transform each JSON output file before parsing.
+	if r.IgnoreParentFailuresOnSubtests {
+		transformedPaths, err := r.transformTestOutputFiles(filePaths)
+		if err != nil {
+			return nil, err
+		}
+		filePaths = transformedPaths
+	}
+
 	var (
 		testDetails         = make(map[string]*reports.TestResult) // Holds run, pass counts, and other details for each test
 		panickedPackages    = map[string]struct{}{}                // Packages with tests that panicked
@@ -595,6 +606,36 @@ func (r *Runner) parseTestResults(filePaths []string) ([]reports.TestResult, err
 	return results, nil
 }
 
+// transformTestOutputFiles transforms the test output JSON files to ignore parent failures when only subtests fail.
+// It returns the paths to the transformed files.
+func (r *Runner) transformTestOutputFiles(filePaths []string) ([]string, error) {
+	transformedPaths := make([]string, len(filePaths))
+	for i, origPath := range filePaths {
+		inFile, err := os.Open(origPath)
+		if err != nil {
+			return nil, fmt.Errorf("failed to open original file %s: %w", origPath, err)
+		}
+		// Create a temporary file for the transformed output.
+		outFile, err := os.CreateTemp("", "transformed-output-*.json")
+		if err != nil {
+			inFile.Close()
+			return nil, fmt.Errorf("failed to create transformed temp file: %w", err)
+		}
+		// Transform the JSON output.
+		// The transformer option is set to ignore parent failures when only subtests fail.
+		err = transformer.TransformJSON(inFile, outFile, transformer.NewOptions(true))
+		inFile.Close()
+		outFile.Close()
+		if err != nil {
+			return nil, fmt.Errorf("failed to transform output file %s: %v", origPath, err)
+		}
+		// Use the transformed file path.
+		transformedPaths[i] = outFile.Name()
+		os.Remove(origPath)
+	}
+	return transformedPaths, nil
+}
+
 // attributePanicToTest properly attributes panics to the test that caused them.
 func attributePanicToTest(panicPackage string, panicEntries []entry) (test string, timeout bool, err error) {
 	regexSanitizePanicPackage := filepath.Base(panicPackage)