TT-1995 Prevent double-counting of test failures in Flakeguard (#1665)

Author: lukaszcl

tools/flakeguard/cmd/run.go

@@ -41,6 +41,7 @@ var RunTestsCmd = &cobra.Command{
 		useShuffle, _ := cmd.Flags().GetBool("shuffle")
 		shuffleSeed, _ := cmd.Flags().GetString("shuffle-seed")
 		omitOutputsOnSuccess, _ := cmd.Flags().GetBool("omit-test-outputs-on-success")
+		ignoreParentFailuresOnSubtests, _ := cmd.Flags().GetBool("ignore-parent-failures-on-subtests")
 
 		outputDir := filepath.Dir(outputPath)
 		initialDirSize, err := getDirSize(outputDir)
@@ -63,7 +64,6 @@ var RunTestsCmd = &cobra.Command{
 		// Determine test packages
 		var testPackages []string
 		if len(testCmdStrings) == 0 {
-			// No custom command -> parse packages
 			if testPackagesJson != "" {
 				if err := json.Unmarshal([]byte(testPackagesJson), &testPackages); err != nil {
 					log.Error().Err(err).Msg("Error decoding test packages JSON")
@@ -79,18 +79,19 @@ var RunTestsCmd = &cobra.Command{
 
 		// Initialize the runner
 		testRunner := runner.Runner{
-			ProjectPath:          projectPath,
-			Verbose:              true,
-			RunCount:             runCount,
-			Timeout:              timeout,
-			Tags:                 tags,
-			UseRace:              useRace,
-			SkipTests:            skipTests,
-			SelectTests:          selectTests,
-			UseShuffle:           useShuffle,
-			ShuffleSeed:          shuffleSeed,
-			OmitOutputsOnSuccess: omitOutputsOnSuccess,
-			MaxPassRatio:         maxPassRatio,
+			ProjectPath:                    projectPath,
+			Verbose:                        true,
+			RunCount:                       runCount,
+			Timeout:                        timeout,
+			Tags:                           tags,
+			UseRace:                        useRace,
+			SkipTests:                      skipTests,
+			SelectTests:                    selectTests,
+			UseShuffle:                     useShuffle,
+			ShuffleSeed:                    shuffleSeed,
+			OmitOutputsOnSuccess:           omitOutputsOnSuccess,
+			MaxPassRatio:                   maxPassRatio,
+			IgnoreParentFailuresOnSubtests: ignoreParentFailuresOnSubtests,
 		}
 
 		// Run the tests
@@ -105,7 +106,6 @@ var RunTestsCmd = &cobra.Command{
 				os.Exit(ErrorExitCode)
 			}
 		} else {
-			// Otherwise, use the normal go test approach
 			testReport, err = testRunner.RunTestPackages(testPackages)
 			if err != nil {
 				log.Fatal().Err(err).Msg("Error running test packages")
@@ -180,6 +180,7 @@ func init() {
 	RunTestsCmd.Flags().StringSlice("select-tests", nil, "Comma-separated list of test names to specifically run")
 	RunTestsCmd.Flags().Float64("max-pass-ratio", 1.0, "The maximum pass ratio threshold for a test to be considered flaky. Any tests below this pass rate will be considered flaky.")
 	RunTestsCmd.Flags().Bool("omit-test-outputs-on-success", true, "Omit test outputs and package outputs for tests that pass")
+	RunTestsCmd.Flags().Bool("ignore-parent-failures-on-subtests", false, "Ignore failures in parent tests when only subtests fail")
 }
 
 func checkDependencies(projectPath string) error {

tools/flakeguard/go-test-transform/README.md

@@ -0,0 +1,54 @@
+# Go Test Transform
+
+This utility transforms the output of Go's test JSON format to handle subtest failures more intelligently. It prevents parent tests from failing when only their subtests fail, while preserving the original test structure and output format.
+
+## Features
+
+- Transforms Go test JSON output to modify how test failures propagate
+- Parent tests won't fail if they only have failing subtests but no direct failures
+- Maintains the original JSON format for compatibility with other tools
+
+## Key Behavior
+
+- **Subtest Failure Handling**: When a subtest fails, the parent test will not be marked as failed unless the parent itself has a direct failure
+- **Important Note**: If a parent test has both log messages (`t.Log()`) and failing subtests, it will still be marked as failed, as seen in the `TestLogMessagesNotDirectFailures` test
+
+## Usage
+
+```bash
+go test -json ./... | go-test-transform -ignore-all
+```
+
+This will transform the test output to prevent parent tests from failing when only their subtests fail.
+
+## Options
+
+- `-ignore-all`: Ignore all subtest failures
+- `-input`: Input file (default: stdin)
+- `-output`: Output file (default: stdout)
+
+## Example
+
+For a test structure like:
+
+```go
+func TestParent(t *testing.T) {
+	t.Run("Subtest1", func(t *testing.T) {
+		t.Error("Subtest 1 failed")
+	})
+}
+```
+
+The transformed output will be:
+
+```json
+{"Time":"2023-05-10T15:04:05.123Z","Action":"run","Package":"example/pkg","Test":"TestParent"}
+{"Time":"2023-05-10T15:04:05.124Z","Action":"run","Package":"example/pkg","Test":"TestParent/Subtest1"}
+{"Time":"2023-05-10T15:04:05.125Z","Action":"output","Package":"example/pkg","Test":"TestParent/Subtest1","Output":"    subtest1_test.go:12: Subtest 1 failed\n"}
+{"Time":"2023-05-10T15:04:05.126Z","Action":"fail","Package":"example/pkg","Test":"TestParent/Subtest1","Elapsed":0.001}
+{"Time":"2023-05-10T15:04:05.127Z","Action":"pass","Package":"example/pkg","Test":"TestParent","Elapsed":0.004}
+{"Time":"2023-05-10T15:04:05.128Z","Action":"output","Package":"example/pkg","Output":"FAIL\texample/pkg\t0.004s\n"}
+{"Time":"2023-05-10T15:04:05.129Z","Action":"fail","Package":"example/pkg","Elapsed":0.005}
+```
+
+Note that in the original output, both `TestParent/Subtest1` and `TestParent` would be marked as failed. After transformation, `TestParent/Subtest1` remains failed, but `TestParent` is changed to pass since it doesn't have a direct failure.

tools/flakeguard/go-test-transform/cmd/main.go

@@ -0,0 +1,69 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"io"
+	"os"
+
+	"github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard/go-test-transform/pkg/transformer"
+)
+
+func main() {
+	// Parse command-line flags
+	var (
+		ignoreAll  bool
+		inputFile  string
+		outputFile string
+	)
+
+	flag.BoolVar(&ignoreAll, "ignore-all", false, "Ignore all subtest failures")
+	flag.StringVar(&inputFile, "input", "", "Input JSON file (if not provided, reads from stdin)")
+	flag.StringVar(&outputFile, "output", "", "File to write the report to (default: stdout)")
+	flag.Parse()
+
+	// Set up options
+	opts := transformer.NewOptions(ignoreAll)
+
+	// Determine input source
+	var input io.Reader
+	if inputFile != "" {
+		// Read from file
+		file, err := os.Open(inputFile)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "Error opening input file: %v\n", err)
+			os.Exit(1)
+		}
+		defer file.Close()
+		input = file
+	} else {
+		// Read from stdin
+		input = os.Stdin
+	}
+
+	// Determine output destination
+	var output io.Writer
+	if outputFile != "" {
+		// Write to file
+		file, err := os.Create(outputFile)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "Error creating output file: %v\n", err)
+			os.Exit(1)
+		}
+		defer file.Close()
+		output = file
+	} else {
+		// Write to stdout
+		output = os.Stdout
+	}
+
+	// Transform the output
+	err := transformer.TransformJSON(input, output, opts)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error transforming JSON: %v\n", err)
+		os.Exit(1)
+	}
+
+	// Exit with the appropriate code
+	os.Exit(0)
+}

tools/flakeguard/go-test-transform/pkg/transformer/options.go

@@ -0,0 +1,21 @@
+package transformer
+
+// Options defines configuration options for the test transformer
+type Options struct {
+	// IgnoreAllSubtestFailures determines if all subtest failures should be ignored
+	IgnoreAllSubtestFailures bool
+}
+
+// DefaultOptions returns a new Options with default values
+func DefaultOptions() *Options {
+	return &Options{
+		IgnoreAllSubtestFailures: false,
+	}
+}
+
+// NewOptions creates a new Options with the specified parameters
+func NewOptions(ignoreAll bool) *Options {
+	return &Options{
+		IgnoreAllSubtestFailures: ignoreAll,
+	}
+}