Merge branch 'main' of github.com:smartcontractkit/chainlink-testing-framework into useParrot

Author: kalverra

seth/.changeset/v1.51.1.md

@@ -0,0 +1 @@
+- Use more conservative gas tip estimation, i.e. call `FeeHistory` endpoint with tip percentile corresponding to given priory and then select median value

seth/gas_adjuster.go

@@ -504,38 +504,58 @@ func classifyCongestion(congestionMetric float64) string {
 }
 
 func (m *Client) HistoricalFeeData(priority string) (baseFee float64, historicalGasTipCap float64, err error) {
+	var percentileTip float64
+
+	// based on priority decide, which percentile to use to get historical tip values, when calling FeeHistory
+	switch priority {
+	case Priority_Degen:
+		percentileTip = 100
+	case Priority_Fast:
+		percentileTip = 99
+	case Priority_Standard:
+		percentileTip = 50
+	case Priority_Slow:
+		percentileTip = 25
+	default:
+		err = fmt.Errorf("unknown priority: %s", priority)
+		L.Debug().
+			Str("Priority", priority).
+			Msgf("Unknown priority: %s", err.Error())
+
+		return
+	}
+
 	estimator := NewGasEstimator(m)
-	stats, err := estimator.Stats(m.Cfg.Network.GasPriceEstimationBlocks, 99)
+	stats, err := estimator.Stats(m.Cfg.Network.GasPriceEstimationBlocks, percentileTip)
 	if err != nil {
 		L.Debug().
 			Msgf("Failed to get fee history due to: %s", err.Error())
 
 		return
 	}
 
+	// base fee should still be based on priority, because FeeHistory returns whole base fee history, not just the requested percentile
 	switch priority {
 	case Priority_Degen:
 		baseFee = stats.GasPrice.Max
-		historicalGasTipCap = stats.TipCap.Max
 	case Priority_Fast:
 		baseFee = stats.GasPrice.Perc99
-		historicalGasTipCap = stats.TipCap.Perc99
 	case Priority_Standard:
 		baseFee = stats.GasPrice.Perc50
-		historicalGasTipCap = stats.TipCap.Perc50
 	case Priority_Slow:
 		baseFee = stats.GasPrice.Perc25
-		historicalGasTipCap = stats.TipCap.Perc25
 	default:
 		err = fmt.Errorf("unknown priority: %s", priority)
 		L.Debug().
 			Str("Priority", priority).
 			Msgf("Unknown priority: %s", err.Error())
 
 		return
-
 	}
 
+	// since we have already requested reward percentiles based on priority, let's now use the median, i.e. most common tip
+	historicalGasTipCap = stats.TipCap.Perc50
+
 	return
 }
 

seth/gas_test.go

@@ -13,7 +13,7 @@ func TestGasEstimator(t *testing.T) {
 	c := newClient(t)
 	bn, err := c.Client.BlockNumber(context.Background())
 	require.NoError(t, err, "BlockNumber should not error")
-	for i := 0; i < 10; i++ {
+	for range 10 {
 		_, err := c.DeployContractFromContractStore(c.NewTXOpts(), "NetworkDebugSubContract")
 		require.NoError(t, err, "Deploying contract should not error")
 	}

seth/seth.toml

@@ -49,7 +49,7 @@ root_key_funds_buffer = 10 # 10 ether
 
 # feature-flagged expriments; first one sets funds return priority to 'slow' (core only!), second one
 # sets the tip/base fee to the higher value in case there's 3+ orders of magnitude difference between them
-experiments_enabled = ["slow_funds_return", "eip_1559_fee_equalizer"]
+# experiments_enabled = ["slow_funds_return", "eip_1559_fee_equalizer"]
 
 # when enabled when creating a new Seth client we will send 10k wei from root address to root address
 # to make sure transaction can be submitted and mined
@@ -81,6 +81,32 @@ key_sync_retries = 10
 [block_stats]
 rpc_requests_per_second_limit = 15
 
+[[networks]]
+name = "Ethereum_Mainnet"
+dial_timeout = "1m"
+transaction_timeout = "30s"
+eip_1559_dynamic_fees = true
+
+# automated gas estimation
+gas_price_estimation_enabled = true
+gas_price_estimation_blocks = 20
+gas_price_estimation_tx_priority = "standard"
+# how many times to try fetching & calculating gas price data in case of errors, defaults to 1 if empty
+gas_price_estimation_attempt_count = 2
+# urls_secret = ["xxx"]
+
+# gas limits
+transfer_gas_fee = 21_000
+# gas limit should be explicitly set only if you are connecting to a node that's incapable of estimating gas limit itself (should only happen for very old versions)
+# gas_limit = 14_000_000
+
+# manual settings, used when gas_price_estimation_enabled is false or when it fails
+# legacy transactions
+gas_price = 1_000_000_000
+# EIP-1559 transactions
+gas_fee_cap = 25_000_000_000
+gas_tip_cap = 5_000_000_000
+
 [[networks]]
 name = "Anvil"
 dial_timeout = "1m"

tools/flakeguard/cmd/run.go

@@ -41,6 +41,7 @@ var RunTestsCmd = &cobra.Command{
 		useShuffle, _ := cmd.Flags().GetBool("shuffle")
 		shuffleSeed, _ := cmd.Flags().GetString("shuffle-seed")
 		omitOutputsOnSuccess, _ := cmd.Flags().GetBool("omit-test-outputs-on-success")
+		ignoreParentFailuresOnSubtests, _ := cmd.Flags().GetBool("ignore-parent-failures-on-subtests")
 
 		outputDir := filepath.Dir(outputPath)
 		initialDirSize, err := getDirSize(outputDir)
@@ -63,7 +64,6 @@ var RunTestsCmd = &cobra.Command{
 		// Determine test packages
 		var testPackages []string
 		if len(testCmdStrings) == 0 {
-			// No custom command -> parse packages
 			if testPackagesJson != "" {
 				if err := json.Unmarshal([]byte(testPackagesJson), &testPackages); err != nil {
 					log.Error().Err(err).Msg("Error decoding test packages JSON")
@@ -79,18 +79,19 @@ var RunTestsCmd = &cobra.Command{
 
 		// Initialize the runner
 		testRunner := runner.Runner{
-			ProjectPath:          projectPath,
-			Verbose:              true,
-			RunCount:             runCount,
-			Timeout:              timeout,
-			Tags:                 tags,
-			UseRace:              useRace,
-			SkipTests:            skipTests,
-			SelectTests:          selectTests,
-			UseShuffle:           useShuffle,
-			ShuffleSeed:          shuffleSeed,
-			OmitOutputsOnSuccess: omitOutputsOnSuccess,
-			MaxPassRatio:         maxPassRatio,
+			ProjectPath:                    projectPath,
+			Verbose:                        true,
+			RunCount:                       runCount,
+			Timeout:                        timeout,
+			Tags:                           tags,
+			UseRace:                        useRace,
+			SkipTests:                      skipTests,
+			SelectTests:                    selectTests,
+			UseShuffle:                     useShuffle,
+			ShuffleSeed:                    shuffleSeed,
+			OmitOutputsOnSuccess:           omitOutputsOnSuccess,
+			MaxPassRatio:                   maxPassRatio,
+			IgnoreParentFailuresOnSubtests: ignoreParentFailuresOnSubtests,
 		}
 
 		// Run the tests
@@ -105,7 +106,6 @@ var RunTestsCmd = &cobra.Command{
 				os.Exit(ErrorExitCode)
 			}
 		} else {
-			// Otherwise, use the normal go test approach
 			testReport, err = testRunner.RunTestPackages(testPackages)
 			if err != nil {
 				log.Fatal().Err(err).Msg("Error running test packages")
@@ -180,6 +180,7 @@ func init() {
 	RunTestsCmd.Flags().StringSlice("select-tests", nil, "Comma-separated list of test names to specifically run")
 	RunTestsCmd.Flags().Float64("max-pass-ratio", 1.0, "The maximum pass ratio threshold for a test to be considered flaky. Any tests below this pass rate will be considered flaky.")
 	RunTestsCmd.Flags().Bool("omit-test-outputs-on-success", true, "Omit test outputs and package outputs for tests that pass")
+	RunTestsCmd.Flags().Bool("ignore-parent-failures-on-subtests", false, "Ignore failures in parent tests when only subtests fail")
 }
 
 func checkDependencies(projectPath string) error {

tools/flakeguard/go-test-transform/README.md

@@ -0,0 +1,54 @@
+# Go Test Transform
+
+This utility transforms the output of Go's test JSON format to handle subtest failures more intelligently. It prevents parent tests from failing when only their subtests fail, while preserving the original test structure and output format.
+
+## Features
+
+- Transforms Go test JSON output to modify how test failures propagate
+- Parent tests won't fail if they only have failing subtests but no direct failures
+- Maintains the original JSON format for compatibility with other tools
+
+## Key Behavior
+
+- **Subtest Failure Handling**: When a subtest fails, the parent test will not be marked as failed unless the parent itself has a direct failure
+- **Important Note**: If a parent test has both log messages (`t.Log()`) and failing subtests, it will still be marked as failed, as seen in the `TestLogMessagesNotDirectFailures` test
+
+## Usage
+
+```bash
+go test -json ./... | go-test-transform -ignore-all
+```
+
+This will transform the test output to prevent parent tests from failing when only their subtests fail.
+
+## Options
+
+- `-ignore-all`: Ignore all subtest failures
+- `-input`: Input file (default: stdin)
+- `-output`: Output file (default: stdout)
+
+## Example
+
+For a test structure like:
+
+```go
+func TestParent(t *testing.T) {
+	t.Run("Subtest1", func(t *testing.T) {
+		t.Error("Subtest 1 failed")
+	})
+}
+```
+
+The transformed output will be:
+
+```json
+{"Time":"2023-05-10T15:04:05.123Z","Action":"run","Package":"example/pkg","Test":"TestParent"}
+{"Time":"2023-05-10T15:04:05.124Z","Action":"run","Package":"example/pkg","Test":"TestParent/Subtest1"}
+{"Time":"2023-05-10T15:04:05.125Z","Action":"output","Package":"example/pkg","Test":"TestParent/Subtest1","Output":"    subtest1_test.go:12: Subtest 1 failed\n"}
+{"Time":"2023-05-10T15:04:05.126Z","Action":"fail","Package":"example/pkg","Test":"TestParent/Subtest1","Elapsed":0.001}
+{"Time":"2023-05-10T15:04:05.127Z","Action":"pass","Package":"example/pkg","Test":"TestParent","Elapsed":0.004}
+{"Time":"2023-05-10T15:04:05.128Z","Action":"output","Package":"example/pkg","Output":"FAIL\texample/pkg\t0.004s\n"}
+{"Time":"2023-05-10T15:04:05.129Z","Action":"fail","Package":"example/pkg","Elapsed":0.005}
+```
+
+Note that in the original output, both `TestParent/Subtest1` and `TestParent` would be marked as failed. After transformation, `TestParent/Subtest1` remains failed, but `TestParent` is changed to pass since it doesn't have a direct failure.

tools/flakeguard/go-test-transform/cmd/main.go

@@ -0,0 +1,69 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"io"
+	"os"
+
+	"github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard/go-test-transform/pkg/transformer"
+)
+
+func main() {
+	// Parse command-line flags
+	var (
+		ignoreAll  bool
+		inputFile  string
+		outputFile string
+	)
+
+	flag.BoolVar(&ignoreAll, "ignore-all", false, "Ignore all subtest failures")
+	flag.StringVar(&inputFile, "input", "", "Input JSON file (if not provided, reads from stdin)")
+	flag.StringVar(&outputFile, "output", "", "File to write the report to (default: stdout)")
+	flag.Parse()
+
+	// Set up options
+	opts := transformer.NewOptions(ignoreAll)
+
+	// Determine input source
+	var input io.Reader
+	if inputFile != "" {
+		// Read from file
+		file, err := os.Open(inputFile)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "Error opening input file: %v\n", err)
+			os.Exit(1)
+		}
+		defer file.Close()
+		input = file
+	} else {
+		// Read from stdin
+		input = os.Stdin
+	}
+
+	// Determine output destination
+	var output io.Writer
+	if outputFile != "" {
+		// Write to file
+		file, err := os.Create(outputFile)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "Error creating output file: %v\n", err)
+			os.Exit(1)
+		}
+		defer file.Close()
+		output = file
+	} else {
+		// Write to stdout
+		output = os.Stdout
+	}
+
+	// Transform the output
+	err := transformer.TransformJSON(input, output, opts)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error transforming JSON: %v\n", err)
+		os.Exit(1)
+	}
+
+	// Exit with the appropriate code
+	os.Exit(0)
+}

tools/flakeguard/go-test-transform/pkg/transformer/options.go

@@ -0,0 +1,21 @@
+package transformer
+
+// Options defines configuration options for the test transformer
+type Options struct {
+	// IgnoreAllSubtestFailures determines if all subtest failures should be ignored
+	IgnoreAllSubtestFailures bool
+}
+
+// DefaultOptions returns a new Options with default values
+func DefaultOptions() *Options {
+	return &Options{
+		IgnoreAllSubtestFailures: false,
+	}
+}
+
+// NewOptions creates a new Options with the specified parameters
+func NewOptions(ignoreAll bool) *Options {
+	return &Options{
+		IgnoreAllSubtestFailures: ignoreAll,
+	}
+}