[TT-1923] retry gas estimation in Seth (#1532)

Author: Tofel

.nancy-ignore

@@ -13,4 +13,5 @@ CVE-2024-32972 # CWE-400: Uncontrolled Resource Consumption ('Resource Exhaustio
 CVE-2023-42319 # CWE-noinfo: lol... go-ethereum v1.13.8 again
 CVE-2024-10086 # Improper Neutralization of Input During Web Page Generation ('Cross-site Scripting')
 CVE-2024-51744 # CWE-755: Improper Handling of Exceptional Conditions
-CVE-2024-45338 # CWE-770: Allocation of Resources Without Limits or Throttling
+CVE-2024-45338 # CWE-770: Allocation of Resources Without Limits or Throttling
+CVE-2024-45337 # CWE-863: Incorrect Authorization in golang.org/x/[email protected]

book/src/libs/seth.md

@@ -70,7 +70,7 @@ Reliable and debug-friendly Ethereum client
 - [ ] Tracing support (prestate)
 - [x] Tracing decoding
 - [x] Tracing tests
-- [ ] More tests for corner cases of decoding/tracing
+- [x] More tests for corner cases of decoding/tracing
 - [x] Saving of deployed contracts mapping (`address -> ABI_name`) for live networks
 - [x] Reading of deployed contracts mappings for live networks
 - [x] Automatic gas estimator (experimental)
@@ -233,7 +233,9 @@ client, err := NewClientBuilder().
     // EIP-1559 and gas estimations
     WithEIP1559DynamicFees(true).
     WithDynamicGasPrices(120_000_000_000, 44_000_000_000).
-    WithGasPriceEstimations(true, 10, seth.Priority_Fast).
+    // estimate gas prices based on the information from the RPC based on 10 last blocks
+    // adjust the value to fast priority and 3 attempts to get the estimations
+    WithGasPriceEstimations(true, 10, seth.Priority_Fast, 3).
     // gas bumping: retries, max gas price, bumping strategy function
     WithGasBumping(5, 100_000_000_000, PriorityBasedGasBumpingStrategyFn).
     Build()
@@ -441,6 +443,8 @@ For real networks, the estimation process differs for legacy transactions and th
 5. **Final Fee Calculation**: Sum the base fee and adjusted tip to set the `gas_fee_cap`.
 6. **Congestion Buffer**: Similar to legacy transactions, analyze congestion and apply a buffer to both the fee cap and the tip to secure transaction inclusion.
 
+Regardless of transaction type, if fetching data from RPC or calculating prices fails due to any issue and `gas_price_estimation_attempt_count` is > 1 we will retry it N-1 number of times.
+
 Understanding and setting these parameters correctly ensures that your transactions are processed efficiently and cost-effectively on the network.
 
 When fetching historical base fee and tip data, we will use the last `gas_price_estimation_blocks` blocks. If it's set to `0` we will default to `100` last blocks. If the blockchain has less than `100` blocks we will use all of them.
@@ -492,6 +496,14 @@ case Congestion_VeryHigh:
 For low congestion rate we will increase gas price by 10%, for medium by 20%, for high by 30% and for very high by 40%. We cache block header data in an in-memory cache, so we don't have to fetch it every time we estimate gas. The cache has capacity equal to `gas_price_estimation_blocks` and every time we add a new element, we remove one that is least frequently used and oldest (with block number being a constant and chain always moving forward it makes no sense to keep old blocks). It's important to know that in order to use congestion metrics we need to fetch at least 80% of the requested blocks. If that fails, we will skip this part of the estimation and only adjust the gas price based on priority.
 For both transaction types if any of the steps fails, we fall back to hardcoded values.
 
+##### Gas estimations attemps
+
+If for any reason fetching gas price suggestions or fee history from the RPC fails, or subsequent calulation of percentiles fails, it can be retried. This behaviour is controlled by `gas_price_estimation_attempt_count`, which if empty or set to `0` will default to
+just one attempt, which means that if it fails, it won't be retried. Set it to `2` to allow a single retry, etc.
+
+> [!NOTE]
+> To disable gas estimation set `gas_price_estimation_enabled` to `false`. Setting `gas_price_estimation_attempt_count` to `0` won't have such effect.
+
 ### DOT graphs
 
 There are multiple ways of visualising DOT graphs:

seth/block_stats.go

@@ -125,7 +125,7 @@ func (cs *BlockStats) CalculateBlockDurations(blocks []*types.Block) error {
 	totalSize := uint64(0)
 
 	for i := 1; i < len(blocks); i++ {
-		duration := time.Unix(int64(blocks[i].Time()), 0).Sub(time.Unix(int64(blocks[i-1].Time()), 0))
+		duration := time.Unix(mustSafeInt64(blocks[i].Time()), 0).Sub(time.Unix(mustSafeInt64(blocks[i-1].Time()), 0))
 		durations = append(durations, duration)
 		totalDuration += duration
 
@@ -155,7 +155,7 @@ func (cs *BlockStats) CalculateBlockDurations(blocks []*types.Block) error {
 
 		L.Debug().
 			Uint64("BlockNumber", blocks[i].Number().Uint64()).
-			Time("BlockTime", time.Unix(int64(blocks[i].Time()), 0)).
+			Time("BlockTime", time.Unix(mustSafeInt64(blocks[i].Time()), 0)).
 			Str("Duration", duration.String()).
 			Float64("GasUsedPercentage", calculateRatioPercentage(blocks[i].GasUsed(), blocks[i].GasLimit())).
 			Float64("TPS", tps).

seth/client.go

@@ -83,9 +83,12 @@ type Client struct {
 func NewClientWithConfig(cfg *Config) (*Client, error) {
 	initDefaultLogging()
 
-	err := ValidateConfig(cfg)
-	if err != nil {
-		return nil, err
+	if cfg == nil {
+		return nil, errors.New("seth config cannot be nil")
+	}
+
+	if cfgErr := cfg.Validate(); cfgErr != nil {
+		return nil, cfgErr
 	}
 
 	L.Debug().Msgf("Using tracing level: %s", cfg.TracingLevel)
@@ -155,73 +158,6 @@ func NewClientWithConfig(cfg *Config) (*Client, error) {
 	)
 }
 
-// ValidateConfig checks and validates the provided Config struct.
-// It ensures essential fields have valid values or default to appropriate values
-// when necessary. This function performs validation on gas price estimation,
-// gas limit, tracing level, trace outputs, network dial timeout, and pending nonce protection timeout.
-// If any configuration is invalid, it returns an error.
-func ValidateConfig(cfg *Config) error {
-	if cfg.Network.GasPriceEstimationEnabled {
-		if cfg.Network.GasPriceEstimationBlocks == 0 {
-			L.Debug().Msg("Gas estimation is enabled, but block headers to use is set to 0. Will not use block congestion for gas estimation")
-		}
-		cfg.Network.GasPriceEstimationTxPriority = strings.ToLower(cfg.Network.GasPriceEstimationTxPriority)
-
-		if cfg.Network.GasPriceEstimationTxPriority == "" {
-			cfg.Network.GasPriceEstimationTxPriority = Priority_Standard
-		}
-
-		switch cfg.Network.GasPriceEstimationTxPriority {
-		case Priority_Degen:
-		case Priority_Fast:
-		case Priority_Standard:
-		case Priority_Slow:
-		default:
-			return errors.New("when automating gas estimation is enabled priority must be fast, standard or slow. fix it or disable gas estimation")
-		}
-
-	}
-
-	if cfg.Network.GasLimit != 0 {
-		L.Warn().
-			Msg("Gas limit is set, this will override the gas limit set by the network. This option should be used **ONLY** if node is incapable of estimating gas limit itself, which happens only with very old versions")
-	}
-
-	if cfg.TracingLevel == "" {
-		cfg.TracingLevel = TracingLevel_Reverted
-	}
-
-	cfg.TracingLevel = strings.ToUpper(cfg.TracingLevel)
-
-	switch cfg.TracingLevel {
-	case TracingLevel_None:
-	case TracingLevel_Reverted:
-	case TracingLevel_All:
-	default:
-		return errors.New("tracing level must be one of: NONE, REVERTED, ALL")
-	}
-
-	for _, output := range cfg.TraceOutputs {
-		switch strings.ToLower(output) {
-		case TraceOutput_Console:
-		case TraceOutput_JSON:
-		case TraceOutput_DOT:
-		default:
-			return errors.New("trace output must be one of: console, json, dot")
-		}
-	}
-
-	if cfg.Network.DialTimeout == nil {
-		cfg.Network.DialTimeout = &Duration{D: DefaultDialTimeout}
-	}
-
-	if cfg.PendingNonceProtectionTimeout == nil {
-		cfg.PendingNonceProtectionTimeout = &Duration{D: DefaultPendingNonceProtectionTimeout}
-	}
-
-	return nil
-}
-
 // NewClient creates a new raw seth client with all deps setup from env vars
 func NewClient() (*Client, error) {
 	cfg, err := ReadConfig()
@@ -277,7 +213,7 @@ func NewClientRaw(
 		Addresses:   addrs,
 		PrivateKeys: pkeys,
 		URL:         cfg.FirstNetworkURL(),
-		ChainID:     int64(cfg.Network.ChainID),
+		ChainID:     mustSafeInt64(cfg.Network.ChainID),
 		Context:     ctx,
 		CancelFunc:  cancelFunc,
 	}
@@ -481,7 +417,7 @@ func (m *Client) TransferETHFromKey(ctx context.Context, fromKeyNum int, to stri
 	if err != nil {
 		gasLimit = m.Cfg.Network.TransferGasFee
 	} else {
-		gasLimit = int64(gasLimitRaw)
+		gasLimit = mustSafeInt64(gasLimitRaw)
 	}
 
 	if gasPrice == nil {
@@ -492,7 +428,7 @@ func (m *Client) TransferETHFromKey(ctx context.Context, fromKeyNum int, to stri
 		Nonce:    m.NonceManager.NextNonce(m.Addresses[fromKeyNum]).Uint64(),
 		To:       &toAddr,
 		Value:    value,
-		Gas:      uint64(gasLimit),
+		Gas:      mustSafeUint64(gasLimit),
 		GasPrice: gasPrice,
 	}
 	L.Debug().Interface("TransferTx", rawTx).Send()
@@ -607,7 +543,7 @@ func WithPending(pending bool) CallOpt {
 // WithBlockNumber sets blockNumber option for bind.CallOpts
 func WithBlockNumber(bn uint64) CallOpt {
 	return func(o *bind.CallOpts) {
-		o.BlockNumber = big.NewInt(int64(bn))
+		o.BlockNumber = big.NewInt(mustSafeInt64(bn))
 	}
 }
 
@@ -1002,7 +938,7 @@ func (m *Client) configureTransactionOpts(
 	estimations GasEstimations,
 	o ...TransactOpt,
 ) *bind.TransactOpts {
-	opts.Nonce = big.NewInt(int64(nonce))
+	opts.Nonce = big.NewInt(mustSafeInt64(nonce))
 	opts.GasPrice = estimations.GasPrice
 	opts.GasLimit = m.Cfg.Network.GasLimit
 

seth/client_api_test.go

@@ -124,6 +124,7 @@ func TestAPINonces(t *testing.T) {
 		{
 			name: "with nonce override",
 			transactionOpts: []seth.TransactOpt{
+				//nolint
 				seth.WithNonce(big.NewInt(int64(pnonce))),
 			},
 		},

seth/client_builder.go

@@ -23,17 +23,18 @@ type ClientBuilder struct {
 // NewClientBuilder creates a new ClientBuilder with reasonable default values. You only need to pass private key(s) and RPC URL to build a usable config.
 func NewClientBuilder() *ClientBuilder {
 	network := &Network{
-		Name:                         DefaultNetworkName,
-		EIP1559DynamicFees:           true,
-		TxnTimeout:                   MustMakeDuration(5 * time.Minute),
-		DialTimeout:                  MustMakeDuration(DefaultDialTimeout),
-		TransferGasFee:               DefaultTransferGasFee,
-		GasPriceEstimationEnabled:    true,
-		GasPriceEstimationBlocks:     200,
-		GasPriceEstimationTxPriority: Priority_Standard,
-		GasPrice:                     DefaultGasPrice,
-		GasFeeCap:                    DefaultGasFeeCap,
-		GasTipCap:                    DefaultGasTipCap,
+		Name:                           DefaultNetworkName,
+		EIP1559DynamicFees:             true,
+		TxnTimeout:                     MustMakeDuration(5 * time.Minute),
+		DialTimeout:                    MustMakeDuration(DefaultDialTimeout),
+		TransferGasFee:                 DefaultTransferGasFee,
+		GasPriceEstimationEnabled:      true,
+		GasPriceEstimationBlocks:       200,
+		GasPriceEstimationTxPriority:   Priority_Standard,
+		GasPrice:                       DefaultGasPrice,
+		GasFeeCap:                      DefaultGasFeeCap,
+		GasTipCap:                      DefaultGasTipCap,
+		GasPriceEstimationAttemptCount: DefaultGasPriceEstimationsAttemptCount,
 	}
 
 	return &ClientBuilder{
@@ -176,22 +177,25 @@ func (c *ClientBuilder) WithNetworkChainId(chainId uint64) *ClientBuilder {
 // WithGasPriceEstimations enables or disables gas price estimations, sets the number of blocks to use for estimation or transaction priority.
 // Even with estimations enabled you should still either set legacy gas price with `WithLegacyGasPrice()` or EIP-1559 dynamic fees with `WithDynamicGasPrices()`
 // ss they will be used as fallback values, if the estimations fail.
+// To disable gas price estimations, set the enabled parameter to false. Setting attemptCount to 0 won't disable it, it will be treated as "no value" and default to 1.
 // Following priorities are supported: "slow", "standard" and "fast"
-// Default values are true for enabled, 200 blocks for estimation and "standard" for priority.
-func (c *ClientBuilder) WithGasPriceEstimations(enabled bool, estimationBlocks uint64, txPriority string) *ClientBuilder {
+// Default values are true for enabled, 200 blocks for estimation, "standard" for priority and 1 attempt.
+func (c *ClientBuilder) WithGasPriceEstimations(enabled bool, estimationBlocks uint64, txPriority string, attemptCount uint) *ClientBuilder {
 	if !c.checkIfNetworkIsSet() {
 		return c
 	}
 	c.config.Network.GasPriceEstimationEnabled = enabled
 	c.config.Network.GasPriceEstimationBlocks = estimationBlocks
 	c.config.Network.GasPriceEstimationTxPriority = txPriority
+	c.config.Network.GasPriceEstimationAttemptCount = attemptCount
 	// defensive programming
 	if len(c.config.Networks) == 0 {
 		c.config.Networks = append(c.config.Networks, c.config.Network)
 	} else if net := c.config.findNetworkByName(c.config.Network.Name); net != nil {
 		net.GasPriceEstimationEnabled = enabled
 		net.GasPriceEstimationBlocks = estimationBlocks
 		net.GasPriceEstimationTxPriority = txPriority
+		net.GasPriceEstimationAttemptCount = attemptCount
 	}
 	return c
 }

seth/client_decode_test.go

@@ -10,7 +10,7 @@ import (
 )
 
 func TestSmokeDebugReverts(t *testing.T) {
-	c := newClient(t)
+	c := newClientWithContractMapFromEnv(t)
 
 	type test struct {
 		name   string
@@ -63,7 +63,7 @@ func TestSmokeDebugReverts(t *testing.T) {
 }
 
 func TestSmokeDebugData(t *testing.T) {
-	c := newClient(t)
+	c := newClientWithContractMapFromEnv(t)
 	c.Cfg.TracingLevel = seth.TracingLevel_All
 
 	type test struct {
@@ -226,9 +226,10 @@ func TestSmokeDebugData(t *testing.T) {
 				require.NoError(t, err)
 				require.Equal(t, dtx.Input, tc.output.Input)
 				require.Equal(t, dtx.Output, tc.output.Output)
+				require.Equal(t, len(tc.output.Events), len(dtx.Events))
 				for i, e := range tc.output.Events {
 					require.NotNil(t, dtx.Events[i])
-					require.Equal(t, dtx.Events[i].EventData, e.EventData)
+					require.Equal(t, e.EventData, dtx.Events[i].EventData)
 				}
 			}
 		})