feat(multiclient): adds timeouts to retryWithBackups and dialWithRetry (#84)

This PR enhances the `MultiClient` implementation by adding timeout
support and improving test coverage.

Timeout Handling:

- Added `Timeout` and `DialTimeout` to `RetryConfig`, with defaults for
RPC and dial operations.
- Updated `retryWithBackups` and `dialWithRetry` to use
`context.WithTimeout`, enforcing timeouts for RPC calls and connections.

Context Propagation:

- All MultiClient methods (SendTransaction, CallContract, etc.) now pass
context.Context through to support cancellation and timeouts
consistently.

Test Improvements:

- Added `TestMultiClient_dialWithRetry` to cover dial timeouts and
failure scenarios.
- Expanded `TestMultiClient_retryWithBackups` to simulate timeouts and
long-running operations using context.

Author: giogam

.changeset/tidy-items-shake.md

@@ -0,0 +1,5 @@
+---
+"chainlink-deployments-framework": patch
+---
+
+feat(multiclient): adds timeouts to retryWithBackups and dialWithRetry

deployment/multiclient.go

@@ -24,25 +24,31 @@ const (
 	// Default retry configuration for RPC calls
 	RPCDefaultRetryAttempts = 1
 	RPCDefaultRetryDelay    = 1000 * time.Millisecond
+	RPCDefaultRetryTimeout  = 10 * time.Second
 
 	// Default retry configuration for dialing RPC endpoints
 	RPCDefaultDialRetryAttempts = 1
 	RPCDefaultDialRetryDelay    = 1000 * time.Millisecond
+	RPCDefaultDialTimeout       = 10 * time.Second
 )
 
 type RetryConfig struct {
 	Attempts     uint
 	Delay        time.Duration
+	Timeout      time.Duration
 	DialAttempts uint
 	DialDelay    time.Duration
+	DialTimeout  time.Duration
 }
 
 func defaultRetryConfig() RetryConfig {
 	return RetryConfig{
 		Attempts:     RPCDefaultRetryAttempts,
 		Delay:        RPCDefaultRetryDelay,
+		Timeout:      RPCDefaultRetryTimeout,
 		DialAttempts: RPCDefaultDialRetryAttempts,
 		DialDelay:    RPCDefaultDialRetryDelay,
+		DialTimeout:  RPCDefaultDialTimeout,
 	}
 }
 
@@ -95,16 +101,16 @@ func NewMultiClient(lggr logger.Logger, rpcsCfg RPCConfig, opts ...func(client *
 }
 
 func (mc *MultiClient) SendTransaction(ctx context.Context, tx *types.Transaction) error {
-	return mc.retryWithBackups("SendTransaction", func(client *ethclient.Client) error {
-		return client.SendTransaction(ctx, tx)
+	return mc.retryWithBackups(ctx, "SendTransaction", func(ct context.Context, client *ethclient.Client) error {
+		return client.SendTransaction(ct, tx)
 	})
 }
 
 func (mc *MultiClient) CallContract(ctx context.Context, msg ethereum.CallMsg, blockNumber *big.Int) ([]byte, error) {
 	var result []byte
-	err := mc.retryWithBackups("CallContract", func(client *ethclient.Client) error {
+	err := mc.retryWithBackups(ctx, "CallContract", func(ct context.Context, client *ethclient.Client) error {
 		var err error
-		result, err = client.CallContract(ctx, msg, blockNumber)
+		result, err = client.CallContract(ct, msg, blockNumber)
 
 		return err
 	})
@@ -114,9 +120,9 @@ func (mc *MultiClient) CallContract(ctx context.Context, msg ethereum.CallMsg, b
 
 func (mc *MultiClient) CallContractAtHash(ctx context.Context, msg ethereum.CallMsg, blockHash common.Hash) ([]byte, error) {
 	var result []byte
-	err := mc.retryWithBackups("CallContractAtHash", func(client *ethclient.Client) error {
+	err := mc.retryWithBackups(ctx, "CallContractAtHash", func(ct context.Context, client *ethclient.Client) error {
 		var err error
-		result, err = client.CallContractAtHash(ctx, msg, blockHash)
+		result, err = client.CallContractAtHash(ct, msg, blockHash)
 
 		return err
 	})
@@ -126,9 +132,9 @@ func (mc *MultiClient) CallContractAtHash(ctx context.Context, msg ethereum.Call
 
 func (mc *MultiClient) CodeAt(ctx context.Context, account common.Address, blockNumber *big.Int) ([]byte, error) {
 	var code []byte
-	err := mc.retryWithBackups("CodeAt", func(client *ethclient.Client) error {
+	err := mc.retryWithBackups(ctx, "CodeAt", func(ct context.Context, client *ethclient.Client) error {
 		var err error
-		code, err = client.CodeAt(ctx, account, blockNumber)
+		code, err = client.CodeAt(ct, account, blockNumber)
 
 		return err
 	})
@@ -138,9 +144,9 @@ func (mc *MultiClient) CodeAt(ctx context.Context, account common.Address, block
 
 func (mc *MultiClient) CodeAtHash(ctx context.Context, account common.Address, blockHash common.Hash) ([]byte, error) {
 	var code []byte
-	err := mc.retryWithBackups("CodeAtHash", func(client *ethclient.Client) error {
+	err := mc.retryWithBackups(ctx, "CodeAtHash", func(ct context.Context, client *ethclient.Client) error {
 		var err error
-		code, err = client.CodeAtHash(ctx, account, blockHash)
+		code, err = client.CodeAtHash(ct, account, blockHash)
 
 		return err
 	})
@@ -150,9 +156,9 @@ func (mc *MultiClient) CodeAtHash(ctx context.Context, account common.Address, b
 
 func (mc *MultiClient) NonceAt(ctx context.Context, account common.Address, block *big.Int) (uint64, error) {
 	var count uint64
-	err := mc.retryWithBackups("NonceAt", func(client *ethclient.Client) error {
+	err := mc.retryWithBackups(ctx, "NonceAt", func(ct context.Context, client *ethclient.Client) error {
 		var err error
-		count, err = client.NonceAt(ctx, account, block)
+		count, err = client.NonceAt(ct, account, block)
 
 		return err
 	})
@@ -162,9 +168,9 @@ func (mc *MultiClient) NonceAt(ctx context.Context, account common.Address, bloc
 
 func (mc *MultiClient) NonceAtHash(ctx context.Context, account common.Address, blockHash common.Hash) (uint64, error) {
 	var count uint64
-	err := mc.retryWithBackups("NonceAtHash", func(client *ethclient.Client) error {
+	err := mc.retryWithBackups(ctx, "NonceAtHash", func(ct context.Context, client *ethclient.Client) error {
 		var err error
-		count, err = client.NonceAtHash(ctx, account, blockHash)
+		count, err = client.NonceAtHash(ct, account, blockHash)
 
 		return err
 	})
@@ -174,9 +180,9 @@ func (mc *MultiClient) NonceAtHash(ctx context.Context, account common.Address,
 
 func (mc *MultiClient) HeaderByNumber(ctx context.Context, number *big.Int) (*types.Header, error) {
 	var header *types.Header
-	err := mc.retryWithBackups("HeaderByNumber", func(client *ethclient.Client) error {
+	err := mc.retryWithBackups(ctx, "HeaderByNumber", func(ct context.Context, client *ethclient.Client) error {
 		var err error
-		header, err = client.HeaderByNumber(ctx, number)
+		header, err = client.HeaderByNumber(ct, number)
 
 		return err
 	})
@@ -186,9 +192,9 @@ func (mc *MultiClient) HeaderByNumber(ctx context.Context, number *big.Int) (*ty
 
 func (mc *MultiClient) SuggestGasPrice(ctx context.Context) (*big.Int, error) {
 	var gasPrice *big.Int
-	err := mc.retryWithBackups("SuggestGasPrice", func(client *ethclient.Client) error {
+	err := mc.retryWithBackups(ctx, "SuggestGasPrice", func(ct context.Context, client *ethclient.Client) error {
 		var err error
-		gasPrice, err = client.SuggestGasPrice(ctx)
+		gasPrice, err = client.SuggestGasPrice(ct)
 
 		return err
 	})
@@ -198,9 +204,9 @@ func (mc *MultiClient) SuggestGasPrice(ctx context.Context) (*big.Int, error) {
 
 func (mc *MultiClient) SuggestGasTipCap(ctx context.Context) (*big.Int, error) {
 	var gasTipCap *big.Int
-	err := mc.retryWithBackups("SuggestGasTipCap", func(client *ethclient.Client) error {
+	err := mc.retryWithBackups(ctx, "SuggestGasTipCap", func(ct context.Context, client *ethclient.Client) error {
 		var err error
-		gasTipCap, err = client.SuggestGasTipCap(ctx)
+		gasTipCap, err = client.SuggestGasTipCap(ct)
 
 		return err
 	})
@@ -210,9 +216,9 @@ func (mc *MultiClient) SuggestGasTipCap(ctx context.Context) (*big.Int, error) {
 
 func (mc *MultiClient) PendingCodeAt(ctx context.Context, account common.Address) ([]byte, error) {
 	var code []byte
-	err := mc.retryWithBackups("PendingCodeAt", func(client *ethclient.Client) error {
+	err := mc.retryWithBackups(ctx, "PendingCodeAt", func(ct context.Context, client *ethclient.Client) error {
 		var err error
-		code, err = client.PendingCodeAt(ctx, account)
+		code, err = client.PendingCodeAt(ct, account)
 
 		return err
 	})
@@ -222,9 +228,9 @@ func (mc *MultiClient) PendingCodeAt(ctx context.Context, account common.Address
 
 func (mc *MultiClient) PendingNonceAt(ctx context.Context, account common.Address) (uint64, error) {
 	var count uint64
-	err := mc.retryWithBackups("PendingNonceAt", func(client *ethclient.Client) error {
+	err := mc.retryWithBackups(ctx, "PendingNonceAt", func(ct context.Context, client *ethclient.Client) error {
 		var err error
-		count, err = client.PendingNonceAt(ctx, account)
+		count, err = client.PendingNonceAt(ct, account)
 
 		return err
 	})
@@ -234,9 +240,9 @@ func (mc *MultiClient) PendingNonceAt(ctx context.Context, account common.Addres
 
 func (mc *MultiClient) EstimateGas(ctx context.Context, call ethereum.CallMsg) (uint64, error) {
 	var gas uint64
-	err := mc.retryWithBackups("EstimateGas", func(client *ethclient.Client) error {
+	err := mc.retryWithBackups(ctx, "EstimateGas", func(ct context.Context, client *ethclient.Client) error {
 		var err error
-		gas, err = client.EstimateGas(ctx, call)
+		gas, err = client.EstimateGas(ct, call)
 
 		return err
 	})
@@ -246,9 +252,9 @@ func (mc *MultiClient) EstimateGas(ctx context.Context, call ethereum.CallMsg) (
 
 func (mc *MultiClient) BalanceAt(ctx context.Context, account common.Address, blockNumber *big.Int) (*big.Int, error) {
 	var balance *big.Int
-	err := mc.retryWithBackups("BalanceAt", func(client *ethclient.Client) error {
+	err := mc.retryWithBackups(ctx, "BalanceAt", func(ct context.Context, client *ethclient.Client) error {
 		var err error
-		balance, err = client.BalanceAt(ctx, account, blockNumber)
+		balance, err = client.BalanceAt(ct, account, blockNumber)
 
 		return err
 	})
@@ -258,9 +264,9 @@ func (mc *MultiClient) BalanceAt(ctx context.Context, account common.Address, bl
 
 func (mc *MultiClient) FilterLogs(ctx context.Context, q ethereum.FilterQuery) ([]types.Log, error) {
 	var logs []types.Log
-	err := mc.retryWithBackups("FilterLogs", func(client *ethclient.Client) error {
+	err := mc.retryWithBackups(ctx, "FilterLogs", func(ct context.Context, client *ethclient.Client) error {
 		var err error
-		logs, err = client.FilterLogs(ctx, q)
+		logs, err = client.FilterLogs(ct, q)
 
 		return err
 	})
@@ -270,16 +276,18 @@ func (mc *MultiClient) FilterLogs(ctx context.Context, q ethereum.FilterQuery) (
 
 func (mc *MultiClient) SubscribeFilterLogs(ctx context.Context, q ethereum.FilterQuery, ch chan<- types.Log) (ethereum.Subscription, error) {
 	var sub ethereum.Subscription
-	err := mc.retryWithBackups("SubscribeFilterLogs", func(client *ethclient.Client) error {
+	err := mc.retryWithBackups(ctx, "SubscribeFilterLogs", func(ct context.Context, client *ethclient.Client) error {
 		var err error
-		sub, err = client.SubscribeFilterLogs(ctx, q, ch)
+		sub, err = client.SubscribeFilterLogs(ct, q, ch)
 
 		return err
 	})
 
 	return sub, err
 }
 
+// WaitMined waits for a transaction to be mined and returns the receipt.
+// Note: retryConfig timeout settings are not used for this operation, a timeout can be set in the context.
 func (mc *MultiClient) WaitMined(ctx context.Context, tx *types.Transaction) (*types.Receipt, error) {
 	mc.lggr.Debugf("Waiting for tx %s to be mined for chain %s", tx.Hash().Hex(), mc.chainName)
 	// no retries here because we want to wait for the tx to be mined
@@ -318,13 +326,16 @@ func (mc *MultiClient) WaitMined(ctx context.Context, tx *types.Transaction) (*t
 	}
 }
 
-func (mc *MultiClient) retryWithBackups(opName string, op func(*ethclient.Client) error) error {
+func (mc *MultiClient) retryWithBackups(ctx context.Context, opName string, op func(context.Context, *ethclient.Client) error) error {
 	var err error
 	traceID := uuid.New()
 	for i, client := range append([]*ethclient.Client{mc.Client}, mc.Backups...) {
 		retryCount := 0
 		err2 := retry.Do(func() error {
-			err = op(client)
+			timeoutCtx, cancel := ensureTimeout(ctx, mc.RetryConfig.Timeout)
+			defer cancel()
+
+			err = op(timeoutCtx, client)
 			if err != nil {
 				mc.lggr.Warnf("traceID %q: chain %q: op: %q: client index %d: failed execution - retryable error '%s'", traceID.String(), mc.chainName, opName, i, MaybeDataErr(err))
 				return err
@@ -356,9 +367,12 @@ func (mc *MultiClient) dialWithRetry(rpc RPC, lggr logger.Logger) (*ethclient.Cl
 	var client *ethclient.Client
 	retryCount := 0
 	err = retry.Do(func() error {
+		ctx, cancel := context.WithTimeout(context.Background(), mc.RetryConfig.DialTimeout)
+		defer cancel()
+
 		var err2 error
 		mc.lggr.Debugf("traceID %q: chain %q: rpc: %q: dialing endpoint '%s'", traceID.String(), mc.chainName, rpc.Name, endpoint)
-		client, err2 = ethclient.Dial(endpoint)
+		client, err2 = ethclient.DialContext(ctx, endpoint)
 		if err2 != nil {
 			lggr.Warnf("traceID %q: chain %q: rpc: %q: dialing failed - retryable error: %s: %v", traceID.String(), mc.chainName, rpc.Name, endpoint, err2)
 			return err2
@@ -377,3 +391,17 @@ func (mc *MultiClient) dialWithRetry(rpc RPC, lggr logger.Logger) (*ethclient.Cl
 
 	return client, nil
 }
+
+// ensureTimeout checks if the parent context has a deadline.
+// If it does, it returns a new cancelable context using the parent's deadline.
+// If it doesn't, it creates a new context with the specified timeout.
+func ensureTimeout(parent context.Context, timeout time.Duration) (context.Context, context.CancelFunc) {
+	// check if the parent context already has a deadline
+	if _, hasDeadline := parent.Deadline(); hasDeadline {
+		// derive a new cancelable context from the parent context with the same deadline
+		return context.WithCancel(parent)
+	}
+
+	// create a new context with the specified timeout
+	return context.WithTimeout(parent, timeout)
+}