Skip to content

feat(multiclient): adds timeouts to retryWithBackups and dialWithRetry #84

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
giogam merged 8 commits into from
May 19, 2025
Merged

feat(multiclient): adds timeouts to retryWithBackups and dialWithRetry #84

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
86dad12
feat(multiclient): adds timeouts to dialWithRetry
giogam May 15, 2025
4390dfa
feat(multiclient): adds timeouts to retryWithBackups
giogam May 15, 2025
b247719
Create tidy-items-shake.md
giogam May 15, 2025
71e2364
Merge branch 'main' into CLD-252/add-a-default-request-timeout-to-mul…
giogam May 16, 2025
3566c19
feat: check errors in tests
giogam May 19, 2025
6cff0a9
feat(multiclient): make sure parent context deadlines are used
giogam May 19, 2025
ae836e6
chore(multiclient): fix linter issues
giogam May 19, 2025
2b3b0c8
chore(multiclient): reword comment
giogam May 19, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .changeset/tidy-items-shake.md
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
---
"chainlink-deployments-framework": patch
---

feat(multiclient): adds timeouts to retryWithBackups and dialWithRetry
98 changes: 63 additions & 35 deletions deployment/multiclient.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,25 +24,31 @@ const (
// Default retry configuration for RPC calls
RPCDefaultRetryAttempts = 1
RPCDefaultRetryDelay = 1000 * time.Millisecond
RPCDefaultRetryTimeout = 10 * time.Second

// Default retry configuration for dialing RPC endpoints
RPCDefaultDialRetryAttempts = 1
RPCDefaultDialRetryDelay = 1000 * time.Millisecond
RPCDefaultDialTimeout = 10 * time.Second
)

type RetryConfig struct {
Attempts uint
Delay time.Duration
Timeout time.Duration
DialAttempts uint
DialDelay time.Duration
DialTimeout time.Duration
}

func defaultRetryConfig() RetryConfig {
return RetryConfig{
Attempts: RPCDefaultRetryAttempts,
Delay: RPCDefaultRetryDelay,
Timeout: RPCDefaultRetryTimeout,
DialAttempts: RPCDefaultDialRetryAttempts,
DialDelay: RPCDefaultDialRetryDelay,
DialTimeout: RPCDefaultDialTimeout,
}
}

Expand Down Expand Up @@ -95,16 +101,16 @@ func NewMultiClient(lggr logger.Logger, rpcsCfg RPCConfig, opts ...func(client *
}

func (mc *MultiClient) SendTransaction(ctx context.Context, tx *types.Transaction) error {
return mc.retryWithBackups("SendTransaction", func(client *ethclient.Client) error {
return client.SendTransaction(ctx, tx)
return mc.retryWithBackups(ctx, "SendTransaction", func(ct context.Context, client *ethclient.Client) error {
return client.SendTransaction(ct, tx)
})
}

func (mc *MultiClient) CallContract(ctx context.Context, msg ethereum.CallMsg, blockNumber *big.Int) ([]byte, error) {
var result []byte
err := mc.retryWithBackups("CallContract", func(client *ethclient.Client) error {
err := mc.retryWithBackups(ctx, "CallContract", func(ct context.Context, client *ethclient.Client) error {
var err error
result, err = client.CallContract(ctx, msg, blockNumber)
result, err = client.CallContract(ct, msg, blockNumber)

return err
})
Expand All @@ -114,9 +120,9 @@ func (mc *MultiClient) CallContract(ctx context.Context, msg ethereum.CallMsg, b

func (mc *MultiClient) CallContractAtHash(ctx context.Context, msg ethereum.CallMsg, blockHash common.Hash) ([]byte, error) {
var result []byte
err := mc.retryWithBackups("CallContractAtHash", func(client *ethclient.Client) error {
err := mc.retryWithBackups(ctx, "CallContractAtHash", func(ct context.Context, client *ethclient.Client) error {
var err error
result, err = client.CallContractAtHash(ctx, msg, blockHash)
result, err = client.CallContractAtHash(ct, msg, blockHash)

return err
})
Expand All @@ -126,9 +132,9 @@ func (mc *MultiClient) CallContractAtHash(ctx context.Context, msg ethereum.Call

func (mc *MultiClient) CodeAt(ctx context.Context, account common.Address, blockNumber *big.Int) ([]byte, error) {
var code []byte
err := mc.retryWithBackups("CodeAt", func(client *ethclient.Client) error {
err := mc.retryWithBackups(ctx, "CodeAt", func(ct context.Context, client *ethclient.Client) error {
var err error
code, err = client.CodeAt(ctx, account, blockNumber)
code, err = client.CodeAt(ct, account, blockNumber)

return err
})
Expand All @@ -138,9 +144,9 @@ func (mc *MultiClient) CodeAt(ctx context.Context, account common.Address, block

func (mc *MultiClient) CodeAtHash(ctx context.Context, account common.Address, blockHash common.Hash) ([]byte, error) {
var code []byte
err := mc.retryWithBackups("CodeAtHash", func(client *ethclient.Client) error {
err := mc.retryWithBackups(ctx, "CodeAtHash", func(ct context.Context, client *ethclient.Client) error {
var err error
code, err = client.CodeAtHash(ctx, account, blockHash)
code, err = client.CodeAtHash(ct, account, blockHash)

return err
})
Expand All @@ -150,9 +156,9 @@ func (mc *MultiClient) CodeAtHash(ctx context.Context, account common.Address, b

func (mc *MultiClient) NonceAt(ctx context.Context, account common.Address, block *big.Int) (uint64, error) {
var count uint64
err := mc.retryWithBackups("NonceAt", func(client *ethclient.Client) error {
err := mc.retryWithBackups(ctx, "NonceAt", func(ct context.Context, client *ethclient.Client) error {
var err error
count, err = client.NonceAt(ctx, account, block)
count, err = client.NonceAt(ct, account, block)

return err
})
Expand All @@ -162,9 +168,9 @@ func (mc *MultiClient) NonceAt(ctx context.Context, account common.Address, bloc

func (mc *MultiClient) NonceAtHash(ctx context.Context, account common.Address, blockHash common.Hash) (uint64, error) {
var count uint64
err := mc.retryWithBackups("NonceAtHash", func(client *ethclient.Client) error {
err := mc.retryWithBackups(ctx, "NonceAtHash", func(ct context.Context, client *ethclient.Client) error {
var err error
count, err = client.NonceAtHash(ctx, account, blockHash)
count, err = client.NonceAtHash(ct, account, blockHash)

return err
})
Expand All @@ -174,9 +180,9 @@ func (mc *MultiClient) NonceAtHash(ctx context.Context, account common.Address,

func (mc *MultiClient) HeaderByNumber(ctx context.Context, number *big.Int) (*types.Header, error) {
var header *types.Header
err := mc.retryWithBackups("HeaderByNumber", func(client *ethclient.Client) error {
err := mc.retryWithBackups(ctx, "HeaderByNumber", func(ct context.Context, client *ethclient.Client) error {
var err error
header, err = client.HeaderByNumber(ctx, number)
header, err = client.HeaderByNumber(ct, number)

return err
})
Expand All @@ -186,9 +192,9 @@ func (mc *MultiClient) HeaderByNumber(ctx context.Context, number *big.Int) (*ty

func (mc *MultiClient) SuggestGasPrice(ctx context.Context) (*big.Int, error) {
var gasPrice *big.Int
err := mc.retryWithBackups("SuggestGasPrice", func(client *ethclient.Client) error {
err := mc.retryWithBackups(ctx, "SuggestGasPrice", func(ct context.Context, client *ethclient.Client) error {
var err error
gasPrice, err = client.SuggestGasPrice(ctx)
gasPrice, err = client.SuggestGasPrice(ct)

return err
})
Expand All @@ -198,9 +204,9 @@ func (mc *MultiClient) SuggestGasPrice(ctx context.Context) (*big.Int, error) {

func (mc *MultiClient) SuggestGasTipCap(ctx context.Context) (*big.Int, error) {
var gasTipCap *big.Int
err := mc.retryWithBackups("SuggestGasTipCap", func(client *ethclient.Client) error {
err := mc.retryWithBackups(ctx, "SuggestGasTipCap", func(ct context.Context, client *ethclient.Client) error {
var err error
gasTipCap, err = client.SuggestGasTipCap(ctx)
gasTipCap, err = client.SuggestGasTipCap(ct)

return err
})
Expand All @@ -210,9 +216,9 @@ func (mc *MultiClient) SuggestGasTipCap(ctx context.Context) (*big.Int, error) {

func (mc *MultiClient) PendingCodeAt(ctx context.Context, account common.Address) ([]byte, error) {
var code []byte
err := mc.retryWithBackups("PendingCodeAt", func(client *ethclient.Client) error {
err := mc.retryWithBackups(ctx, "PendingCodeAt", func(ct context.Context, client *ethclient.Client) error {
var err error
code, err = client.PendingCodeAt(ctx, account)
code, err = client.PendingCodeAt(ct, account)

return err
})
Expand All @@ -222,9 +228,9 @@ func (mc *MultiClient) PendingCodeAt(ctx context.Context, account common.Address

func (mc *MultiClient) PendingNonceAt(ctx context.Context, account common.Address) (uint64, error) {
var count uint64
err := mc.retryWithBackups("PendingNonceAt", func(client *ethclient.Client) error {
err := mc.retryWithBackups(ctx, "PendingNonceAt", func(ct context.Context, client *ethclient.Client) error {
var err error
count, err = client.PendingNonceAt(ctx, account)
count, err = client.PendingNonceAt(ct, account)

return err
})
Expand All @@ -234,9 +240,9 @@ func (mc *MultiClient) PendingNonceAt(ctx context.Context, account common.Addres

func (mc *MultiClient) EstimateGas(ctx context.Context, call ethereum.CallMsg) (uint64, error) {
var gas uint64
err := mc.retryWithBackups("EstimateGas", func(client *ethclient.Client) error {
err := mc.retryWithBackups(ctx, "EstimateGas", func(ct context.Context, client *ethclient.Client) error {
var err error
gas, err = client.EstimateGas(ctx, call)
gas, err = client.EstimateGas(ct, call)

return err
})
Expand All @@ -246,9 +252,9 @@ func (mc *MultiClient) EstimateGas(ctx context.Context, call ethereum.CallMsg) (

func (mc *MultiClient) BalanceAt(ctx context.Context, account common.Address, blockNumber *big.Int) (*big.Int, error) {
var balance *big.Int
err := mc.retryWithBackups("BalanceAt", func(client *ethclient.Client) error {
err := mc.retryWithBackups(ctx, "BalanceAt", func(ct context.Context, client *ethclient.Client) error {
var err error
balance, err = client.BalanceAt(ctx, account, blockNumber)
balance, err = client.BalanceAt(ct, account, blockNumber)

return err
})
Expand All @@ -258,9 +264,9 @@ func (mc *MultiClient) BalanceAt(ctx context.Context, account common.Address, bl

func (mc *MultiClient) FilterLogs(ctx context.Context, q ethereum.FilterQuery) ([]types.Log, error) {
var logs []types.Log
err := mc.retryWithBackups("FilterLogs", func(client *ethclient.Client) error {
err := mc.retryWithBackups(ctx, "FilterLogs", func(ct context.Context, client *ethclient.Client) error {
var err error
logs, err = client.FilterLogs(ctx, q)
logs, err = client.FilterLogs(ct, q)

return err
})
Expand All @@ -270,16 +276,18 @@ func (mc *MultiClient) FilterLogs(ctx context.Context, q ethereum.FilterQuery) (

func (mc *MultiClient) SubscribeFilterLogs(ctx context.Context, q ethereum.FilterQuery, ch chan<- types.Log) (ethereum.Subscription, error) {
var sub ethereum.Subscription
err := mc.retryWithBackups("SubscribeFilterLogs", func(client *ethclient.Client) error {
err := mc.retryWithBackups(ctx, "SubscribeFilterLogs", func(ct context.Context, client *ethclient.Client) error {
var err error
sub, err = client.SubscribeFilterLogs(ctx, q, ch)
sub, err = client.SubscribeFilterLogs(ct, q, ch)

return err
})

return sub, err
}

// WaitMined waits for a transaction to be mined and returns the receipt.
// Note: retryConfig timeout settings are not used for this operation, a timeout can be set in the context.
func (mc *MultiClient) WaitMined(ctx context.Context, tx *types.Transaction) (*types.Receipt, error) {
mc.lggr.Debugf("Waiting for tx %s to be mined for chain %s", tx.Hash().Hex(), mc.chainName)
// no retries here because we want to wait for the tx to be mined
Expand Down Expand Up @@ -318,13 +326,16 @@ func (mc *MultiClient) WaitMined(ctx context.Context, tx *types.Transaction) (*t
}
}

func (mc *MultiClient) retryWithBackups(opName string, op func(*ethclient.Client) error) error {
func (mc *MultiClient) retryWithBackups(ctx context.Context, opName string, op func(context.Context, *ethclient.Client) error) error {
var err error
traceID := uuid.New()
for i, client := range append([]*ethclient.Client{mc.Client}, mc.Backups...) {
retryCount := 0
err2 := retry.Do(func() error {
err = op(client)
timeoutCtx, cancel := ensureTimeout(ctx, mc.RetryConfig.Timeout)
defer cancel()

err = op(timeoutCtx, client)
if err != nil {
mc.lggr.Warnf("traceID %q: chain %q: op: %q: client index %d: failed execution - retryable error '%s'", traceID.String(), mc.chainName, opName, i, MaybeDataErr(err))
return err
Expand Down Expand Up @@ -356,9 +367,12 @@ func (mc *MultiClient) dialWithRetry(rpc RPC, lggr logger.Logger) (*ethclient.Cl
var client *ethclient.Client
retryCount := 0
err = retry.Do(func() error {
ctx, cancel := context.WithTimeout(context.Background(), mc.RetryConfig.DialTimeout)
defer cancel()

var err2 error
mc.lggr.Debugf("traceID %q: chain %q: rpc: %q: dialing endpoint '%s'", traceID.String(), mc.chainName, rpc.Name, endpoint)
client, err2 = ethclient.Dial(endpoint)
client, err2 = ethclient.DialContext(ctx, endpoint)
if err2 != nil {
lggr.Warnf("traceID %q: chain %q: rpc: %q: dialing failed - retryable error: %s: %v", traceID.String(), mc.chainName, rpc.Name, endpoint, err2)
return err2
Expand All @@ -377,3 +391,17 @@ func (mc *MultiClient) dialWithRetry(rpc RPC, lggr logger.Logger) (*ethclient.Cl

return client, nil
}

// ensureTimeout checks if the parent context has a deadline.
// If it does, it returns a new cancelable context using the parent's deadline.
// If it doesn't, it creates a new context with the specified timeout.
func ensureTimeout(parent context.Context, timeout time.Duration) (context.Context, context.CancelFunc) {
// check if the parent context already has a deadline
if _, hasDeadline := parent.Deadline(); hasDeadline {
// derive a new cancelable context from the parent context with the same deadline
return context.WithCancel(parent)
}

// create a new context with the specified timeout
return context.WithTimeout(parent, timeout)
}
Loading
Loading