Skip to content

Commit 4390dfa

Browse files
giogamgiogam
committed
feat(multiclient): adds timeouts to retryWithBackups
1 parent 86dad12 commit 4390dfa

File tree

2 files changed

+88
-66
lines changed

2 files changed

+88
-66
lines changed

‎deployment/multiclient.go‎

Lines changed: 40 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ const (
2424
// Default retry configuration for RPC calls
2525
RPCDefaultRetryAttempts = 1
2626
RPCDefaultRetryDelay = 1000 * time.Millisecond
27+
RPCDefaultRetryTimeout = 10 * time.Second
2728

2829
// Default retry configuration for dialing RPC endpoints
2930
RPCDefaultDialRetryAttempts = 1
@@ -34,6 +35,7 @@ const (
3435
type RetryConfig struct {
3536
Attempts uint
3637
Delay time.Duration
38+
Timeout time.Duration
3739
DialAttempts uint
3840
DialDelay time.Duration
3941
DialTimeout time.Duration
@@ -43,6 +45,7 @@ func defaultRetryConfig() RetryConfig {
4345
return RetryConfig{
4446
Attempts: RPCDefaultRetryAttempts,
4547
Delay: RPCDefaultRetryDelay,
48+
Timeout: RPCDefaultRetryTimeout,
4649
DialAttempts: RPCDefaultDialRetryAttempts,
4750
DialDelay: RPCDefaultDialRetryDelay,
4851
DialTimeout: RPCDefaultDialTimeout,
@@ -98,16 +101,16 @@ func NewMultiClient(lggr logger.Logger, rpcsCfg RPCConfig, opts ...func(client *
98101
}
99102

100103
func (mc *MultiClient) SendTransaction(ctx context.Context, tx *types.Transaction) error {
101-
return mc.retryWithBackups("SendTransaction", func(client *ethclient.Client) error {
102-
return client.SendTransaction(ctx, tx)
104+
return mc.retryWithBackups(ctx, "SendTransaction", func(ct context.Context, client *ethclient.Client) error {
105+
return client.SendTransaction(ct, tx)
103106
})
104107
}
105108

106109
func (mc *MultiClient) CallContract(ctx context.Context, msg ethereum.CallMsg, blockNumber *big.Int) ([]byte, error) {
107110
var result []byte
108-
err := mc.retryWithBackups("CallContract", func(client *ethclient.Client) error {
111+
err := mc.retryWithBackups(ctx, "CallContract", func(ct context.Context, client *ethclient.Client) error {
109112
var err error
110-
result, err = client.CallContract(ctx, msg, blockNumber)
113+
result, err = client.CallContract(ct, msg, blockNumber)
111114

112115
return err
113116
})
@@ -117,9 +120,9 @@ func (mc *MultiClient) CallContract(ctx context.Context, msg ethereum.CallMsg, b
117120

118121
func (mc *MultiClient) CallContractAtHash(ctx context.Context, msg ethereum.CallMsg, blockHash common.Hash) ([]byte, error) {
119122
var result []byte
120-
err := mc.retryWithBackups("CallContractAtHash", func(client *ethclient.Client) error {
123+
err := mc.retryWithBackups(ctx, "CallContractAtHash", func(ct context.Context, client *ethclient.Client) error {
121124
var err error
122-
result, err = client.CallContractAtHash(ctx, msg, blockHash)
125+
result, err = client.CallContractAtHash(ct, msg, blockHash)
123126

124127
return err
125128
})
@@ -129,9 +132,9 @@ func (mc *MultiClient) CallContractAtHash(ctx context.Context, msg ethereum.Call
129132

130133
func (mc *MultiClient) CodeAt(ctx context.Context, account common.Address, blockNumber *big.Int) ([]byte, error) {
131134
var code []byte
132-
err := mc.retryWithBackups("CodeAt", func(client *ethclient.Client) error {
135+
err := mc.retryWithBackups(ctx, "CodeAt", func(ct context.Context, client *ethclient.Client) error {
133136
var err error
134-
code, err = client.CodeAt(ctx, account, blockNumber)
137+
code, err = client.CodeAt(ct, account, blockNumber)
135138

136139
return err
137140
})
@@ -141,9 +144,9 @@ func (mc *MultiClient) CodeAt(ctx context.Context, account common.Address, block
141144

142145
func (mc *MultiClient) CodeAtHash(ctx context.Context, account common.Address, blockHash common.Hash) ([]byte, error) {
143146
var code []byte
144-
err := mc.retryWithBackups("CodeAtHash", func(client *ethclient.Client) error {
147+
err := mc.retryWithBackups(ctx, "CodeAtHash", func(ct context.Context, client *ethclient.Client) error {
145148
var err error
146-
code, err = client.CodeAtHash(ctx, account, blockHash)
149+
code, err = client.CodeAtHash(ct, account, blockHash)
147150

148151
return err
149152
})
@@ -153,9 +156,9 @@ func (mc *MultiClient) CodeAtHash(ctx context.Context, account common.Address, b
153156

154157
func (mc *MultiClient) NonceAt(ctx context.Context, account common.Address, block *big.Int) (uint64, error) {
155158
var count uint64
156-
err := mc.retryWithBackups("NonceAt", func(client *ethclient.Client) error {
159+
err := mc.retryWithBackups(ctx, "NonceAt", func(ct context.Context, client *ethclient.Client) error {
157160
var err error
158-
count, err = client.NonceAt(ctx, account, block)
161+
count, err = client.NonceAt(ct, account, block)
159162

160163
return err
161164
})
@@ -165,9 +168,9 @@ func (mc *MultiClient) NonceAt(ctx context.Context, account common.Address, bloc
165168

166169
func (mc *MultiClient) NonceAtHash(ctx context.Context, account common.Address, blockHash common.Hash) (uint64, error) {
167170
var count uint64
168-
err := mc.retryWithBackups("NonceAtHash", func(client *ethclient.Client) error {
171+
err := mc.retryWithBackups(ctx, "NonceAtHash", func(ct context.Context, client *ethclient.Client) error {
169172
var err error
170-
count, err = client.NonceAtHash(ctx, account, blockHash)
173+
count, err = client.NonceAtHash(ct, account, blockHash)
171174

172175
return err
173176
})
@@ -177,9 +180,9 @@ func (mc *MultiClient) NonceAtHash(ctx context.Context, account common.Address,
177180

178181
func (mc *MultiClient) HeaderByNumber(ctx context.Context, number *big.Int) (*types.Header, error) {
179182
var header *types.Header
180-
err := mc.retryWithBackups("HeaderByNumber", func(client *ethclient.Client) error {
183+
err := mc.retryWithBackups(ctx, "HeaderByNumber", func(ct context.Context, client *ethclient.Client) error {
181184
var err error
182-
header, err = client.HeaderByNumber(ctx, number)
185+
header, err = client.HeaderByNumber(ct, number)
183186

184187
return err
185188
})
@@ -189,9 +192,9 @@ func (mc *MultiClient) HeaderByNumber(ctx context.Context, number *big.Int) (*ty
189192

190193
func (mc *MultiClient) SuggestGasPrice(ctx context.Context) (*big.Int, error) {
191194
var gasPrice *big.Int
192-
err := mc.retryWithBackups("SuggestGasPrice", func(client *ethclient.Client) error {
195+
err := mc.retryWithBackups(ctx, "SuggestGasPrice", func(ct context.Context, client *ethclient.Client) error {
193196
var err error
194-
gasPrice, err = client.SuggestGasPrice(ctx)
197+
gasPrice, err = client.SuggestGasPrice(ct)
195198

196199
return err
197200
})
@@ -201,9 +204,9 @@ func (mc *MultiClient) SuggestGasPrice(ctx context.Context) (*big.Int, error) {
201204

202205
func (mc *MultiClient) SuggestGasTipCap(ctx context.Context) (*big.Int, error) {
203206
var gasTipCap *big.Int
204-
err := mc.retryWithBackups("SuggestGasTipCap", func(client *ethclient.Client) error {
207+
err := mc.retryWithBackups(ctx, "SuggestGasTipCap", func(ct context.Context, client *ethclient.Client) error {
205208
var err error
206-
gasTipCap, err = client.SuggestGasTipCap(ctx)
209+
gasTipCap, err = client.SuggestGasTipCap(ct)
207210

208211
return err
209212
})
@@ -213,9 +216,9 @@ func (mc *MultiClient) SuggestGasTipCap(ctx context.Context) (*big.Int, error) {
213216

214217
func (mc *MultiClient) PendingCodeAt(ctx context.Context, account common.Address) ([]byte, error) {
215218
var code []byte
216-
err := mc.retryWithBackups("PendingCodeAt", func(client *ethclient.Client) error {
219+
err := mc.retryWithBackups(ctx, "PendingCodeAt", func(ct context.Context, client *ethclient.Client) error {
217220
var err error
218-
code, err = client.PendingCodeAt(ctx, account)
221+
code, err = client.PendingCodeAt(ct, account)
219222

220223
return err
221224
})
@@ -225,9 +228,9 @@ func (mc *MultiClient) PendingCodeAt(ctx context.Context, account common.Address
225228

226229
func (mc *MultiClient) PendingNonceAt(ctx context.Context, account common.Address) (uint64, error) {
227230
var count uint64
228-
err := mc.retryWithBackups("PendingNonceAt", func(client *ethclient.Client) error {
231+
err := mc.retryWithBackups(ctx, "PendingNonceAt", func(ct context.Context, client *ethclient.Client) error {
229232
var err error
230-
count, err = client.PendingNonceAt(ctx, account)
233+
count, err = client.PendingNonceAt(ct, account)
231234

232235
return err
233236
})
@@ -237,9 +240,9 @@ func (mc *MultiClient) PendingNonceAt(ctx context.Context, account common.Addres
237240

238241
func (mc *MultiClient) EstimateGas(ctx context.Context, call ethereum.CallMsg) (uint64, error) {
239242
var gas uint64
240-
err := mc.retryWithBackups("EstimateGas", func(client *ethclient.Client) error {
243+
err := mc.retryWithBackups(ctx, "EstimateGas", func(ct context.Context, client *ethclient.Client) error {
241244
var err error
242-
gas, err = client.EstimateGas(ctx, call)
245+
gas, err = client.EstimateGas(ct, call)
243246

244247
return err
245248
})
@@ -249,9 +252,9 @@ func (mc *MultiClient) EstimateGas(ctx context.Context, call ethereum.CallMsg) (
249252

250253
func (mc *MultiClient) BalanceAt(ctx context.Context, account common.Address, blockNumber *big.Int) (*big.Int, error) {
251254
var balance *big.Int
252-
err := mc.retryWithBackups("BalanceAt", func(client *ethclient.Client) error {
255+
err := mc.retryWithBackups(ctx, "BalanceAt", func(ct context.Context, client *ethclient.Client) error {
253256
var err error
254-
balance, err = client.BalanceAt(ctx, account, blockNumber)
257+
balance, err = client.BalanceAt(ct, account, blockNumber)
255258

256259
return err
257260
})
@@ -261,9 +264,9 @@ func (mc *MultiClient) BalanceAt(ctx context.Context, account common.Address, bl
261264

262265
func (mc *MultiClient) FilterLogs(ctx context.Context, q ethereum.FilterQuery) ([]types.Log, error) {
263266
var logs []types.Log
264-
err := mc.retryWithBackups("FilterLogs", func(client *ethclient.Client) error {
267+
err := mc.retryWithBackups(ctx, "FilterLogs", func(ct context.Context, client *ethclient.Client) error {
265268
var err error
266-
logs, err = client.FilterLogs(ctx, q)
269+
logs, err = client.FilterLogs(ct, q)
267270

268271
return err
269272
})
@@ -273,9 +276,9 @@ func (mc *MultiClient) FilterLogs(ctx context.Context, q ethereum.FilterQuery) (
273276

274277
func (mc *MultiClient) SubscribeFilterLogs(ctx context.Context, q ethereum.FilterQuery, ch chan<- types.Log) (ethereum.Subscription, error) {
275278
var sub ethereum.Subscription
276-
err := mc.retryWithBackups("SubscribeFilterLogs", func(client *ethclient.Client) error {
279+
err := mc.retryWithBackups(ctx, "SubscribeFilterLogs", func(ct context.Context, client *ethclient.Client) error {
277280
var err error
278-
sub, err = client.SubscribeFilterLogs(ctx, q, ch)
281+
sub, err = client.SubscribeFilterLogs(ct, q, ch)
279282

280283
return err
281284
})
@@ -321,13 +324,16 @@ func (mc *MultiClient) WaitMined(ctx context.Context, tx *types.Transaction) (*t
321324
}
322325
}
323326

324-
func (mc *MultiClient) retryWithBackups(opName string, op func(*ethclient.Client) error) error {
327+
func (mc *MultiClient) retryWithBackups(ctx context.Context, opName string, op func(context.Context, *ethclient.Client) error) error {
325328
var err error
326329
traceID := uuid.New()
327330
for i, client := range append([]*ethclient.Client{mc.Client}, mc.Backups...) {
328331
retryCount := 0
329332
err2 := retry.Do(func() error {
330-
err = op(client)
333+
timeoutCtx, cancel := context.WithTimeout(ctx, mc.RetryConfig.Timeout)
334+
defer cancel()
335+
336+
err = op(timeoutCtx, client)
331337
if err != nil {
332338
mc.lggr.Warnf("traceID %q: chain %q: op: %q: client index %d: failed execution - retryable error '%s'", traceID.String(), mc.chainName, opName, i, MaybeDataErr(err))
333339
return err

‎deployment/multiclient_test.go‎

Lines changed: 48 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package deployment
22

33
import (
4+
"context"
45
"errors"
56
"testing"
67
"time"
@@ -107,22 +108,43 @@ func TestMultiClient_retryWithBackups(t *testing.T) {
107108
t.Parallel()
108109

109110
tests := []struct {
110-
name string
111-
URL string
112-
retryDelay time.Duration
113-
opName string
114-
op func(client *ethclient.Client) error
115-
wantErr string
111+
name string
112+
URL string
113+
retryConf RetryConfig
114+
call func(ctx context.Context, client *ethclient.Client) error
116115
}{
117116
{
118-
name: "All retries fail with http",
119-
URL: "http://example.com",
120-
retryDelay: 100,
121-
opName: "test-operation",
122-
op: func(client *ethclient.Client) error {
117+
// this test case triggers a context timeout error for all dial attempts.
118+
// without proper timeout the dial logic inside will hang forever and the test
119+
// will timeout.
120+
name: "All dial attempts fail due to context timeout",
121+
URL: "http://rpcs.cldev.sh/avalanche/test",
122+
retryConf: RetryConfig{
123+
Attempts: 2,
124+
Delay: 10 * time.Millisecond,
125+
Timeout: 3 * time.Second,
126+
},
127+
call: func(ctx context.Context, client *ethclient.Client) error {
123128
return errors.New("operation failed")
124129
},
125-
wantErr: "operation failed\nall backup clients failed for chain \"ethereum-testnet-sepolia\"",
130+
},
131+
{
132+
name: "All dial attempts fail due to malformed URL",
133+
URL: "http://rpcs.cldev.sh/avalanche/test",
134+
retryConf: RetryConfig{
135+
Attempts: 2,
136+
Delay: 10 * time.Millisecond,
137+
Timeout: 3 * time.Second,
138+
},
139+
call: func(ctx context.Context, client *ethclient.Client) error {
140+
// Simulate a long-running operation that will cause the context to timeout
141+
select {
142+
case <-ctx.Done():
143+
return ctx.Err()
144+
case <-time.After(5 * time.Second):
145+
return errors.New("operation failed")
146+
}
147+
},
126148
},
127149
}
128150

@@ -131,30 +153,24 @@ func TestMultiClient_retryWithBackups(t *testing.T) {
131153
t.Parallel()
132154

133155
lggr := logger.Test(t)
134-
chainSelector := uint64(16015286601757825753) // "ethereum-testnet-sepolia"
135-
136-
// Create MultiClient with retry configuration
137-
mc, err := NewMultiClient(lggr, RPCConfig{
138-
ChainSelector: chainSelector,
139-
RPCs: []RPC{
140-
{Name: "test-rpc", HTTPURL: tt.URL, PreferredURLScheme: URLSchemePreferenceHTTP},
141-
{Name: "test-rpc-2", HTTPURL: tt.URL, PreferredURLScheme: URLSchemePreferenceHTTP},
142-
},
143-
})
144156

157+
client, err := ethclient.Dial(tt.URL)
145158
require.NoError(t, err)
146-
require.NotNil(t, mc)
147159

148-
mc.RetryConfig.Delay = tt.retryDelay
149-
150-
// Run operation and check expectations
151-
err = mc.retryWithBackups(tt.opName, tt.op)
152-
if tt.wantErr != "" {
153-
require.Error(t, err)
154-
require.ErrorContains(t, err, tt.wantErr)
155-
} else {
156-
require.NoError(t, err)
160+
mc := MultiClient{
161+
Client: client,
162+
chainName: "ethereum-testnet-sepolia",
163+
RetryConfig: tt.retryConf,
164+
lggr: lggr,
157165
}
166+
167+
err = mc.retryWithBackups(
168+
context.Background(),
169+
"test-operation",
170+
tt.call,
171+
)
172+
173+
require.Error(t, err)
158174
})
159175
}
160176
}

0 commit comments

Comments
 (0)