Skip to content

Commit 8aa9298

Browse files
wyxloadingwuyuxiang
wyxloading
and
wuyuxiang
authored
fix: cache auto retry on MOVED / ASK / ... (#701)
* fix: cache auto retry on MOVED / ASK / ... * fix: askingMultiCache should retry {Cmd} with ASKING * fix DoXCache testcase * Revert "fix DoXCache testcase" This reverts commit 182897e. * to: simplify error handler * fix: ClientSideCachingExecAbort cases * to: add testcase for askingMultiCache --------- Co-authored-by: wuyuxiang <[email protected]>
1 parent 95b5b32 commit 8aa9298

File tree

4 files changed

+178
-24
lines changed

4 files changed

+178
-24
lines changed

‎cluster.go‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -837,6 +837,9 @@ func askingMultiCache(cc conn, ctx context.Context, multi []CacheableTTL) *redis
837837
resps := cc.DoMulti(ctx, commands...)
838838
for i := 5; i < len(resps.s); i += 6 {
839839
if arr, err := resps.s[i].ToArray(); err != nil {
840+
if preErr := resps.s[i-1].Error(); preErr != nil { // if {Cmd} get a RedisError
841+
err = preErr
842+
}
840843
results.s = append(results.s, newErrResult(err))
841844
} else {
842845
results.s = append(results.s, newResult(arr[len(arr)-1], nil))

‎cluster_test.go‎

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5535,6 +5535,75 @@ func TestClusterClientMovedRetry(t *testing.T) {
55355535
})
55365536
}
55375537

5538+
func TestClusterClientCacheASKRetry(t *testing.T) {
5539+
defer ShouldNotLeaked(SetupLeakDetection())
5540+
5541+
setup := func() (*clusterClient, *mockConn) {
5542+
m := &mockConn{
5543+
DoFn: func(cmd Completed) RedisResult {
5544+
if strings.Join(cmd.Commands(), " ") == "CLUSTER SLOTS" {
5545+
return slotsMultiResp
5546+
}
5547+
return RedisResult{}
5548+
},
5549+
}
5550+
client, err := newClusterClient(
5551+
&ClientOption{InitAddress: []string{":0"}},
5552+
func(dst string, opt *ClientOption) conn { return m },
5553+
newRetryer(defaultRetryDelayFn),
5554+
)
5555+
if err != nil {
5556+
t.Fatalf("unexpected err %v", err)
5557+
}
5558+
return client, m
5559+
}
5560+
5561+
t.Run("DoCache Retry on ASK", func(t *testing.T) {
5562+
client, m := setup()
5563+
attempts := 0
5564+
m.DoCacheFn = func(cmd Cacheable, ttl time.Duration) RedisResult {
5565+
return newResult(RedisMessage{typ: '-', string: "ASK 0 :0"}, nil)
5566+
}
5567+
m.DoMultiFn = func(multi ...Completed) *redisresults {
5568+
attempts++
5569+
if attempts == 1 {
5570+
return &redisresults{s: []RedisResult{{}, {}, {}, {}, newResult(RedisMessage{typ: '-', string: "ASK 0 :0"}, nil), newResult(RedisMessage{typ: '_'}, nil)}}
5571+
}
5572+
return &redisresults{s: []RedisResult{{}, {}, {}, {}, {}, newResult(RedisMessage{typ: '*', values: []RedisMessage{{}, {}, {}, {}, {}, {typ: '+', string: "OK"}}}, nil)}}
5573+
}
5574+
resp := client.DoCache(context.Background(), client.B().Get().Key("a1").Cache(), 10*time.Second)
5575+
if v, err := resp.ToString(); err != nil || v != "OK" {
5576+
t.Fatalf("unexpected response %v %v", v, err)
5577+
}
5578+
if attempts != 2 {
5579+
t.Fatalf("expected 2 attempts, got %v", attempts)
5580+
}
5581+
})
5582+
5583+
t.Run("DoMultiCache Retry on ASK", func(t *testing.T) {
5584+
client, m := setup()
5585+
5586+
attempts := 0
5587+
m.DoMultiCacheFn = func(multi ...CacheableTTL) *redisresults {
5588+
return &redisresults{s: []RedisResult{newResult(RedisMessage{typ: '-', string: "ASK 0 :0"}, nil)}}
5589+
}
5590+
m.DoMultiFn = func(multi ...Completed) *redisresults {
5591+
attempts++
5592+
if attempts == 1 {
5593+
return &redisresults{s: []RedisResult{{}, {}, {}, {}, newResult(RedisMessage{typ: '-', string: "ASK 0 :0"}, nil), newResult(RedisMessage{typ: '_'}, nil)}}
5594+
}
5595+
return &redisresults{s: []RedisResult{{}, {}, {}, {}, {}, newResult(RedisMessage{typ: '*', values: []RedisMessage{{}, {}, {}, {}, {}, RedisMessage{typ: '+', string: "OK"}}}, nil)}}
5596+
}
5597+
resps := client.DoMultiCache(context.Background(), CT(client.B().Get().Key("a1").Cache(), 10*time.Second))
5598+
if v, err := resps[0].ToString(); err != nil || v != "OK" {
5599+
t.Fatalf("unexpected response %v %v", v, err)
5600+
}
5601+
if attempts != 2 {
5602+
t.Fatalf("expected 2 attempts, got %v", attempts)
5603+
}
5604+
})
5605+
}
5606+
55385607
//gocyclo:ignore
55395608
func TestClusterClient_SendReadOperationToReplicaNodeWriteOperationToPrimaryNode(t *testing.T) {
55405609
defer ShouldNotLeaked(SetupLeakDetection())

‎pipe.go‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1315,6 +1315,11 @@ func (p *pipe) DoCache(ctx context.Context, cmd Cacheable, ttl time.Duration) Re
13151315
if err != nil {
13161316
if _, ok := err.(*RedisError); ok {
13171317
err = ErrDoCacheAborted
1318+
if preErr := resp.s[3].Error(); preErr != nil { // if {cmd} get a RedisError
1319+
if _, ok := preErr.(*RedisError); ok {
1320+
err = preErr
1321+
}
1322+
}
13181323
}
13191324
p.cache.Cancel(ck, cc, err)
13201325
return newErrResult(err)
@@ -1379,6 +1384,11 @@ func (p *pipe) doCacheMGet(ctx context.Context, cmd Cacheable, ttl time.Duration
13791384
if err != nil {
13801385
if _, ok := err.(*RedisError); ok {
13811386
err = ErrDoCacheAborted
1387+
if preErr := resp.s[len(multi)-2].Error(); preErr != nil { // if {rewritten} get a RedisError
1388+
if _, ok := preErr.(*RedisError); ok {
1389+
err = preErr
1390+
}
1391+
}
13821392
}
13831393
for _, key := range rewritten.Commands()[1 : keys+1] {
13841394
p.cache.Cancel(key, mgetcc, err)
@@ -1474,6 +1484,11 @@ func (p *pipe) DoMultiCache(ctx context.Context, multi ...CacheableTTL) *redisre
14741484
if err := resp.s[i].Error(); err != nil {
14751485
if _, ok := err.(*RedisError); ok {
14761486
err = ErrDoCacheAborted
1487+
if preErr := resp.s[i-1].Error(); preErr != nil { // if {cmd} get a RedisError
1488+
if _, ok := preErr.(*RedisError); ok {
1489+
err = preErr
1490+
}
1491+
}
14771492
}
14781493
ck, cc := cmds.CacheKey(Cacheable(missing[i-1]))
14791494
p.cache.Cancel(ck, cc, err)
@@ -1497,6 +1512,11 @@ func (p *pipe) DoMultiCache(ctx context.Context, multi ...CacheableTTL) *redisre
14971512
if err != nil {
14981513
if _, ok := err.(*RedisError); ok {
14991514
err = ErrDoCacheAborted
1515+
if preErr := resp.s[i-1].Error(); preErr != nil { // if {cmd} get a RedisError
1516+
if _, ok := preErr.(*RedisError); ok {
1517+
err = preErr
1518+
}
1519+
}
15001520
}
15011521
results.s[j] = newErrResult(err)
15021522
} else {

‎pipe_test.go‎

Lines changed: 86 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -1316,25 +1316,45 @@ func TestClientSideCachingExecAbort(t *testing.T) {
13161316
go func() {
13171317
mock.Expect("CLIENT", "CACHING", "YES").
13181318
Expect("MULTI").
1319-
Expect("PTTL", "a").
1320-
Expect("GET", "a").
1319+
Expect("PTTL", "a1").
1320+
Expect("GET", "a1").
13211321
Expect("EXEC").
13221322
ReplyString("OK").
13231323
ReplyString("OK").
13241324
ReplyString("OK").
13251325
ReplyString("OK").
13261326
Reply(RedisMessage{typ: '_'})
1327+
mock.Expect("CLIENT", "CACHING", "YES").
1328+
Expect("MULTI").
1329+
Expect("PTTL", "a2").
1330+
Expect("GET", "a2").
1331+
Expect("EXEC").
1332+
ReplyString("OK").
1333+
ReplyString("OK").
1334+
ReplyError("MOVED 0 127.0.0.1").
1335+
ReplyError("MOVED 0 127.0.0.1").
1336+
Reply(RedisMessage{typ: '_'})
13271337
}()
13281338

1329-
v, err := p.DoCache(context.Background(), Cacheable(cmds.NewCompleted([]string{"GET", "a"})), 10*time.Second).ToMessage()
1330-
if err != ErrDoCacheAborted {
1331-
t.Errorf("unexpected err, got %v", err)
1332-
}
1333-
if v.IsCacheHit() {
1334-
t.Errorf("unexpected cache hit")
1335-
}
1336-
if v, entry := p.cache.Flight("a", "GET", time.Second, time.Now()); v.typ != 0 || entry != nil {
1337-
t.Errorf("unexpected cache value and entry %v %v", v, entry)
1339+
for i, key := range []string{"a1", "a2"} {
1340+
v, err := p.DoCache(context.Background(), Cacheable(cmds.NewCompleted([]string{"GET", key})), 10*time.Second).ToMessage()
1341+
if i == 0 {
1342+
if err != ErrDoCacheAborted {
1343+
t.Errorf("unexpected err, got %v", err)
1344+
}
1345+
} else {
1346+
if re, ok := err.(*RedisError); !ok {
1347+
t.Errorf("unexpected err, got %v", err)
1348+
} else if _, moved := re.IsMoved(); !moved {
1349+
t.Errorf("unexpected err, got %v", err)
1350+
}
1351+
}
1352+
if v.IsCacheHit() {
1353+
t.Errorf("unexpected cache hit")
1354+
}
1355+
if v, entry := p.cache.Flight(key, "GET", time.Second, time.Now()); v.typ != 0 || entry != nil {
1356+
t.Errorf("unexpected cache value and entry %v %v", v, entry)
1357+
}
13381358
}
13391359
}
13401360

@@ -1641,20 +1661,42 @@ func TestClientSideCachingExecAbortMGet(t *testing.T) {
16411661
ReplyString("OK").
16421662
ReplyString("OK").
16431663
Reply(RedisMessage{typ: '_'})
1664+
mock.Expect("CLIENT", "CACHING", "YES").
1665+
Expect("MULTI").
1666+
Expect("PTTL", "b1").
1667+
Expect("PTTL", "b2").
1668+
Expect("MGET", "b1", "b2").
1669+
Expect("EXEC").
1670+
ReplyString("OK").
1671+
ReplyString("OK").
1672+
ReplyString("OK").
1673+
ReplyString("OK").
1674+
ReplyError("MOVED 0 127.0.0.1").
1675+
Reply(RedisMessage{typ: '_'})
16441676
}()
16451677

1646-
v, err := p.DoCache(context.Background(), Cacheable(cmds.NewMGetCompleted([]string{"MGET", "a1", "a2"})), 10*time.Second).ToMessage()
1647-
if err != ErrDoCacheAborted {
1648-
t.Errorf("unexpected err, got %v", err)
1649-
}
1650-
if v.IsCacheHit() {
1651-
t.Errorf("unexpected cache hit")
1652-
}
1653-
if v, entry := p.cache.Flight("a1", "GET", time.Second, time.Now()); v.typ != 0 || entry != nil {
1654-
t.Errorf("unexpected cache value and entry %v %v", v, entry)
1655-
}
1656-
if v, entry := p.cache.Flight("a2", "GET", time.Second, time.Now()); v.typ != 0 || entry != nil {
1657-
t.Errorf("unexpected cache value and entry %v %v", v, entry)
1678+
for i, pair := range [][2]string{{"a1", "a2"}, {"b1", "b2"}} {
1679+
v, err := p.DoCache(context.Background(), Cacheable(cmds.NewMGetCompleted([]string{"MGET", pair[0], pair[1]})), 10*time.Second).ToMessage()
1680+
if i == 0 {
1681+
if err != ErrDoCacheAborted {
1682+
t.Errorf("unexpected err, got %v", err)
1683+
}
1684+
} else {
1685+
if re, ok := err.(*RedisError); !ok {
1686+
t.Errorf("unexpected err, got %v", err)
1687+
} else if _, moved := re.IsMoved(); !moved {
1688+
t.Errorf("unexpected err, got %v", err)
1689+
}
1690+
}
1691+
if v.IsCacheHit() {
1692+
t.Errorf("unexpected cache hit")
1693+
}
1694+
if v, entry := p.cache.Flight(pair[0], "GET", time.Second, time.Now()); v.typ != 0 || entry != nil {
1695+
t.Errorf("unexpected cache value and entry %v %v", v, entry)
1696+
}
1697+
if v, entry := p.cache.Flight(pair[1], "GET", time.Second, time.Now()); v.typ != 0 || entry != nil {
1698+
t.Errorf("unexpected cache value and entry %v %v", v, entry)
1699+
}
16581700
}
16591701
}
16601702

@@ -1925,6 +1967,11 @@ func TestClientSideCachingExecAbortDoMultiCache(t *testing.T) {
19251967
Expect("PTTL", "a2").
19261968
Expect("GET", "a2").
19271969
Expect("EXEC").
1970+
Expect("CLIENT", "CACHING", "YES").
1971+
Expect("MULTI").
1972+
Expect("PTTL", "a3").
1973+
Expect("GET", "a3").
1974+
Expect("EXEC").
19281975
ReplyString("OK").
19291976
ReplyString("OK").
19301977
ReplyString("OK").
@@ -1937,26 +1984,41 @@ func TestClientSideCachingExecAbortDoMultiCache(t *testing.T) {
19371984
ReplyString("OK").
19381985
ReplyString("OK").
19391986
ReplyString("OK").
1987+
Reply(RedisMessage{typ: '_'}).
1988+
ReplyString("OK").
1989+
ReplyString("OK").
1990+
ReplyString("OK").
1991+
ReplyError("MOVED 0 127.0.0.1").
19401992
Reply(RedisMessage{typ: '_'})
19411993
}()
19421994

19431995
arr := p.DoMultiCache(context.Background(), []CacheableTTL{
19441996
CT(Cacheable(cmds.NewCompleted([]string{"GET", "a1"})), time.Second*10),
19451997
CT(Cacheable(cmds.NewCompleted([]string{"GET", "a2"})), time.Second*10),
1998+
CT(Cacheable(cmds.NewCompleted([]string{"GET", "a3"})), time.Second*10),
19461999
}...).s
19472000
for i, resp := range arr {
19482001
v, err := resp.ToMessage()
19492002
if i == 0 {
19502003
if v.integer != 1 {
19512004
t.Errorf("unexpected cached response, expected %v, got %v", 1, v.integer)
19522005
}
1953-
} else {
2006+
} else if i == 1 {
19542007
if err != ErrDoCacheAborted {
19552008
t.Errorf("unexpected err, got %v", err)
19562009
}
19572010
if v.IsCacheHit() {
19582011
t.Errorf("unexpected cache hit")
19592012
}
2013+
} else if i == 2 {
2014+
if re, ok := err.(*RedisError); !ok {
2015+
t.Errorf("unexpected err, got %v", err)
2016+
} else if _, moved := re.IsMoved(); !moved {
2017+
t.Errorf("unexpected err, got %v", err)
2018+
}
2019+
if v.IsCacheHit() {
2020+
t.Errorf("unexpected cache hit")
2021+
}
19602022
}
19612023
}
19622024
if v, entry := p.cache.Flight("a1", "GET", time.Second, time.Now()); v.integer != 1 {

0 commit comments

Comments
 (0)