Skip to content

[DNM] #1688

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
xzhangxian1008 wants to merge 11 commits into
base: master
Choose a base branch
from
Open

[DNM] #1688

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
a63649a
init
xzhangxian1008 Jul 1, 2025
4e959ea
add logs
xzhangxian1008 Jul 2, 2025
837d008
add log info
xzhangxian1008 Jul 3, 2025
99b7125
add more logs
gengliqi Jul 7, 2025
f888514
add logs
xzhangxian1008 Jul 8, 2025
d0f5da3
add more logs
gengliqi Jul 9, 2025
5d90000
add logs
xzhangxian1008 Aug 4, 2025
69fdd3c
merge master
xzhangxian1008 Aug 5, 2025
8f94345
tweaking
xzhangxian1008 Aug 5, 2025
324a104
revert invalidate valid region
xzhangxian1008 Aug 5, 2025
9711a69
fix a bug & add more logs
gengliqi Aug 13, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 44 additions & 4 deletions internal/locate/region_cache.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1057,6 +1057,15 @@ func (c *RegionCache) GetTiFlashRPCContext(bo *retry.Backoffer, id RegionVerID,

cachedRegion := c.GetCachedRegionWithRLock(id)
if !cachedRegion.isValid() {
isNil := cachedRegion == nil
syncFlag := false
expired := false
if !isNil {
syncFlag = cachedRegion.checkSyncFlags(needReloadOnAccess)
expired = !cachedRegion.checkRegionCacheTTL(time.Now().Unix())
}
logutil.Logger(bo.GetCtx()).Info("tcmsdebug GetTiFlashRPCContext cachedRegion is invalid", zap.String("id", id.String()),
zap.Bool("isNil", isNil), zap.Bool("syncFlag", syncFlag), zap.Bool("expired", expired))
return nil, nil
}

Expand All @@ -1077,10 +1086,12 @@ func (c *RegionCache) GetTiFlashRPCContext(bo *retry.Backoffer, id RegionVerID,
}
addr, err := c.getStoreAddr(bo, cachedRegion, store)
if err != nil {
logutil.Logger(bo.GetCtx()).Info("tcmsdebug GetTiFlashRPCContext Fail to get store addr", zap.String("addr", store.GetAddr()), zap.String("id", id.String()))
return nil, err
}
if len(addr) == 0 {
cachedRegion.invalidate(StoreNotFound)
logutil.Logger(bo.GetCtx()).Info("tcmsdebug GetTiFlashRPCContext address length is 0", zap.String("addr", store.GetAddr()), zap.String("id", id.String()))
return nil, nil
}
if store.getResolveState() == needCheck {
Expand Down Expand Up @@ -1112,6 +1123,7 @@ func (c *RegionCache) GetTiFlashRPCContext(bo *retry.Backoffer, id RegionVerID,
}

cachedRegion.invalidate(Other)
logutil.Logger(bo.GetCtx()).Info("tcmsdebug At the end", zap.Int64("access store num", int64(regionStore.accessStoreNum(tiFlashOnly))))
return nil, nil
}

Expand Down Expand Up @@ -1312,8 +1324,11 @@ func (c *RegionCache) BatchLocateKeyRanges(bo *retry.Backoffer, keyRanges []kv.K
if r == nil {
// region cache miss, add the cut range to uncachedRanges, load from PD later.
uncachedRanges = append(uncachedRanges, router.KeyRange{StartKey: keyRange.StartKey, EndKey: keyRange.EndKey})
logutil.Logger(bo.GetCtx()).Info("tcmsdebug 1 append uncached Ranges", zap.String("start", string(keyRange.StartKey)), zap.String("end", string(keyRange.EndKey)))
continue
}
verID := r.VerID()
logutil.Logger(bo.GetCtx()).Info("tcmsdebug tryFindRegionByKey get region", zap.String("id", verID.String()))
// region cache hit, add the region to cachedRegions.
cachedRegions = append(cachedRegions, r)
if r.ContainsByEnd(keyRange.EndKey) {
Expand All @@ -1329,10 +1344,13 @@ func (c *RegionCache) BatchLocateKeyRanges(bo *retry.Backoffer, keyRanges []kv.K
if err != nil {
return nil, err
}
debugInfo := "tcmsdebug tryFindRegionByKey scan region: "
for _, r = range batchRegionInCache {
if !r.Contains(keyRange.StartKey) { // uncached hole, load the rest regions
break outer
}
vid := r.VerID()
debugInfo = fmt.Sprintf("%s <region: %d, ver: %d, conf-ver: %d>", debugInfo, vid.GetID(), vid.GetVer(), vid.GetConfVer())
cachedRegions = append(cachedRegions, r)
lastRegion = r
if r.ContainsByEnd(keyRange.EndKey) {
Expand All @@ -1342,11 +1360,13 @@ func (c *RegionCache) BatchLocateKeyRanges(bo *retry.Backoffer, keyRanges []kv.K
}
keyRange.StartKey = r.EndKey()
}
logutil.Logger(bo.GetCtx()).Info(debugInfo)
if len(batchRegionInCache) < defaultRegionsPerBatch { // region cache miss, load the rest regions
break
}
}
if !containsAll {
logutil.Logger(bo.GetCtx()).Info("tcmsdebug 2 append uncached Ranges", zap.String("start", string(keyRange.StartKey)), zap.String("end", string(keyRange.EndKey)))
uncachedRanges = append(uncachedRanges, router.KeyRange{StartKey: keyRange.StartKey, EndKey: keyRange.EndKey})
}
}
Expand Down Expand Up @@ -1619,6 +1639,10 @@ func (c *RegionCache) OnSendFailForTiFlash(bo *retry.Backoffer, store *Store, re

// force reload region when retry all known peers in region.
if scheduleReload {
logutil.Logger(bo.GetCtx()).Info("tcmsdebug 1 set sync flag",
zap.Uint64("region", r.VerID().id),
zap.Uint64("ver", r.VerID().ver),
zap.Uint64("conf", r.VerID().confVer))
r.setSyncFlags(needReloadOnAccess)
}
}
Expand Down Expand Up @@ -1681,6 +1705,10 @@ func (c *RegionCache) OnSendFail(bo *retry.Backoffer, ctx *RPCContext, scheduleR

// force reload region when retry all known peers in region.
if scheduleReload {
logutil.Logger(bo.GetCtx()).Info("tcmsdebug 2 set sync flag",
zap.Uint64("region", r.VerID().id),
zap.Uint64("ver", r.VerID().ver),
zap.Uint64("conf", r.VerID().confVer))
r.setSyncFlags(needReloadOnAccess)
}

Expand Down Expand Up @@ -1805,6 +1833,7 @@ func (c *RegionCache) BatchLoadRegionsWithKeyRange(bo *retry.Backoffer, startKey
return
}
if len(regions) == 0 {
logutil.BgLogger().Info("tcmsdebug no region")
err = errors.Errorf("PD returned no region, start_key: %q, end_key: %q, encode_start_key: %q, encode_end_key: %q",
redact.Key(startKey), redact.Key(endKey),
redact.Key(c.codec.EncodeRegionKey(startKey)), redact.Key(c.codec.EncodeRegionKey(endKey)))
Expand All @@ -1814,12 +1843,16 @@ func (c *RegionCache) BatchLoadRegionsWithKeyRange(bo *retry.Backoffer, startKey
c.mu.Lock()
defer c.mu.Unlock()

debugInfo := "tcmsdebug for regions: "
// TODO(youjiali1995): scanRegions always fetch regions from PD and these regions don't contain buckets information
// for less traffic, so newly inserted regions in region cache don't have buckets information. We should improve it.
for _, region := range regions {
c.insertRegionToCache(region, false, false)
c.insertRegionToCache(region, true, false)
vid := region.VerID()
debugInfo = fmt.Sprintf("%s <region: %d, ver: %d, conf-ver: %d>", debugInfo, vid.GetID(), vid.GetVer(), vid.GetConfVer())
}

logutil.BgLogger().Info(debugInfo)
return
}

Expand All @@ -1841,9 +1874,13 @@ func (c *RegionCache) BatchLoadRegionsWithKeyRanges(bo *retry.Backoffer, keyRang
c.mu.Lock()
defer c.mu.Unlock()

debugInfo := "tcmsdebug for regions: "
for _, region := range regions {
c.insertRegionToCache(region, false, false)
c.insertRegionToCache(region, true, false)
vid := region.VerID()
debugInfo = fmt.Sprintf("%s <region: %d, ver: %d, conf-ver: %d>", debugInfo, vid.GetID(), vid.GetVer(), vid.GetConfVer())
}
logutil.BgLogger().Info(debugInfo)
return
}

Expand Down Expand Up @@ -1929,7 +1966,7 @@ func (mu *regionIndexMu) insertRegionToCache(cachedRegion *Region, invalidateOld
// So we should check the epoch.
if ok && (oldVer.GetVer() > newVer.GetVer() || oldVer.GetConfVer() > newVer.GetConfVer()) {
metrics.TiKVStaleRegionFromPDCounter.Inc()
logutil.BgLogger().Debug("get stale region",
logutil.BgLogger().Info("tcmsdebug get stale region",
zap.Uint64("region", newVer.GetID()), zap.Uint64("new-ver", newVer.GetVer()), zap.Uint64("new-conf", newVer.GetConfVer()),
zap.Uint64("old-ver", oldVer.GetVer()), zap.Uint64("old-conf", oldVer.GetConfVer()))
return false
Expand Down Expand Up @@ -1974,7 +2011,10 @@ func (mu *regionIndexMu) insertRegionToCache(cachedRegion *Region, invalidateOld
region.cachedRegion.invalidate(Other, !shouldCount)
}
}
// update related vars.
logutil.BgLogger().Info("tcmsdebug insert cachedRegion",
zap.Uint64("region", newVer.GetID()),
zap.Uint64("ver", newVer.GetVer()),
zap.Uint64("conf", newVer.GetConfVer()))
mu.regions[newVer] = cachedRegion
mu.latestVersions[newVer.id] = newVer
return true
Expand Down
14 changes: 13 additions & 1 deletion internal/locate/sorted_btree.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,7 @@ func (s *SortedRegions) AscendGreaterOrEqual(startKey, endKey []byte, limit int)
if len(endKey) > 0 && bytes.Compare(region.StartKey(), endKey) >= 0 {
return false
}
if !region.checkRegionCacheTTL(now) {
if !region.checkRegionCacheTTL(now) || region.checkSyncFlags(needReloadOnAccess) {
return false
}
if !region.Contains(lastStartKey) { // uncached hole
Expand Down Expand Up @@ -149,3 +149,15 @@ func (s *SortedRegions) ValidRegionsInBtree(ts int64) (len int) {
})
return
}

func (s *SortedRegions) getByRegionID(id RegionVerID) *Region {
var region *Region
s.b.Descend(func(item *btreeItem) bool {
if item.cachedRegion.VerID() == id {
region = item.cachedRegion
return false
}
return true
})
return region
}