Skip to content

Commit e714d52

Browse files
itsbilalitsbilal
committed
kvserver: Add ability to filter replicated spans in Select/Iterate
This change adds the ability to select for just the replicated span in rditer.Select and rditer.IterateReplicaKeySpans. Also adds a new rditer.IterateReplicaKeySpansShared that does a ScanInternal on just the user key span, to be able to collect metadata of shared sstables as well as any internal keys above them. We only use skip-shared iteration for the replicated user key span of a range, and in practice, only if it's a non-system range. Part of cockroachdb#103028. Epic: none Release note: None
1 parent 3771e33 commit e714d52

File tree

34 files changed

+434
-60
lines changed

34 files changed

+434
-60
lines changed

‎pkg/cli/debug.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -472,7 +472,7 @@ func runDebugRangeData(cmd *cobra.Command, args []string) error {
472472
defer snapshot.Close()
473473

474474
var results int
475-
return rditer.IterateReplicaKeySpans(&desc, snapshot, debugCtx.replicated,
475+
return rditer.IterateReplicaKeySpans(&desc, snapshot, debugCtx.replicated, rditer.ReplicatedSpansAll,
476476
func(iter storage.EngineIterator, _ roachpb.Span, keyType storage.IterKeyType) error {
477477
for ok := true; ok && err == nil; ok, err = iter.NextEngineKey() {
478478
switch keyType {

‎pkg/kv/kvserver/client_merge_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3812,7 +3812,7 @@ func TestStoreRangeMergeRaftSnapshot(t *testing.T) {
38123812
}
38133813
}
38143814

3815-
err := rditer.IterateReplicaKeySpans(inSnap.Desc, sendingEngSnapshot, true, /* replicatedOnly */
3815+
err := rditer.IterateReplicaKeySpans(inSnap.Desc, sendingEngSnapshot, true /* replicatedOnly */, rditer.ReplicatedSpansAll,
38163816
func(iter storage.EngineIterator, span roachpb.Span, keyType storage.IterKeyType) error {
38173817
fw, ok := sstFileWriters[string(span.Key)]
38183818
if !ok || !fw.span.Equal(span) {

‎pkg/kv/kvserver/rditer/BUILD.bazel‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,8 @@ go_library(
1515
"//pkg/storage",
1616
"//pkg/storage/enginepb",
1717
"//pkg/util/iterutil",
18+
"@com_github_cockroachdb_pebble//:pebble",
19+
"@com_github_cockroachdb_pebble//rangekey",
1820
],
1921
)
2022

‎pkg/kv/kvserver/rditer/replica_data_iter.go‎

Lines changed: 45 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,10 +11,14 @@
1111
package rditer
1212

1313
import (
14+
"context"
15+
1416
"github.com/cockroachdb/cockroach/pkg/keys"
1517
"github.com/cockroachdb/cockroach/pkg/roachpb"
1618
"github.com/cockroachdb/cockroach/pkg/storage"
1719
"github.com/cockroachdb/cockroach/pkg/util/iterutil"
20+
"github.com/cockroachdb/pebble"
21+
"github.com/cockroachdb/pebble/rangekey"
1822
)
1923

2024
// ReplicaDataIteratorOptions defines ReplicaMVCCDataIterator creation options.
@@ -315,16 +319,26 @@ func IterateReplicaKeySpans(
315319
desc *roachpb.RangeDescriptor,
316320
reader storage.Reader,
317321
replicatedOnly bool,
322+
replicatedSpansFilter ReplicatedSpansFilter,
318323
visitor func(storage.EngineIterator, roachpb.Span, storage.IterKeyType) error,
319324
) error {
320325
if !reader.ConsistentIterators() {
321326
panic("reader must provide consistent iterators")
322327
}
323328
var spans []roachpb.Span
324329
if replicatedOnly {
325-
spans = MakeReplicatedKeySpans(desc)
330+
spans = Select(desc.RangeID, SelectOpts{
331+
ReplicatedSpansFilter: replicatedSpansFilter,
332+
ReplicatedBySpan: desc.RSpan(),
333+
ReplicatedByRangeID: true,
334+
})
326335
} else {
327-
spans = makeAllKeySpans(desc)
336+
spans = Select(desc.RangeID, SelectOpts{
337+
ReplicatedBySpan: desc.RSpan(),
338+
ReplicatedSpansFilter: replicatedSpansFilter,
339+
ReplicatedByRangeID: true,
340+
UnreplicatedByRangeID: true,
341+
})
328342
}
329343
keyTypes := []storage.IterKeyType{storage.IterKeyTypePointsOnly, storage.IterKeyTypeRangesOnly}
330344
for _, span := range spans {
@@ -350,6 +364,35 @@ func IterateReplicaKeySpans(
350364
return nil
351365
}
352366

367+
// IterateReplicaKeySpansShared iterates over the range's user key span,
368+
// skipping any keys present in shared files. It calls the appropriate visitor
369+
// function for the type of key visited, namely, point keys, range deletes and
370+
// range keys. Shared files that are skipped during this iteration are also
371+
// surfaced through a dedicated visitor. Note that this method only iterates
372+
// over a range's user key span; IterateReplicaKeySpans must be called to
373+
// iterate over the other key spans.
374+
//
375+
// Must use a reader with consistent iterators.
376+
func IterateReplicaKeySpansShared(
377+
ctx context.Context,
378+
desc *roachpb.RangeDescriptor,
379+
reader storage.Reader,
380+
visitPoint func(key *pebble.InternalKey, val pebble.LazyValue, info pebble.IteratorLevel) error,
381+
visitRangeDel func(start, end []byte, seqNum uint64) error,
382+
visitRangeKey func(start, end []byte, keys []rangekey.Key) error,
383+
visitSharedFile func(sst *pebble.SharedSSTMeta) error,
384+
) error {
385+
if !reader.ConsistentIterators() {
386+
panic("reader must provide consistent iterators")
387+
}
388+
spans := Select(desc.RangeID, SelectOpts{
389+
ReplicatedSpansFilter: ReplicatedSpansUserOnly,
390+
ReplicatedBySpan: desc.RSpan(),
391+
})
392+
span := spans[0]
393+
return reader.ScanInternal(ctx, span.Key, span.EndKey, visitPoint, visitRangeDel, visitRangeKey, visitSharedFile)
394+
}
395+
353396
// IterateOptions instructs how points and ranges should be presented to visitor
354397
// and if iterators should be visited in forward or reverse order.
355398
// Reverse iterator are also positioned at the end of the range prior to being

‎pkg/kv/kvserver/rditer/replica_data_iter_test.go‎

Lines changed: 35 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -148,6 +148,7 @@ func verifyIterateReplicaKeySpans(
148148
desc *roachpb.RangeDescriptor,
149149
eng storage.Engine,
150150
replicatedOnly bool,
151+
replicatedSpansFilter ReplicatedSpansFilter,
151152
) {
152153
readWriter := eng.NewSnapshot()
153154
defer readWriter.Close()
@@ -161,7 +162,7 @@ func verifyIterateReplicaKeySpans(
161162
"pretty",
162163
})
163164

164-
require.NoError(t, IterateReplicaKeySpans(desc, readWriter, replicatedOnly,
165+
require.NoError(t, IterateReplicaKeySpans(desc, readWriter, replicatedOnly, replicatedSpansFilter,
165166
func(iter storage.EngineIterator, span roachpb.Span, keyType storage.IterKeyType) error {
166167
var err error
167168
for ok := true; ok && err == nil; ok, err = iter.NextEngineKey() {
@@ -180,12 +181,18 @@ func verifyIterateReplicaKeySpans(
180181
var err error
181182
mvccKey, err = key.ToMVCCKey()
182183
require.NoError(t, err)
184+
if replicatedSpansFilter == ReplicatedSpansExcludeUser && desc.KeySpan().AsRawSpanWithNoLocals().ContainsKey(key.Key) {
185+
t.Fatalf("unexpected user key when user key are expected to be skipped: %s", mvccKey)
186+
}
183187
} else { // lock key
184188
ltk, err := key.ToLockTableKey()
185189
require.NoError(t, err)
186190
mvccKey = storage.MVCCKey{
187191
Key: ltk.Key,
188192
}
193+
if replicatedSpansFilter == ReplicatedSpansUserOnly {
194+
t.Fatalf("unexpected lock table key when only table keys requested: %s", ltk.Key)
195+
}
189196
}
190197
tbl.Append([]string{
191198
span.String(),
@@ -271,21 +278,33 @@ func TestReplicaDataIterator(t *testing.T) {
271278
parName := fmt.Sprintf("r%d", tc.desc.RangeID)
272279
t.Run(parName, func(t *testing.T) {
273280
testutils.RunTrueAndFalse(t, "replicatedOnly", func(t *testing.T, replicatedOnly bool) {
274-
name := "all"
275-
if replicatedOnly {
276-
name = "replicatedOnly"
277-
}
278-
w := echotest.NewWalker(t, filepath.Join(path, parName, name))
281+
replicatedSpans := []ReplicatedSpansFilter{ReplicatedSpansAll, ReplicatedSpansExcludeUser, ReplicatedSpansUserOnly}
282+
for i := range replicatedSpans {
283+
replicatedKeysName := "all"
284+
switch replicatedSpans[i] {
285+
case ReplicatedSpansExcludeUser:
286+
replicatedKeysName = "exclude-user"
287+
case ReplicatedSpansUserOnly:
288+
replicatedKeysName = "user-only"
289+
}
290+
t.Run(fmt.Sprintf("replicatedSpans=%v", replicatedKeysName), func(t *testing.T) {
291+
name := "all"
292+
if replicatedOnly {
293+
name = "replicatedOnly"
294+
}
295+
w := echotest.NewWalker(t, filepath.Join(path, parName, name, replicatedKeysName))
279296

280-
w.Run(t, "output", func(t *testing.T) string {
281-
var innerBuf strings.Builder
282-
tbl := tablewriter.NewWriter(&innerBuf)
283-
// Print contents of the Replica according to the iterator.
284-
verifyIterateReplicaKeySpans(t, tbl, &tc.desc, eng, replicatedOnly)
297+
w.Run(t, "output", func(t *testing.T) string {
298+
var innerBuf strings.Builder
299+
tbl := tablewriter.NewWriter(&innerBuf)
300+
// Print contents of the Replica according to the iterator.
301+
verifyIterateReplicaKeySpans(t, tbl, &tc.desc, eng, replicatedOnly, replicatedSpans[i])
285302

286-
tbl.Render()
287-
return innerBuf.String()
288-
})(t)
303+
tbl.Render()
304+
return innerBuf.String()
305+
})(t)
306+
})
307+
}
289308
})
290309
})
291310
}
@@ -449,7 +468,7 @@ func TestReplicaDataIteratorGlobalRangeKey(t *testing.T) {
449468
}
450469

451470
var actualSpans []roachpb.Span
452-
require.NoError(t, IterateReplicaKeySpans(&desc, snapshot, replicatedOnly,
471+
require.NoError(t, IterateReplicaKeySpans(&desc, snapshot, replicatedOnly, ReplicatedSpansAll,
453472
func(iter storage.EngineIterator, span roachpb.Span, keyType storage.IterKeyType) error {
454473
// We should never see any point keys.
455474
require.Equal(t, storage.IterKeyTypeRangesOnly, keyType)
@@ -556,7 +575,7 @@ func benchReplicaEngineDataIterator(b *testing.B, numRanges, numKeysPerRange, va
556575

557576
for i := 0; i < b.N; i++ {
558577
for _, desc := range descs {
559-
err := IterateReplicaKeySpans(&desc, snapshot, false, /* replicatedOnly */
578+
err := IterateReplicaKeySpans(&desc, snapshot, false /* replicatedOnly */, ReplicatedSpansAll,
560579
func(iter storage.EngineIterator, _ roachpb.Span, _ storage.IterKeyType) error {
561580
var err error
562581
for ok := true; ok && err == nil; ok, err = iter.NextEngineKey() {

‎pkg/kv/kvserver/rditer/select.go‎

Lines changed: 40 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,19 @@ import (
1515
"github.com/cockroachdb/cockroach/pkg/roachpb"
1616
)
1717

18+
type ReplicatedSpansFilter int
19+
20+
const (
21+
// ReplicatedSpansAll includes all replicated spans, including user keys,
22+
// range descriptors, and lock keys.
23+
ReplicatedSpansAll ReplicatedSpansFilter = iota
24+
// ReplicatedSpansExcludeUser includes all replicated spans except for user keys.
25+
ReplicatedSpansExcludeUser
26+
// ReplicatedSpansUserOnly includes just user keys, and no other replicated
27+
// spans.
28+
ReplicatedSpansUserOnly
29+
)
30+
1831
// SelectOpts configures which spans for a Replica to return from Select.
1932
// A Replica comprises replicated (i.e. belonging to the state machine) spans
2033
// and unreplicated spans, and depending on external circumstances one may want
@@ -24,6 +37,10 @@ type SelectOpts struct {
2437
// key. This includes user keys, range descriptors, and locks (separated
2538
// intents).
2639
ReplicatedBySpan roachpb.RSpan
40+
// ReplicatedSpansFilter specifies which of the replicated spans indicated by
41+
// ReplicatedBySpan should be returned or excluded. The zero value,
42+
// ReplicatedSpansAll, returns all replicated spans.
43+
ReplicatedSpansFilter ReplicatedSpansFilter
2744
// ReplicatedByRangeID selects all RangeID-keyed replicated keys. An example
2845
// of a key that falls into this Span is the GCThresholdKey.
2946
ReplicatedByRangeID bool
@@ -60,27 +77,31 @@ func Select(rangeID roachpb.RangeID, opts SelectOpts) []roachpb.Span {
6077
// See also the comment on KeySpan.
6178
in := opts.ReplicatedBySpan
6279
adjustedIn := in.KeySpan()
63-
sl = append(sl, makeRangeLocalKeySpan(in))
80+
if opts.ReplicatedSpansFilter != ReplicatedSpansUserOnly {
81+
sl = append(sl, makeRangeLocalKeySpan(in))
6482

65-
// Lock table.
66-
{
67-
// Handle doubly-local lock table keys since range descriptor key
68-
// is a range local key that can have a replicated lock acquired on it.
69-
startRangeLocal, _ := keys.LockTableSingleKey(keys.MakeRangeKeyPrefix(in.Key), nil)
70-
endRangeLocal, _ := keys.LockTableSingleKey(keys.MakeRangeKeyPrefix(in.EndKey), nil)
71-
// Need adjusted start key to avoid overlapping with the local lock span right above.
72-
startGlobal, _ := keys.LockTableSingleKey(adjustedIn.Key.AsRawKey(), nil)
73-
endGlobal, _ := keys.LockTableSingleKey(adjustedIn.EndKey.AsRawKey(), nil)
74-
sl = append(sl, roachpb.Span{
75-
Key: startRangeLocal,
76-
EndKey: endRangeLocal,
77-
}, roachpb.Span{
78-
Key: startGlobal,
79-
EndKey: endGlobal,
80-
})
83+
// Lock table.
84+
{
85+
// Handle doubly-local lock table keys since range descriptor key
86+
// is a range local key that can have a replicated lock acquired on it.
87+
startRangeLocal, _ := keys.LockTableSingleKey(keys.MakeRangeKeyPrefix(in.Key), nil)
88+
endRangeLocal, _ := keys.LockTableSingleKey(keys.MakeRangeKeyPrefix(in.EndKey), nil)
89+
// Need adjusted start key to avoid overlapping with the local lock span right above.
90+
startGlobal, _ := keys.LockTableSingleKey(adjustedIn.Key.AsRawKey(), nil)
91+
endGlobal, _ := keys.LockTableSingleKey(adjustedIn.EndKey.AsRawKey(), nil)
92+
sl = append(sl, roachpb.Span{
93+
Key: startRangeLocal,
94+
EndKey: endRangeLocal,
95+
}, roachpb.Span{
96+
Key: startGlobal,
97+
EndKey: endGlobal,
98+
})
99+
}
100+
}
101+
if opts.ReplicatedSpansFilter != ReplicatedSpansExcludeUser {
102+
// Adjusted span because r1's "normal" keyspace starts only at LocalMax, not RKeyMin.
103+
sl = append(sl, adjustedIn.AsRawSpanWithNoLocals())
81104
}
82-
// Adjusted span because r1's "normal" keyspace starts only at LocalMax, not RKeyMin.
83-
sl = append(sl, adjustedIn.AsRawSpanWithNoLocals())
84105
}
85106
return sl
86107
}

‎pkg/kv/kvserver/rditer/select_test.go‎

Lines changed: 21 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -29,8 +29,9 @@ func TestSelect(t *testing.T) {
2929

3030
w := echotest.NewWalker(t, datapathutils.TestDataPath(t, t.Name()))
3131
for _, tc := range []struct {
32-
name string
33-
sp roachpb.RSpan
32+
name string
33+
sp roachpb.RSpan
34+
filter ReplicatedSpansFilter
3435
}{
3536
{
3637
name: "no_span",
@@ -49,6 +50,23 @@ func TestSelect(t *testing.T) {
4950
Key: roachpb.RKey("a"),
5051
EndKey: roachpb.RKey("c"),
5152
},
53+
filter: ReplicatedSpansAll,
54+
},
55+
{
56+
name: "r2_excludeuser",
57+
sp: roachpb.RSpan{
58+
Key: roachpb.RKey("a"),
59+
EndKey: roachpb.RKey("c"),
60+
},
61+
filter: ReplicatedSpansExcludeUser,
62+
},
63+
{
64+
name: "r2_useronly",
65+
sp: roachpb.RSpan{
66+
Key: roachpb.RKey("a"),
67+
EndKey: roachpb.RKey("c"),
68+
},
69+
filter: ReplicatedSpansUserOnly,
5270
},
5371
{
5472
name: "r3",
@@ -64,6 +82,7 @@ func TestSelect(t *testing.T) {
6482
for _, unreplicatedByRangeID := range []bool{false, true} {
6583
opts := SelectOpts{
6684
ReplicatedBySpan: tc.sp,
85+
ReplicatedSpansFilter: tc.filter,
6786
ReplicatedByRangeID: replicatedByRangeID,
6887
UnreplicatedByRangeID: unreplicatedByRangeID,
6988
}

‎pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r1/all/output‎ renamed to ‎pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r1/all/all/output‎

File renamed without changes.

‎pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r1/all/exclude-user/output‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
echo
2+
----
3+
+----------------------------------------+--------------------------------------------------------------+--------------+------------------------------------+------------------------------------------------------------------------------------+
4+
| SPAN | KEY HEX | ENDKEY HEX | VERSION HEX | PRETTY |
5+
+----------------------------------------+--------------------------------------------------------------+--------------+------------------------------------+------------------------------------------------------------------------------------+
6+
| /Local/RangeID/1/{r""-s""} | 016989726162632d120ce61c175eb445878c36dcf4062ada4c0001 | | | /Local/RangeID/1/r/AbortSpan/"0ce61c17-5eb4-4587-8c36-dcf4062ada4c" |
7+
| /Local/RangeID/1/{r""-s""} | 016989726162632d129855a1ef8eb94c06a106cab1dda78a2b0001 | | | /Local/RangeID/1/r/AbortSpan/"9855a1ef-8eb9-4c06-a106-cab1dda78a2b" |
8+
| /Local/RangeID/1/{r""-s""} | 016989726c67632d | | | /Local/RangeID/1/r/RangeGCThreshold |
9+
| /Local/RangeID/1/{r""-s""} | 016989727261736b | | | /Local/RangeID/1/r/RangeAppliedState |
10+
| /Local/RangeID/1/{r""-s""} | 01698972726c6c2d | | | /Local/RangeID/1/r/RangeLease |
11+
| /Local/RangeID/1/{r""-s""} | 016989723a61 | 016989723a78 | 000000000000000109 | /Local/RangeID/1/r":{a"-x"}/0.000000001,0 |
12+
| /Local/RangeID/1/{u""-v""} | 0169897572667462 | | | /Local/RangeID/1/u/RangeTombstone |
13+
| /Local/RangeID/1/{u""-v""} | 0169897572667468 | | | /Local/RangeID/1/u/RaftHardState |
14+
| /Local/RangeID/1/{u""-v""} | 016989757266746c0000000000000001 | | | /Local/RangeID/1/u/RaftLog/logIndex:1 |
15+
| /Local/RangeID/1/{u""-v""} | 016989757266746c0000000000000002 | | | /Local/RangeID/1/u/RaftLog/logIndex:2 |
16+
| /Local/RangeID/1/{u""-v""} | 01698975726c7274 | | | /Local/RangeID/1/u/RangeLastReplicaGCTimestamp |
17+
| /Local/RangeID/1/{u""-v""} | 016989753a61 | 016989753a78 | 000000000000000109 | /Local/RangeID/1/u":{a"-x"}/0.000000001,0 |
18+
| /Local/Range"{a"-b"} | 016b1261000172647363 | | 0000000000000001 | /Local/Range"a"/RangeDescriptor/0.000000001,0 |
19+
| /Local/Range"{a"-b"} | 016b1261000174786e2d0ce61c175eb445878c36dcf4062ada4c | | | /Local/Range"a"/Transaction/"0ce61c17-5eb4-4587-8c36-dcf4062ada4c" |
20+
| /Local/Range"{a"-b"} | 016b126100ff000174786e2d9855a1ef8eb94c06a106cab1dda78a2b | | | /Local/Range"a\x00"/Transaction/"9855a1ef-8eb9-4c06-a106-cab1dda78a2b" |
21+
| /Local/Range"{a"-b"} | 016b1261ffffffff000174786e2d295e727c8ca9437cbb5e8e2ebbad996f | | | /Local/Range"a\xff\xff\xff\xff"/Transaction/"295e727c-8ca9-437c-bb5e-8e2ebbad996f" |
22+
| /Local/Lock/Intent/Local/Range"{a"-b"} | 017a6b12016b126100ff01726473630001 | | 030ce61c175eb445878c36dcf4062ada4c | /Local/Range"a"/RangeDescriptor |
23+
| /Local/Lock/Intent"{a"-b"} | 017a6b12610001 | | 030ce61c175eb445878c36dcf4062ada4c | "a" |
24+
+----------------------------------------+--------------------------------------------------------------+--------------+------------------------------------+------------------------------------------------------------------------------------+

0 commit comments

Comments
 (0)