Skip to content

Commit 9a789cc

Browse files
annrpomannrpom
committed
db: enable new formats
1 parent 86e23cb commit 9a789cc

19 files changed

+1018
-323
lines changed

‎cockroachkvs/cockroachkvs_bench_test.go‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -79,6 +79,18 @@ func BenchmarkRandSeekInSST(b *testing.B) {
7979
valueLen: 128, // ~200 KVs per data block
8080
version: sstable.TableFormatPebblev7,
8181
},
82+
{
83+
name: "v8/single-level",
84+
numKeys: 200 * 100, // ~100 data blocks.
85+
valueLen: 128, // ~200 KVs per data block
86+
version: sstable.TableFormatPebblev8,
87+
},
88+
{
89+
name: "v8/two-level",
90+
numKeys: 200 * 5000, // ~5000 data blocks
91+
valueLen: 128, // ~200 KVs per data block
92+
version: sstable.TableFormatPebblev8,
93+
},
8294
}
8395
keyCfg := KeyGenConfig{
8496
PrefixAlphabetLen: 26,

‎format_major_version.go‎

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -253,6 +253,12 @@ const (
253253
// Previously, marking for compaction required a manifest rotation.
254254
FormatMarkForCompactionInVersionEdit
255255

256+
// FormatTieredStorage is a format major version that adds support for
257+
// tiered storage based on the age of a key-value pair. It introduces a new
258+
// columnar block format (among other things) that is required for tracking
259+
// the attribute used to derive the age.
260+
FormatTieredStorage
261+
256262
// -- Add new versions here --
257263

258264
// FormatNewest is the most recent format major version.
@@ -293,6 +299,8 @@ func (v FormatMajorVersion) resolveDefault() FormatMajorVersion {
293299
func (v FormatMajorVersion) MaxTableFormat() sstable.TableFormat {
294300
v = v.resolveDefault()
295301
switch {
302+
case v >= FormatTieredStorage:
303+
return sstable.TableFormatPebblev8
296304
case v >= formatFooterAttributes:
297305
return sstable.TableFormatPebblev7
298306
case v >= FormatTableFormatV6:
@@ -398,6 +406,9 @@ var formatMajorVersionMigrations = map[FormatMajorVersion]func(*DB) error{
398406
FormatMarkForCompactionInVersionEdit: func(d *DB) error {
399407
return d.finalizeFormatVersUpgrade(FormatMarkForCompactionInVersionEdit)
400408
},
409+
FormatTieredStorage: func(d *DB) error {
410+
return d.finalizeFormatVersUpgrade(FormatTieredStorage)
411+
},
401412
}
402413

403414
const formatVersionMarkerName = `format-version`
@@ -511,7 +522,11 @@ func (d *DB) ratchetFormatMajorVersionLocked(formatVers FormatMajorVersion) erro
511522
defer func() { d.mu.formatVers.ratcheting = false }()
512523

513524
for nextVers := d.FormatMajorVersion() + 1; nextVers <= formatVers; nextVers++ {
514-
if err := formatMajorVersionMigrations[nextVers](d); err != nil {
525+
migration, ok := formatMajorVersionMigrations[nextVers]
526+
if !ok || migration == nil {
527+
return errors.Errorf("pebble: no migration function defined for format version %d", nextVers)
528+
}
529+
if err := migration(d); err != nil {
515530
return errors.Wrapf(err, "migrating to version %d", nextVers)
516531
}
517532

‎format_major_version_test.go‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -38,11 +38,12 @@ func TestFormatMajorVersionStableValues(t *testing.T) {
3838
require.Equal(t, FormatV2BlobFiles, FormatMajorVersion(26))
3939
require.Equal(t, FormatBackingValueSize, FormatMajorVersion(27))
4040
require.Equal(t, FormatMarkForCompactionInVersionEdit, FormatMajorVersion(28))
41+
require.Equal(t, FormatTieredStorage, FormatMajorVersion(29))
4142

4243
// When we add a new version, we should add a check for the new version above
4344
// in addition to updating the expected values below.
44-
require.Equal(t, FormatNewest, FormatMajorVersion(28))
45-
require.Equal(t, internalFormatNewest, FormatMajorVersion(28))
45+
require.Equal(t, FormatNewest, FormatMajorVersion(29))
46+
require.Equal(t, internalFormatNewest, FormatMajorVersion(29))
4647
}
4748

4849
func TestFormatMajorVersion_MigrationDefined(t *testing.T) {
@@ -233,6 +234,7 @@ func TestFormatMajorVersions_TableFormat(t *testing.T) {
233234
FormatV2BlobFiles: {sstable.TableFormatPebblev1, sstable.TableFormatPebblev7},
234235
FormatBackingValueSize: {sstable.TableFormatPebblev1, sstable.TableFormatPebblev7},
235236
FormatMarkForCompactionInVersionEdit: {sstable.TableFormatPebblev1, sstable.TableFormatPebblev7},
237+
FormatTieredStorage: {sstable.TableFormatPebblev1, sstable.TableFormatPebblev8},
236238
}
237239

238240
// Valid versions.
@@ -260,6 +262,7 @@ func TestFormatMajorVersions_BlobFileFormat(t *testing.T) {
260262
FormatV2BlobFiles: blob.FileFormatV2,
261263
FormatBackingValueSize: blob.FileFormatV2,
262264
FormatMarkForCompactionInVersionEdit: blob.FileFormatV2,
265+
FormatTieredStorage: blob.FileFormatV2,
263266
}
264267

265268
// Valid versions.
@@ -297,7 +300,7 @@ func TestFormatMajorVersions_MaxTableFormat(t *testing.T) {
297300
},
298301
{
299302
fmv: FormatNewest,
300-
want: sstable.TableFormatPebblev7,
303+
want: sstable.TableFormatPebblev8,
301304
},
302305
}
303306
for _, tc := range testCases {

‎ingest_test.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -453,6 +453,7 @@ func TestIngestLinkFallback(t *testing.T) {
453453
}
454454
}
455455

456+
// TODO failing
456457
func TestOverlappingIngestedSSTs(t *testing.T) {
457458
dir := ""
458459
var (

‎sstable/testdata/rewriter_v8‎

Lines changed: 255 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,255 @@
1+
build block-size=1 index-block-size=1 filter
2+
[email protected]:a
3+
[email protected]:b
4+
[email protected]:c
5+
----
6+
point: [a@xyz#1,SET-c@xyz#1,SET]
7+
seqnums: [1-1]
8+
9+
rewrite from=@xyz to=@123 block-size=1 index-block-size=1 filter comparer=split-4b-suffix
10+
----
11+
rewrite failed: mismatched Comparer pebble.internal.testkeys vs comparer-split-4b-suffix, replacement requires same splitter to copy filters
12+
13+
build block-size=1 index-block-size=1 filter
14+
[email protected]:a
15+
[email protected]:b
16+
[email protected]:c
17+
----
18+
point: [aa@xyz#1,SET-ca@xyz#1,SET]
19+
seqnums: [1-1]
20+
21+
rewrite from=yz to=23 block-size=1 index-block-size=1 filter
22+
----
23+
rewrite failed: rewriting data blocks: key aa@xyz#1,SET has suffix 0x4078797a; require 0x797a
24+
25+
rewrite from=a@xyz to=a@123 block-size=1 index-block-size=1 filter
26+
----
27+
rewrite failed: rewriting data blocks: key aa@xyz#1,SET has suffix 0x4078797a; require 0x614078797a
28+
29+
build block-size=1 index-block-size=1 filter
30+
[email protected]:a
31+
[email protected]:b
32+
[email protected]:c
33+
----
34+
point: [a@0#1,SET-c@0#1,SET]
35+
seqnums: [1-1]
36+
37+
# TODO(sumeer): investigate why the data block sizes shrank from v7 to v8.
38+
# They should increase.
39+
layout
40+
----
41+
sstable
42+
├── data offset: 0 length: 82
43+
├── data offset: 87 length: 82
44+
├── data offset: 174 length: 82
45+
├── index offset: 261 length: 38
46+
├── index offset: 304 length: 39
47+
├── index offset: 348 length: 37
48+
├── top-index offset: 390 length: 52
49+
├── fullfilter.rocksdb.BuiltinBloomFilter offset: 447 length: 69
50+
├── tiering-histogram offset: 521 length: 52
51+
├── properties offset: 578 length: 494
52+
├── meta-index offset: 1077 length: 117
53+
└── footer offset: 1199 length: 61
54+
55+
scan
56+
----
57+
a@0#1,SET:a
58+
b@0#1,SET:b
59+
c@0#1,SET:c
60+
61+
get
62+
b@0
63+
f@0
64+
c@0
65+
----
66+
b
67+
get f@0: pebble: not found
68+
c
69+
70+
# TODO(sumeer): investigate why the data block lengths are 1 greater than the
71+
# original.
72+
rewrite from=@0 to=@123 block-size=1 index-block-size=1 filter
73+
----
74+
point: [a@123#1,SET-c@123#1,SET]
75+
seqnums: [1-1]
76+
77+
layout
78+
----
79+
sstable
80+
├── data offset: 0 length: 83
81+
├── data offset: 88 length: 83
82+
├── data offset: 176 length: 83
83+
├── index offset: 264 length: 40
84+
├── index offset: 309 length: 41
85+
├── index offset: 355 length: 37
86+
├── top-index offset: 397 length: 56
87+
├── fullfilter.rocksdb.BuiltinBloomFilter offset: 458 length: 69
88+
├── tiering-histogram offset: 532 length: 52
89+
├── properties offset: 589 length: 494
90+
├── meta-index offset: 1088 length: 117
91+
└── footer offset: 1210 length: 61
92+
93+
scan
94+
----
95+
a@123#1,SET:a
96+
b@123#1,SET:b
97+
c@123#1,SET:c
98+
99+
get
100+
b@123
101+
f@123
102+
c@123
103+
----
104+
b
105+
get f@123: pebble: not found
106+
c
107+
108+
rewrite from=@123 to=@456 block-size=1 index-block-size=1 filter concurrency=2
109+
----
110+
point: [a@456#1,SET-c@456#1,SET]
111+
seqnums: [1-1]
112+
113+
layout
114+
----
115+
sstable
116+
├── data offset: 0 length: 83
117+
├── data offset: 88 length: 83
118+
├── data offset: 176 length: 83
119+
├── index offset: 264 length: 40
120+
├── index offset: 309 length: 41
121+
├── index offset: 355 length: 37
122+
├── top-index offset: 397 length: 56
123+
├── fullfilter.rocksdb.BuiltinBloomFilter offset: 458 length: 69
124+
├── tiering-histogram offset: 532 length: 52
125+
├── properties offset: 589 length: 494
126+
├── meta-index offset: 1088 length: 117
127+
└── footer offset: 1210 length: 61
128+
129+
scan
130+
----
131+
a@456#1,SET:a
132+
b@456#1,SET:b
133+
c@456#1,SET:c
134+
135+
get
136+
b@456
137+
f@456
138+
c@456
139+
----
140+
b
141+
get f@456: pebble: not found
142+
c
143+
144+
rewrite from=@456 to=@123 block-size=1 index-block-size=1 filter concurrency=3
145+
----
146+
point: [a@123#1,SET-c@123#1,SET]
147+
seqnums: [1-1]
148+
149+
layout
150+
----
151+
sstable
152+
├── data offset: 0 length: 83
153+
├── data offset: 88 length: 83
154+
├── data offset: 176 length: 83
155+
├── index offset: 264 length: 40
156+
├── index offset: 309 length: 41
157+
├── index offset: 355 length: 37
158+
├── top-index offset: 397 length: 56
159+
├── fullfilter.rocksdb.BuiltinBloomFilter offset: 458 length: 69
160+
├── tiering-histogram offset: 532 length: 52
161+
├── properties offset: 589 length: 494
162+
├── meta-index offset: 1088 length: 117
163+
└── footer offset: 1210 length: 61
164+
165+
scan
166+
----
167+
a@123#1,SET:a
168+
b@123#1,SET:b
169+
c@123#1,SET:c
170+
171+
get
172+
b@123
173+
f@123
174+
c@123
175+
----
176+
b
177+
get f@123: pebble: not found
178+
c
179+
180+
181+
rewrite from=@123 to=@0 block-size=1 index-block-size=1 filter concurrency=4
182+
----
183+
point: [a@0#1,SET-c@0#1,SET]
184+
seqnums: [1-1]
185+
186+
layout
187+
----
188+
sstable
189+
├── data offset: 0 length: 82
190+
├── data offset: 87 length: 82
191+
├── data offset: 174 length: 82
192+
├── index offset: 261 length: 38
193+
├── index offset: 304 length: 39
194+
├── index offset: 348 length: 37
195+
├── top-index offset: 390 length: 52
196+
├── fullfilter.rocksdb.BuiltinBloomFilter offset: 447 length: 69
197+
├── tiering-histogram offset: 521 length: 52
198+
├── properties offset: 578 length: 494
199+
├── meta-index offset: 1077 length: 117
200+
└── footer offset: 1199 length: 61
201+
202+
scan
203+
----
204+
a@0#1,SET:a
205+
b@0#1,SET:b
206+
c@0#1,SET:c
207+
208+
get
209+
b@0
210+
f@0
211+
c@0
212+
----
213+
b
214+
get f@0: pebble: not found
215+
c
216+
217+
# Rewrite a table that contain only range keys.
218+
219+
build block-size=1 index-block-size=1 filter
220+
Span: a-b:{(#1,RANGEKEYSET,@0)}
221+
Span: b-c:{(#1,RANGEKEYSET,@0)}
222+
Span: c-d:{(#1,RANGEKEYSET,@0)}
223+
----
224+
rangekey: [a#1,RANGEKEYSET-d#inf,RANGEKEYSET]
225+
seqnums: [1-1]
226+
227+
scan-range-key
228+
----
229+
a-b:{(#1,RANGEKEYSET,@0)}
230+
b-c:{(#1,RANGEKEYSET,@0)}
231+
c-d:{(#1,RANGEKEYSET,@0)}
232+
233+
rewrite from=@0 to=@123 block-size=1 index-block-size=1 filter
234+
----
235+
rangekey: [a#1,RANGEKEYSET-d#inf,RANGEKEYSET]
236+
seqnums: [1-1]
237+
238+
scan-range-key
239+
----
240+
a-b:{(#1,RANGEKEYSET,@123)}
241+
b-c:{(#1,RANGEKEYSET,@123)}
242+
c-d:{(#1,RANGEKEYSET,@123)}
243+
244+
build block-size=1 index-block-size=1 filter
245+
a.SET.1:a
246+
b.SET.1:b
247+
c.SET.1:c
248+
----
249+
point: [a#1,SET-c#1,SET]
250+
seqnums: [1-1]
251+
252+
rewrite from= to=@123 block-size=1 index-block-size=1 filter
253+
----
254+
point: [a@123#1,SET-c@123#1,SET]
255+
seqnums: [1-1]

0 commit comments

Comments
 (0)