Skip to content

Commit 51e41be

Browse files
EdwardX29EdwardX29
committed
sstable: set minlz compression to default for v6+
set minlz to default compression for v6+, have a guard against using minlz for v5 and earlier.
1 parent 6018d0e commit 51e41be

24 files changed

+363
-358
lines changed

‎metamorphic/options.go‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -786,14 +786,16 @@ func RandomOptions(
786786
lopts.FilterPolicy = newTestingFilterPolicy(1 << rng.IntN(5))
787787
}
788788

789-
// We use either no compression, snappy compression or zstd compression.
790-
switch rng.IntN(3) {
789+
// We use either no compression, snappy compression, zstd compression, or minlz compression.
790+
switch rng.IntN(4) {
791791
case 0:
792792
lopts.Compression = func() block.Compression { return pebble.NoCompression }
793793
case 1:
794794
lopts.Compression = func() block.Compression { return pebble.ZstdCompression }
795-
default:
795+
case 2:
796796
lopts.Compression = func() block.Compression { return pebble.SnappyCompression }
797+
default:
798+
lopts.Compression = func() block.Compression { return pebble.MinlzCompression }
797799
}
798800
opts.Levels = []pebble.LevelOptions{lopts}
799801

‎options.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2160,7 +2160,7 @@ func (o *Options) MakeBlobWriterOptions(level int) blob.FileWriterOptions {
21602160

21612161
func resolveDefaultCompression(c Compression) Compression {
21622162
if c <= DefaultCompression || c >= block.NCompression {
2163-
c = SnappyCompression
2163+
c = MinlzCompression
21642164
}
21652165
return c
21662166
}

‎replay/testdata/replay‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ tree
1111
614 000007.sst
1212
0 LOCK
1313
133 MANIFEST-000001
14-
1524 OPTIONS-000003
14+
1523 OPTIONS-000003
1515
0 marker.format-version.000001.013
1616
0 marker.manifest.000001.MANIFEST-000001
1717
simple/
@@ -21,7 +21,7 @@ tree
2121
25 000004.log
2222
586 000005.sst
2323
85 MANIFEST-000001
24-
1524 OPTIONS-000003
24+
1523 OPTIONS-000003
2525
0 marker.format-version.000001.013
2626
0 marker.manifest.000001.MANIFEST-000001
2727

@@ -79,7 +79,7 @@ cat build/OPTIONS-000003
7979
block_restart_interval=16
8080
block_size=4096
8181
block_size_threshold=90
82-
compression=Snappy
82+
compression=Minlz
8383
filter_policy=none
8484
filter_type=table
8585
index_block_size=4096

‎replay/testdata/replay_paced‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ tree
1414
0 LOCK
1515
133 MANIFEST-000001
1616
205 MANIFEST-000010
17-
1524 OPTIONS-000003
17+
1523 OPTIONS-000003
1818
0 marker.format-version.000001.013
1919
0 marker.manifest.000002.MANIFEST-000010
2020
high_read_amp/
@@ -26,7 +26,7 @@ tree
2626
39 000008.log
2727
560 000009.sst
2828
157 MANIFEST-000010
29-
1524 OPTIONS-000003
29+
1523 OPTIONS-000003
3030
0 marker.format-version.000001.013
3131
0 marker.manifest.000001.MANIFEST-000010
3232

‎sstable/options.go‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -331,6 +331,9 @@ func (o WriterOptions) ensureDefaults() WriterOptions {
331331
o.Comparer = base.DefaultComparer
332332
}
333333
if o.Compression <= block.DefaultCompression || o.Compression >= block.NCompression {
334+
o.Compression = block.MinlzCompression
335+
}
336+
if o.Compression == block.MinlzCompression && o.TableFormat <= TableFormatPebblev5 {
334337
o.Compression = block.SnappyCompression
335338
}
336339
if o.IndexBlockSize <= 0 {

‎sstable/testdata/copy_span‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -173,7 +173,7 @@ d#0,SET: foobar
173173

174174
copy-span test3 test4 b.SET.10 cc.SET.0
175175
----
176-
copied 928 bytes
176+
copied 921 bytes
177177

178178
iter test4
179179
----
@@ -183,7 +183,7 @@ d#0,SET: foobar
183183

184184
copy-span test3 test5 a.SET.10 bb.SET.0
185185
----
186-
copied 936 bytes
186+
copied 918 bytes
187187

188188
iter test5
189189
----
@@ -213,7 +213,7 @@ d#0,SET: foobar
213213

214214
copy-span test32 test33 b.SET.10 cc.SET.0
215215
----
216-
copied 928 bytes
216+
copied 921 bytes
217217

218218
iter test33
219219
----
@@ -223,7 +223,7 @@ d#0,SET: foobar
223223

224224
copy-span test32 test34 a.SET.10 bb.SET.0
225225
----
226-
copied 936 bytes
226+
copied 918 bytes
227227

228228
iter test34
229229
----

‎sstable/testdata/rewriter_v6‎

Lines changed: 15 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -45,9 +45,9 @@ sstable
4545
├── index offset: 363 length: 37
4646
├── top-index offset: 405 length: 52
4747
├── fullfilter.rocksdb.BuiltinBloomFilter offset: 462 length: 69
48-
├── properties offset: 536 length: 572
49-
├── meta-index offset: 1113 length: 88
50-
└── footer offset: 1206 length: 57
48+
├── properties offset: 536 length: 583
49+
├── meta-index offset: 1124 length: 88
50+
└── footer offset: 1217 length: 57
5151

5252
scan
5353
----
@@ -80,9 +80,9 @@ sstable
8080
├── index offset: 367 length: 37
8181
├── top-index offset: 409 length: 56
8282
├── fullfilter.rocksdb.BuiltinBloomFilter offset: 470 length: 69
83-
├── properties offset: 544 length: 572
84-
├── meta-index offset: 1121 length: 88
85-
└── footer offset: 1214 length: 57
83+
├── properties offset: 544 length: 583
84+
├── meta-index offset: 1132 length: 88
85+
└── footer offset: 1225 length: 57
8686

8787
scan
8888
----
@@ -115,9 +115,9 @@ sstable
115115
├── index offset: 367 length: 37
116116
├── top-index offset: 409 length: 56
117117
├── fullfilter.rocksdb.BuiltinBloomFilter offset: 470 length: 69
118-
├── properties offset: 544 length: 572
119-
├── meta-index offset: 1121 length: 88
120-
└── footer offset: 1214 length: 57
118+
├── properties offset: 544 length: 583
119+
├── meta-index offset: 1132 length: 88
120+
└── footer offset: 1225 length: 57
121121

122122
scan
123123
----
@@ -150,9 +150,9 @@ sstable
150150
├── index offset: 367 length: 37
151151
├── top-index offset: 409 length: 56
152152
├── fullfilter.rocksdb.BuiltinBloomFilter offset: 470 length: 69
153-
├── properties offset: 544 length: 572
154-
├── meta-index offset: 1121 length: 88
155-
└── footer offset: 1214 length: 57
153+
├── properties offset: 544 length: 583
154+
├── meta-index offset: 1132 length: 88
155+
└── footer offset: 1225 length: 57
156156

157157
scan
158158
----
@@ -186,9 +186,9 @@ sstable
186186
├── index offset: 363 length: 37
187187
├── top-index offset: 405 length: 52
188188
├── fullfilter.rocksdb.BuiltinBloomFilter offset: 462 length: 69
189-
├── properties offset: 536 length: 572
190-
├── meta-index offset: 1113 length: 88
191-
└── footer offset: 1206 length: 57
189+
├── properties offset: 536 length: 583
190+
├── meta-index offset: 1124 length: 88
191+
└── footer offset: 1217 length: 57
192192

193193
scan
194194
----

‎sstable/testdata/writer_blob_value_handles‎

Lines changed: 22 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -111,37 +111,37 @@ sstable
111111
│ ├── 00000 block:0/154
112112
│ │
113113
│ └── trailer [compression=none checksum=0xda1c8436]
114-
├── properties offset: 200 length: 562
114+
├── properties offset: 200 length: 559
115115
│ ├── 00000 obsolete-key (13)
116116
│ ├── 00013 pebble.colblk.schema (65)
117117
│ ├── 00078 pebble.num.values.in.blob-files (32)
118118
│ ├── 00110 pebble.raw.point-tombstone.key.size (36)
119119
│ ├── 00146 rocksdb.block.based.table.index.type (40)
120120
│ ├── 00186 rocksdb.comparator (42)
121-
│ ├── 00228 rocksdb.compression (25)
122-
│ ├── 00253 rocksdb.compression_options (122)
123-
│ ├── 00375 rocksdb.data.size (19)
124-
│ ├── 00394 rocksdb.deleted.keys (21)
125-
│ ├── 00415 rocksdb.filter.size (20)
126-
│ ├── 00435 rocksdb.index.size (19)
127-
│ ├── 00454 rocksdb.merge.operands (23)
128-
│ ├── 00477 rocksdb.merge.operator (40)
129-
│ ├── 00517 rocksdb.num.data.blocks (24)
130-
│ ├── 00541 rocksdb.num.entries (20)
131-
│ ├── 00561 rocksdb.num.range-deletions (28)
132-
│ ├── 00589 rocksdb.property.collectors (41)
133-
│ ├── 00630 rocksdb.raw.key.size (21)
134-
│ ├── 00651 rocksdb.raw.value.size (24)
135-
│ └── trailer [compression=snappy checksum=0xf0c5768d]
136-
├── meta-index offset: 767 length: 46
137-
│ ├── 0000 rocksdb.properties block:200/562
121+
│ ├── 00228 rocksdb.compression (24)
122+
│ ├── 00252 rocksdb.compression_options (122)
123+
│ ├── 00374 rocksdb.data.size (19)
124+
│ ├── 00393 rocksdb.deleted.keys (21)
125+
│ ├── 00414 rocksdb.filter.size (20)
126+
│ ├── 00434 rocksdb.index.size (19)
127+
│ ├── 00453 rocksdb.merge.operands (23)
128+
│ ├── 00476 rocksdb.merge.operator (40)
129+
│ ├── 00516 rocksdb.num.data.blocks (24)
130+
│ ├── 00540 rocksdb.num.entries (20)
131+
│ ├── 00560 rocksdb.num.range-deletions (28)
132+
│ ├── 00588 rocksdb.property.collectors (41)
133+
│ ├── 00629 rocksdb.raw.key.size (21)
134+
│ ├── 00650 rocksdb.raw.value.size (24)
135+
│ └── trailer [compression=minlz checksum=0xefe5c665]
136+
├── meta-index offset: 764 length: 46
137+
│ ├── 0000 rocksdb.properties block:200/559
138138
│ │
139-
│ └── trailer [compression=none checksum=0xaa86f2de]
140-
└── footer offset: 818 length: 57
139+
│ └── trailer [compression=none checksum=0x3da86560]
140+
└── footer offset: 815 length: 57
141141
├── 000 checksum type: crc32c
142-
├── 001 meta: offset=767, length=46
142+
├── 001 meta: offset=764, length=46
143143
├── 004 index: offset=159, length=36
144-
├── 041 footer checksum: 0xe7d8ef2e
144+
├── 041 footer checksum: 0x35ccc1c0
145145
├── 045 version: 6
146146
└── 049 magic number: 0xf09faab3f09faab3
147147

‎sstable/testdata/writer_v6‎

Lines changed: 25 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -96,7 +96,7 @@ rocksdb.raw.value.size: 0
9696
rocksdb.deleted.keys: 9
9797
rocksdb.num.range-deletions: 9
9898
rocksdb.num.data.blocks: 0
99-
rocksdb.compression: Snappy
99+
rocksdb.compression: Minlz
100100
rocksdb.compression_options: window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0;
101101
rocksdb.comparator: pebble.internal.testkeys
102102
rocksdb.data.size: 0
@@ -228,32 +228,32 @@ seqnums: [1-1]
228228
layout
229229
----
230230
sstable
231-
├── data offset: 0 length: 74
232-
├── data offset: 79 length: 74
233-
├── data offset: 158 length: 74
234-
├── index offset: 237 length: 36
235-
├── index offset: 278 length: 37
236-
├── index offset: 320 length: 37
237-
├── top-index offset: 362 length: 48
238-
├── properties offset: 415 length: 554
239-
├── meta-index offset: 974 length: 46
240-
└── footer offset: 1025 length: 57
231+
├── data offset: 0 length: 72
232+
├── data offset: 77 length: 72
233+
├── data offset: 154 length: 72
234+
├── index offset: 231 length: 36
235+
├── index offset: 272 length: 37
236+
├── index offset: 314 length: 37
237+
├── top-index offset: 356 length: 48
238+
├── properties offset: 409 length: 530
239+
├── meta-index offset: 944 length: 46
240+
└── footer offset: 995 length: 57
241241

242242
# Exercise the non-Reader layout-decoding codepath.
243243

244244
decode-layout
245245
----
246246
sstable
247-
├── data offset: 0 length: 74
248-
├── data offset: 79 length: 74
249-
├── data offset: 158 length: 74
250-
├── index offset: 237 length: 36
251-
├── index offset: 278 length: 37
252-
├── index offset: 320 length: 37
253-
├── top-index offset: 362 length: 48
254-
├── properties offset: 415 length: 554
255-
├── meta-index offset: 974 length: 46
256-
└── footer offset: 1025 length: 57
247+
├── data offset: 0 length: 72
248+
├── data offset: 77 length: 72
249+
├── data offset: 154 length: 72
250+
├── index offset: 231 length: 36
251+
├── index offset: 272 length: 37
252+
├── index offset: 314 length: 37
253+
├── top-index offset: 356 length: 48
254+
├── properties offset: 409 length: 530
255+
├── meta-index offset: 944 length: 46
256+
└── footer offset: 995 length: 57
257257

258258
scan
259259
----
@@ -298,9 +298,9 @@ layout
298298
sstable
299299
├── index offset: 0 length: 28
300300
├── range-key offset: 33 length: 84
301-
├── properties offset: 122 length: 577
302-
├── meta-index offset: 704 length: 65
303-
└── footer offset: 774 length: 57
301+
├── properties offset: 122 length: 574
302+
├── meta-index offset: 701 length: 65
303+
└── footer offset: 771 length: 57
304304

305305
props
306306
----
@@ -312,7 +312,7 @@ rocksdb.num.range-deletions: 0
312312
pebble.num.range-key-dels: 0
313313
pebble.num.range-key-sets: 3
314314
rocksdb.num.data.blocks: 0
315-
rocksdb.compression: Snappy
315+
rocksdb.compression: Minlz
316316
rocksdb.compression_options: window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0;
317317
rocksdb.comparator: pebble.internal.testkeys
318318
rocksdb.data.size: 0

0 commit comments

Comments
 (0)