Skip to content

sstable: set minlz compression to default for v6+ #4553

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 1 commit into
base: master
Choose a base branch
from
+365 −360
Draft

sstable: set minlz compression to default for v6+ #4553

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion ingest_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -191,7 +191,7 @@ func TestIngestLoadRand(t *testing.T) {
},
path: paths[i],
}
expected[i].tableMetadata.Stats.CompressionType = block.SnappyCompression
expected[i].tableMetadata.Stats.CompressionType = block.MinlzCompression
expected[i].StatsMarkValid()

func() {
Expand Down
8 changes: 5 additions & 3 deletions metamorphic/options.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -786,14 +786,16 @@ func RandomOptions(
lopts.FilterPolicy = newTestingFilterPolicy(1 << rng.IntN(5))
}

// We use either no compression, snappy compression or zstd compression.
switch rng.IntN(3) {
// We use either no compression, snappy compression, zstd compression, or minlz compression.
switch rng.IntN(4) {
case 0:
lopts.Compression = func() block.Compression { return pebble.NoCompression }
case 1:
lopts.Compression = func() block.Compression { return pebble.ZstdCompression }
default:
case 2:
lopts.Compression = func() block.Compression { return pebble.SnappyCompression }
default:
lopts.Compression = func() block.Compression { return pebble.MinlzCompression }
}
opts.Levels = []pebble.LevelOptions{lopts}

Expand Down
2 changes: 1 addition & 1 deletion options.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2160,7 +2160,7 @@ func (o *Options) MakeBlobWriterOptions(level int) blob.FileWriterOptions {

func resolveDefaultCompression(c Compression) Compression {
if c <= DefaultCompression || c >= block.NCompression {
c = SnappyCompression
c = MinlzCompression
}
return c
}
Expand Down
2 changes: 1 addition & 1 deletion options_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -127,7 +127,7 @@ func TestDefaultOptionsString(t *testing.T) {
block_restart_interval=16
block_size=4096
block_size_threshold=90
compression=Snappy
compression=Minlz
filter_policy=none
filter_type=table
index_block_size=4096
Expand Down
6 changes: 3 additions & 3 deletions replay/testdata/replay
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ tree
614 000007.sst
0 LOCK
133 MANIFEST-000001
1524 OPTIONS-000003
1523 OPTIONS-000003
0 marker.format-version.000001.013
0 marker.manifest.000001.MANIFEST-000001
simple/
Expand All @@ -21,7 +21,7 @@ tree
25 000004.log
586 000005.sst
85 MANIFEST-000001
1524 OPTIONS-000003
1523 OPTIONS-000003
0 marker.format-version.000001.013
0 marker.manifest.000001.MANIFEST-000001

Expand Down Expand Up @@ -79,7 +79,7 @@ cat build/OPTIONS-000003
block_restart_interval=16
block_size=4096
block_size_threshold=90
compression=Snappy
compression=Minlz
filter_policy=none
filter_type=table
index_block_size=4096
Expand Down
4 changes: 2 additions & 2 deletions replay/testdata/replay_paced
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ tree
0 LOCK
133 MANIFEST-000001
205 MANIFEST-000010
1524 OPTIONS-000003
1523 OPTIONS-000003
0 marker.format-version.000001.013
0 marker.manifest.000002.MANIFEST-000010
high_read_amp/
Expand All @@ -26,7 +26,7 @@ tree
39 000008.log
560 000009.sst
157 MANIFEST-000010
1524 OPTIONS-000003
1523 OPTIONS-000003
0 marker.format-version.000001.013
0 marker.manifest.000001.MANIFEST-000010

Expand Down
3 changes: 3 additions & 0 deletions sstable/options.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -331,6 +331,9 @@ func (o WriterOptions) ensureDefaults() WriterOptions {
o.Comparer = base.DefaultComparer
}
if o.Compression <= block.DefaultCompression || o.Compression >= block.NCompression {
o.Compression = block.MinlzCompression
}
if o.Compression == block.MinlzCompression && o.TableFormat <= TableFormatPebblev5 {
o.Compression = block.SnappyCompression
}
if o.IndexBlockSize <= 0 {
Expand Down
8 changes: 4 additions & 4 deletions sstable/testdata/copy_span
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -173,7 +173,7 @@ d#0,SET: foobar

copy-span test3 test4 b.SET.10 cc.SET.0
----
copied 928 bytes
copied 921 bytes

iter test4
----
Expand All @@ -183,7 +183,7 @@ d#0,SET: foobar

copy-span test3 test5 a.SET.10 bb.SET.0
----
copied 936 bytes
copied 918 bytes

iter test5
----
Expand Down Expand Up @@ -213,7 +213,7 @@ d#0,SET: foobar

copy-span test32 test33 b.SET.10 cc.SET.0
----
copied 928 bytes
copied 921 bytes

iter test33
----
Expand All @@ -223,7 +223,7 @@ d#0,SET: foobar

copy-span test32 test34 a.SET.10 bb.SET.0
----
copied 936 bytes
copied 918 bytes

iter test34
----
Expand Down
30 changes: 15 additions & 15 deletions sstable/testdata/rewriter_v6
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,9 +45,9 @@ sstable
├── index offset: 363 length: 37
├── top-index offset: 405 length: 52
├── fullfilter.rocksdb.BuiltinBloomFilter offset: 462 length: 69
├── properties offset: 536 length: 572
├── meta-index offset: 1113 length: 88
└── footer offset: 1206 length: 57
├── properties offset: 536 length: 583
├── meta-index offset: 1124 length: 88
└── footer offset: 1217 length: 57

scan
----
Expand Down Expand Up @@ -80,9 +80,9 @@ sstable
├── index offset: 367 length: 37
├── top-index offset: 409 length: 56
├── fullfilter.rocksdb.BuiltinBloomFilter offset: 470 length: 69
├── properties offset: 544 length: 572
├── meta-index offset: 1121 length: 88
└── footer offset: 1214 length: 57
├── properties offset: 544 length: 583
├── meta-index offset: 1132 length: 88
└── footer offset: 1225 length: 57

scan
----
Expand Down Expand Up @@ -115,9 +115,9 @@ sstable
├── index offset: 367 length: 37
├── top-index offset: 409 length: 56
├── fullfilter.rocksdb.BuiltinBloomFilter offset: 470 length: 69
├── properties offset: 544 length: 572
├── meta-index offset: 1121 length: 88
└── footer offset: 1214 length: 57
├── properties offset: 544 length: 583
├── meta-index offset: 1132 length: 88
└── footer offset: 1225 length: 57

scan
----
Expand Down Expand Up @@ -150,9 +150,9 @@ sstable
├── index offset: 367 length: 37
├── top-index offset: 409 length: 56
├── fullfilter.rocksdb.BuiltinBloomFilter offset: 470 length: 69
├── properties offset: 544 length: 572
├── meta-index offset: 1121 length: 88
└── footer offset: 1214 length: 57
├── properties offset: 544 length: 583
├── meta-index offset: 1132 length: 88
└── footer offset: 1225 length: 57

scan
----
Expand Down Expand Up @@ -186,9 +186,9 @@ sstable
├── index offset: 363 length: 37
├── top-index offset: 405 length: 52
├── fullfilter.rocksdb.BuiltinBloomFilter offset: 462 length: 69
├── properties offset: 536 length: 572
├── meta-index offset: 1113 length: 88
└── footer offset: 1206 length: 57
├── properties offset: 536 length: 583
├── meta-index offset: 1124 length: 88
└── footer offset: 1217 length: 57

scan
----
Expand Down
44 changes: 22 additions & 22 deletions sstable/testdata/writer_blob_value_handles
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -111,37 +111,37 @@ sstable
│ ├── 00000 block:0/154
│ │
│ └── trailer [compression=none checksum=0xda1c8436]
├── properties offset: 200 length: 562
├── properties offset: 200 length: 559
│ ├── 00000 obsolete-key (13)
│ ├── 00013 pebble.colblk.schema (65)
│ ├── 00078 pebble.num.values.in.blob-files (32)
│ ├── 00110 pebble.raw.point-tombstone.key.size (36)
│ ├── 00146 rocksdb.block.based.table.index.type (40)
│ ├── 00186 rocksdb.comparator (42)
│ ├── 00228 rocksdb.compression (25)
│ ├── 00253 rocksdb.compression_options (122)
│ ├── 00375 rocksdb.data.size (19)
│ ├── 00394 rocksdb.deleted.keys (21)
│ ├── 00415 rocksdb.filter.size (20)
│ ├── 00435 rocksdb.index.size (19)
│ ├── 00454 rocksdb.merge.operands (23)
│ ├── 00477 rocksdb.merge.operator (40)
│ ├── 00517 rocksdb.num.data.blocks (24)
│ ├── 00541 rocksdb.num.entries (20)
│ ├── 00561 rocksdb.num.range-deletions (28)
│ ├── 00589 rocksdb.property.collectors (41)
│ ├── 00630 rocksdb.raw.key.size (21)
│ ├── 00651 rocksdb.raw.value.size (24)
│ └── trailer [compression=snappy checksum=0xf0c5768d]
├── meta-index offset: 767 length: 46
│ ├── 0000 rocksdb.properties block:200/562
│ ├── 00228 rocksdb.compression (24)
│ ├── 00252 rocksdb.compression_options (122)
│ ├── 00374 rocksdb.data.size (19)
│ ├── 00393 rocksdb.deleted.keys (21)
│ ├── 00414 rocksdb.filter.size (20)
│ ├── 00434 rocksdb.index.size (19)
│ ├── 00453 rocksdb.merge.operands (23)
│ ├── 00476 rocksdb.merge.operator (40)
│ ├── 00516 rocksdb.num.data.blocks (24)
│ ├── 00540 rocksdb.num.entries (20)
│ ├── 00560 rocksdb.num.range-deletions (28)
│ ├── 00588 rocksdb.property.collectors (41)
│ ├── 00629 rocksdb.raw.key.size (21)
│ ├── 00650 rocksdb.raw.value.size (24)
│ └── trailer [compression=minlz checksum=0xefe5c665]
├── meta-index offset: 764 length: 46
│ ├── 0000 rocksdb.properties block:200/559
│ │
│ └── trailer [compression=none checksum=0xaa86f2de]
└── footer offset: 818 length: 57
│ └── trailer [compression=none checksum=0x3da86560]
└── footer offset: 815 length: 57
├── 000 checksum type: crc32c
├── 001 meta: offset=767, length=46
├── 001 meta: offset=764, length=46
├── 004 index: offset=159, length=36
├── 041 footer checksum: 0xe7d8ef2e
├── 041 footer checksum: 0x35ccc1c0
├── 045 version: 6
└── 049 magic number: 0xf09faab3f09faab3

Expand Down
50 changes: 25 additions & 25 deletions sstable/testdata/writer_v6
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,7 @@ rocksdb.raw.value.size: 0
rocksdb.deleted.keys: 9
rocksdb.num.range-deletions: 9
rocksdb.num.data.blocks: 0
rocksdb.compression: Snappy
rocksdb.compression: Minlz
rocksdb.compression_options: window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0;
rocksdb.comparator: pebble.internal.testkeys
rocksdb.data.size: 0
Expand Down Expand Up @@ -228,32 +228,32 @@ seqnums: [1-1]
layout
----
sstable
├── data offset: 0 length: 74
├── data offset: 79 length: 74
├── data offset: 158 length: 74
├── index offset: 237 length: 36
├── index offset: 278 length: 37
├── index offset: 320 length: 37
├── top-index offset: 362 length: 48
├── properties offset: 415 length: 554
├── meta-index offset: 974 length: 46
└── footer offset: 1025 length: 57
├── data offset: 0 length: 72
├── data offset: 77 length: 72
├── data offset: 154 length: 72
├── index offset: 231 length: 36
├── index offset: 272 length: 37
├── index offset: 314 length: 37
├── top-index offset: 356 length: 48
├── properties offset: 409 length: 530
├── meta-index offset: 944 length: 46
└── footer offset: 995 length: 57

# Exercise the non-Reader layout-decoding codepath.

decode-layout
----
sstable
├── data offset: 0 length: 74
├── data offset: 79 length: 74
├── data offset: 158 length: 74
├── index offset: 237 length: 36
├── index offset: 278 length: 37
├── index offset: 320 length: 37
├── top-index offset: 362 length: 48
├── properties offset: 415 length: 554
├── meta-index offset: 974 length: 46
└── footer offset: 1025 length: 57
├── data offset: 0 length: 72
├── data offset: 77 length: 72
├── data offset: 154 length: 72
├── index offset: 231 length: 36
├── index offset: 272 length: 37
├── index offset: 314 length: 37
├── top-index offset: 356 length: 48
├── properties offset: 409 length: 530
├── meta-index offset: 944 length: 46
└── footer offset: 995 length: 57

scan
----
Expand Down Expand Up @@ -298,9 +298,9 @@ layout
sstable
├── index offset: 0 length: 28
├── range-key offset: 33 length: 84
├── properties offset: 122 length: 577
├── meta-index offset: 704 length: 65
└── footer offset: 774 length: 57
├── properties offset: 122 length: 574
├── meta-index offset: 701 length: 65
└── footer offset: 771 length: 57

props
----
Expand All @@ -312,7 +312,7 @@ rocksdb.num.range-deletions: 0
pebble.num.range-key-dels: 0
pebble.num.range-key-sets: 3
rocksdb.num.data.blocks: 0
rocksdb.compression: Snappy
rocksdb.compression: Minlz
rocksdb.compression_options: window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0;
rocksdb.comparator: pebble.internal.testkeys
rocksdb.data.size: 0
Expand Down
Loading
Loading