Skip to content

Commit 307884c

Browse files
committed
Revert "rocksdb: use slightly lower compression and add dictionary"
This reverts commit 8db34f1.
1 parent d5909c0 commit 307884c

File tree

2 files changed

+3
-25
lines changed

2 files changed

+3
-25
lines changed

execution_chain/db/core_db/backend/aristo_rocksdb.nim

Lines changed: 2 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -66,14 +66,6 @@ proc toRocksDb*(
6666
tableOpts.dataBlockIndexType = DataBlockIndexType.binarySearchAndHash
6767
tableOpts.dataBlockHashRatio = 0.75
6868

69-
# A smaller block size reduces read amplification at the expense of larger
70-
# indices - there should also be some alignment with on-disk blocks so that we
71-
# typically perform reads consistent with the disk block size. It's hard
72-
# to pick a reasonable number here, but assuming compression to about 2/3,
73-
# we should use about ~6k blocks to fit in an ssd block - more benchmarks
74-
# needed!
75-
tableOpts.blockSize = 6000
76-
7769
let cfOpts = defaultColFamilyOptions(autoClose = true)
7870

7971
cfOpts.blockBasedTableFactory = tableOpts
@@ -100,24 +92,10 @@ proc toRocksDb*(
10092
# Compared to LZ4 that was tested earlier, the default ZSTD config results
10193
# in 10% less space and similar or slightly better performance in some
10294
# simple tests around mainnet block 14M.
103-
#
104-
# sst_dump --file=005420.sst --command=recompress --set_block_size=6000 --compression_types=kZSTD --compression_level_from=-3 --compression_level_to=3 --compression_max_dict_bytes=16484 --compression_zstd_max_train_bytes=1638400
105-
# Compression level: -3 Size: 2497674402 Blocks: 522975 Time Taken: 29957380
106-
# Compression level: -1 Size: 2271282060 Blocks: 522975 Time Taken: 34565174
107-
# Compression level: 1 Size: 2260978713 Blocks: 522975 Time Taken: 38725150
108-
# Compression level: 3 Size: 2241970102 Blocks: 522975 Time Taken: 53415641
109-
110-
# Based on the above, -1 and -1 would offer similarly reasonable performance
111-
# while -3 and 3 each feel like the cost/benefit is worse, in either direction
112-
95+
# TODO evaluate zstd dictionary compression
96+
# https://github.com/facebook/rocksdb/wiki/Dictionary-Compression
11397
cfOpts.bottommostCompression = Compression.zstdCompression
11498

115-
# 16kb dictionary size per rocksdb recommendation:
116-
# https://rocksdb.org/blog/2021/05/31/dictionary-compression.html
117-
cfOpts.setBottommostCompressionOptions(level = 1, maxDictBytes = 16384)
118-
cfOpts.bottommostCompressionOptionsZstdMaxTrainBytes = 16384 * 100
119-
cfOpts.bottommostCompressionOptionsUseZstdDictTrainer = false
120-
12199
# TODO In the AriVtx table, we don't do lookups that are expected to result
122100
# in misses thus we could avoid the filter cost - this does not apply to
123101
# other tables since their API admit queries that might result in

0 commit comments

Comments
 (0)