@@ -66,14 +66,6 @@ proc toRocksDb*(
6666 tableOpts.dataBlockIndexType = DataBlockIndexType .binarySearchAndHash
6767 tableOpts.dataBlockHashRatio = 0.75
6868
69- # A smaller block size reduces read amplification at the expense of larger
70- # indices - there should also be some alignment with on-disk blocks so that we
71- # typically perform reads consistent with the disk block size. It's hard
72- # to pick a reasonable number here, but assuming compression to about 2/3,
73- # we should use about ~6k blocks to fit in an ssd block - more benchmarks
74- # needed!
75- tableOpts.blockSize = 6000
76-
7769 let cfOpts = defaultColFamilyOptions (autoClose = true )
7870
7971 cfOpts.blockBasedTableFactory = tableOpts
@@ -100,24 +92,10 @@ proc toRocksDb*(
10092 # Compared to LZ4 that was tested earlier, the default ZSTD config results
10193 # in 10% less space and similar or slightly better performance in some
10294 # simple tests around mainnet block 14M.
103- #
104- # sst_dump --file=005420.sst --command=recompress --set_block_size=6000 --compression_types=kZSTD --compression_level_from=-3 --compression_level_to=3 --compression_max_dict_bytes=16484 --compression_zstd_max_train_bytes=1638400
105- # Compression level: -3 Size: 2497674402 Blocks: 522975 Time Taken: 29957380
106- # Compression level: -1 Size: 2271282060 Blocks: 522975 Time Taken: 34565174
107- # Compression level: 1 Size: 2260978713 Blocks: 522975 Time Taken: 38725150
108- # Compression level: 3 Size: 2241970102 Blocks: 522975 Time Taken: 53415641
109-
110- # Based on the above, -1 and -1 would offer similarly reasonable performance
111- # while -3 and 3 each feel like the cost/benefit is worse, in either direction
112-
95+ # TODO evaluate zstd dictionary compression
96+ # https://github.com/facebook/rocksdb/wiki/Dictionary-Compression
11397 cfOpts.bottommostCompression = Compression .zstdCompression
11498
115- # 16kb dictionary size per rocksdb recommendation:
116- # https://rocksdb.org/blog/2021/05/31/dictionary-compression.html
117- cfOpts.setBottommostCompressionOptions (level = 1 , maxDictBytes = 16384 )
118- cfOpts.bottommostCompressionOptionsZstdMaxTrainBytes = 16384 * 100
119- cfOpts.bottommostCompressionOptionsUseZstdDictTrainer = false
120-
12199 # TODO In the AriVtx table, we don't do lookups that are expected to result
122100 # in misses thus we could avoid the filter cost - this does not apply to
123101 # other tables since their API admit queries that might result in
0 commit comments