diff --git a/config/src/config/storage_config.rs b/config/src/config/storage_config.rs index 72b272dab0c9e..f8b0641ab0d6b 100644 --- a/config/src/config/storage_config.rs +++ b/config/src/config/storage_config.rs @@ -143,12 +143,12 @@ pub struct RocksdbConfig { impl RocksdbConfig { /// Default block cache size is 1GB, - const DEFAULT_BLOCK_CACHE_SIZE: u64 = 1 << 30; + const DEFAULT_BLOCK_CACHE_SIZE: u64 = 2 << 30; /// Default block size is 4KB, const DEFAULT_BLOCK_SIZE: u64 = 4 * (1 << 10); /// Default block cache size for state kv db is 16GB, because the number of different keys /// being read is usually large. - const DEFAULT_STATE_KV_BLOCK_CACHE_SIZE: u64 = 16 * (1 << 30); + const DEFAULT_STATE_KV_BLOCK_CACHE_SIZE: u64 = 30 * (1 << 30); } impl Default for RocksdbConfig { diff --git a/storage/aptosdb/src/db/mod.rs b/storage/aptosdb/src/db/mod.rs index 66eee089fae82..c555e02525225 100644 --- a/storage/aptosdb/src/db/mod.rs +++ b/storage/aptosdb/src/db/mod.rs @@ -11,7 +11,7 @@ use crate::{ use aptos_config::config::{PrunerConfig, RocksdbConfigs, StorageDirPaths}; use aptos_db_indexer::{db_indexer::InternalIndexerDB, Indexer}; use aptos_logger::prelude::*; -use aptos_schemadb::batch::SchemaBatch; +use aptos_schemadb::{batch::SchemaBatch, Cache}; use aptos_storage_interface::{db_ensure as ensure, AptosDbError, Result}; use aptos_types::{ledger_info::LedgerInfoWithSignatures, transaction::Version}; use std::{path::Path, sync::Arc, time::Instant}; @@ -107,18 +107,26 @@ impl AptosDB { readonly: bool, max_num_nodes_per_lru_cache_shard: usize, ) -> Result<(LedgerDb, StateMerkleDb, StateKvDb)> { - let ledger_db = LedgerDb::new(db_paths.ledger_db_root_path(), rocksdb_configs, readonly)?; + let block_cache = Cache::new_hyper_clock_cache(32 * (1 << 30), 0); + let ledger_db = LedgerDb::new( + db_paths.ledger_db_root_path(), + rocksdb_configs, + readonly, + Some(&block_cache), + )?; let state_kv_db = StateKvDb::new( db_paths, rocksdb_configs, readonly, ledger_db.metadata_db_arc(), + Some(&block_cache), )?; let state_merkle_db = StateMerkleDb::new( db_paths, rocksdb_configs, readonly, max_num_nodes_per_lru_cache_shard, + Some(&block_cache), )?; Ok((ledger_db, state_merkle_db, state_kv_db)) diff --git a/storage/aptosdb/src/db_debugger/common/mod.rs b/storage/aptosdb/src/db_debugger/common/mod.rs index dc9a19e390e0c..6df792c601d6b 100644 --- a/storage/aptosdb/src/db_debugger/common/mod.rs +++ b/storage/aptosdb/src/db_debugger/common/mod.rs @@ -34,6 +34,7 @@ impl DbDir { }, false, 0, + None, ) } @@ -47,6 +48,7 @@ impl DbDir { }, true, leger_db.metadata_db_arc(), + None, ) } @@ -58,6 +60,7 @@ impl DbDir { ..Default::default() }, true, + None, ) } } diff --git a/storage/aptosdb/src/ledger_db/mod.rs b/storage/aptosdb/src/ledger_db/mod.rs index c2e3595505d70..bf37fe4d21000 100644 --- a/storage/aptosdb/src/ledger_db/mod.rs +++ b/storage/aptosdb/src/ledger_db/mod.rs @@ -121,13 +121,9 @@ impl LedgerDb { db_root_path: P, rocksdb_configs: RocksdbConfigs, readonly: bool, + block_cache: Option<&Cache>, ) -> Result { let sharding = rocksdb_configs.enable_storage_sharding; - let block_cache = Cache::new_hyper_clock_cache( - rocksdb_configs.ledger_db_config.block_cache_size as usize, - /* estimated_entry_charge = */ 0, - ); - let ledger_metadata_db_path = Self::metadata_db_path(db_root_path.as_ref(), sharding); let ledger_metadata_db = Arc::new(Self::open_rocksdb( ledger_metadata_db_path.clone(), @@ -137,7 +133,7 @@ impl LedgerDb { LEDGER_DB_NAME }, &rocksdb_configs.ledger_db_config, - &block_cache, + block_cache, readonly, )?); @@ -187,7 +183,7 @@ impl LedgerDb { ledger_db_folder.join(EVENT_DB_NAME), EVENT_DB_NAME, &rocksdb_configs.ledger_db_config, - &block_cache, + block_cache, readonly, ) .unwrap(), @@ -203,7 +199,7 @@ impl LedgerDb { ledger_db_folder.join(PERSISTED_AUXILIARY_INFO_DB_NAME), PERSISTED_AUXILIARY_INFO_DB_NAME, &rocksdb_configs.ledger_db_config, - &block_cache, + block_cache, readonly, ) .unwrap(), @@ -215,7 +211,7 @@ impl LedgerDb { ledger_db_folder.join(TRANSACTION_ACCUMULATOR_DB_NAME), TRANSACTION_ACCUMULATOR_DB_NAME, &rocksdb_configs.ledger_db_config, - &block_cache, + block_cache, readonly, ) .unwrap(), @@ -227,7 +223,7 @@ impl LedgerDb { ledger_db_folder.join(TRANSACTION_AUXILIARY_DATA_DB_NAME), TRANSACTION_AUXILIARY_DATA_DB_NAME, &rocksdb_configs.ledger_db_config, - &block_cache, + block_cache, readonly, ) .unwrap(), @@ -239,7 +235,7 @@ impl LedgerDb { ledger_db_folder.join(TRANSACTION_DB_NAME), TRANSACTION_DB_NAME, &rocksdb_configs.ledger_db_config, - &block_cache, + block_cache, readonly, ) .unwrap(), @@ -251,7 +247,7 @@ impl LedgerDb { ledger_db_folder.join(TRANSACTION_INFO_DB_NAME), TRANSACTION_INFO_DB_NAME, &rocksdb_configs.ledger_db_config, - &block_cache, + block_cache, readonly, ) .unwrap(), @@ -263,7 +259,7 @@ impl LedgerDb { ledger_db_folder.join(WRITE_SET_DB_NAME), WRITE_SET_DB_NAME, &rocksdb_configs.ledger_db_config, - &block_cache, + block_cache, readonly, ) .unwrap(), @@ -310,7 +306,12 @@ impl LedgerDb { enable_storage_sharding: sharding, ..Default::default() }; - let ledger_db = Self::new(db_root_path, rocksdb_configs, /*readonly=*/ false)?; + let ledger_db = Self::new( + db_root_path, + rocksdb_configs, + /*readonly=*/ false, + None, + )?; let cp_ledger_db_folder = cp_root_path.as_ref().join(LEDGER_DB_FOLDER_NAME); info!( @@ -443,7 +444,7 @@ impl LedgerDb { path: PathBuf, name: &str, db_config: &RocksdbConfig, - block_cache: &Cache, + block_cache: Option<&Cache>, readonly: bool, ) -> Result { let db = if readonly { @@ -484,10 +485,9 @@ impl LedgerDb { fn gen_cfds_by_name( db_config: &RocksdbConfig, - block_cache: &Cache, + cache: Option<&Cache>, name: &str, ) -> Vec { - let cache = Some(block_cache); match name { LEDGER_DB_NAME => gen_ledger_cfds(db_config, cache), LEDGER_METADATA_DB_NAME => gen_ledger_metadata_cfds(db_config, cache), diff --git a/storage/aptosdb/src/state_kv_db.rs b/storage/aptosdb/src/state_kv_db.rs index 6d51b20ed5ecd..b36fc9db4e040 100644 --- a/storage/aptosdb/src/state_kv_db.rs +++ b/storage/aptosdb/src/state_kv_db.rs @@ -56,6 +56,7 @@ impl StateKvDb { rocksdb_configs: RocksdbConfigs, readonly: bool, ledger_db: Arc, + block_cache: Option<&Cache>, ) -> Result { let sharding = rocksdb_configs.enable_storage_sharding; if !sharding { @@ -68,14 +69,10 @@ impl StateKvDb { }); } - let block_cache = Cache::new_hyper_clock_cache( - rocksdb_configs.state_kv_db_config.block_cache_size as usize, - /* estimated_entry_charge = */ 0, - ); Self::open_sharded( db_paths, rocksdb_configs.state_kv_db_config, - Some(&block_cache), + block_cache, readonly, ) } diff --git a/storage/aptosdb/src/state_merkle_db.rs b/storage/aptosdb/src/state_merkle_db.rs index 59a2dd712d399..533a06e314fcf 100644 --- a/storage/aptosdb/src/state_merkle_db.rs +++ b/storage/aptosdb/src/state_merkle_db.rs @@ -75,6 +75,7 @@ impl StateMerkleDb { // TODO(grao): Currently when this value is set to 0 we disable both caches. This is // hacky, need to revisit. max_nodes_per_lru_cache_shard: usize, + block_cache: Option<&Cache>, ) -> Result { let sharding = rocksdb_configs.enable_storage_sharding; let state_merkle_db_config = rocksdb_configs.state_merkle_db_config; @@ -85,10 +86,6 @@ impl StateMerkleDb { version_caches.insert(Some(i), VersionedNodeCache::new()); } let lru_cache = NonZeroUsize::new(max_nodes_per_lru_cache_shard).map(LruNodeCache::new); - let block_cache = Cache::new_hyper_clock_cache( - state_merkle_db_config.block_cache_size as usize, - /* estimated_entry_charge = */ 0, - ); if !sharding { info!("Sharded state merkle DB is not enabled!"); @@ -97,7 +94,7 @@ impl StateMerkleDb { state_merkle_db_path, STATE_MERKLE_DB_NAME, &state_merkle_db_config, - &block_cache, + block_cache, readonly, )?); return Ok(Self { @@ -112,7 +109,7 @@ impl StateMerkleDb { Self::open( db_paths, state_merkle_db_config, - &block_cache, + block_cache, readonly, version_caches, lru_cache, @@ -179,6 +176,7 @@ impl StateMerkleDb { rocksdb_configs, /*readonly=*/ false, /*max_nodes_per_lru_cache_shard=*/ 0, + None, )?; let cp_state_merkle_db_path = cp_root_path.as_ref().join(STATE_MERKLE_DB_FOLDER_NAME); @@ -559,7 +557,7 @@ impl StateMerkleDb { fn open( db_paths: &StorageDirPaths, state_merkle_db_config: RocksdbConfig, - block_cache: &Cache, + block_cache: Option<&Cache>, readonly: bool, version_caches: HashMap, VersionedNodeCache>, lru_cache: Option, @@ -628,7 +626,7 @@ impl StateMerkleDb { db_root_path: P, shard_id: usize, state_merkle_db_config: &RocksdbConfig, - block_cache: &Cache, + block_cache: Option<&Cache>, readonly: bool, ) -> Result { let db_name = format!("state_merkle_db_shard_{}", shard_id); @@ -645,7 +643,7 @@ impl StateMerkleDb { path: PathBuf, name: &str, state_merkle_db_config: &RocksdbConfig, - block_cache: &Cache, + block_cache: Option<&Cache>, readonly: bool, ) -> Result { Ok(if readonly { @@ -653,14 +651,14 @@ impl StateMerkleDb { &gen_rocksdb_options(state_merkle_db_config, true), path, name, - gen_state_merkle_cfds(state_merkle_db_config, Some(block_cache)), + gen_state_merkle_cfds(state_merkle_db_config, block_cache), )? } else { DB::open_cf( &gen_rocksdb_options(state_merkle_db_config, false), path, name, - gen_state_merkle_cfds(state_merkle_db_config, Some(block_cache)), + gen_state_merkle_cfds(state_merkle_db_config, block_cache), )? }) } diff --git a/storage/rocksdb-options/src/lib.rs b/storage/rocksdb-options/src/lib.rs index 9c62bccc08153..9c80963472951 100644 --- a/storage/rocksdb-options/src/lib.rs +++ b/storage/rocksdb-options/src/lib.rs @@ -25,6 +25,9 @@ pub fn gen_rocksdb_options(config: &RocksdbConfig, readonly: bool) -> Options { db_opts.set_max_total_wal_size(config.max_total_wal_size); db_opts.set_max_background_jobs(config.max_background_jobs); + db_opts.set_use_direct_reads(true); + db_opts.set_use_direct_io_for_flush_and_compaction(true); + if let Some(level) = config.stats_level { db_opts.enable_statistics(); db_opts.set_statistics_level(convert_stats_level(level));