Skip to content

Single block cache #17812

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 1 commit into
base: main
Choose a base branch
from
+43 −37
Draft

Single block cache #17812

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions config/src/config/storage_config.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -143,12 +143,12 @@ pub struct RocksdbConfig {

impl RocksdbConfig {
/// Default block cache size is 1GB,
const DEFAULT_BLOCK_CACHE_SIZE: u64 = 1 << 30;
const DEFAULT_BLOCK_CACHE_SIZE: u64 = 2 << 30;
/// Default block size is 4KB,
const DEFAULT_BLOCK_SIZE: u64 = 4 * (1 << 10);
/// Default block cache size for state kv db is 16GB, because the number of different keys
/// being read is usually large.
const DEFAULT_STATE_KV_BLOCK_CACHE_SIZE: u64 = 16 * (1 << 30);
const DEFAULT_STATE_KV_BLOCK_CACHE_SIZE: u64 = 30 * (1 << 30);
}

impl Default for RocksdbConfig {
Expand Down
12 changes: 10 additions & 2 deletions storage/aptosdb/src/db/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ use crate::{
use aptos_config::config::{PrunerConfig, RocksdbConfigs, StorageDirPaths};
use aptos_db_indexer::{db_indexer::InternalIndexerDB, Indexer};
use aptos_logger::prelude::*;
use aptos_schemadb::batch::SchemaBatch;
use aptos_schemadb::{batch::SchemaBatch, Cache};
use aptos_storage_interface::{db_ensure as ensure, AptosDbError, Result};
use aptos_types::{ledger_info::LedgerInfoWithSignatures, transaction::Version};
use std::{path::Path, sync::Arc, time::Instant};
Expand Down Expand Up @@ -107,18 +107,26 @@ impl AptosDB {
readonly: bool,
max_num_nodes_per_lru_cache_shard: usize,
) -> Result<(LedgerDb, StateMerkleDb, StateKvDb)> {
let ledger_db = LedgerDb::new(db_paths.ledger_db_root_path(), rocksdb_configs, readonly)?;
let block_cache = Cache::new_hyper_clock_cache(32 * (1 << 30), 0);
let ledger_db = LedgerDb::new(
db_paths.ledger_db_root_path(),
rocksdb_configs,
readonly,
Some(&block_cache),
)?;
let state_kv_db = StateKvDb::new(
db_paths,
rocksdb_configs,
readonly,
ledger_db.metadata_db_arc(),
Some(&block_cache),
)?;
let state_merkle_db = StateMerkleDb::new(
db_paths,
rocksdb_configs,
readonly,
max_num_nodes_per_lru_cache_shard,
Some(&block_cache),
)?;

Ok((ledger_db, state_merkle_db, state_kv_db))
Expand Down
3 changes: 3 additions & 0 deletions storage/aptosdb/src/db_debugger/common/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ impl DbDir {
},
false,
0,
None,
)
}

Expand All @@ -47,6 +48,7 @@ impl DbDir {
},
true,
leger_db.metadata_db_arc(),
None,
)
}

Expand All @@ -58,6 +60,7 @@ impl DbDir {
..Default::default()
},
true,
None,
)
}
}
Expand Down
34 changes: 17 additions & 17 deletions storage/aptosdb/src/ledger_db/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -121,13 +121,9 @@ impl LedgerDb {
db_root_path: P,
rocksdb_configs: RocksdbConfigs,
readonly: bool,
block_cache: Option<&Cache>,
) -> Result<Self> {
let sharding = rocksdb_configs.enable_storage_sharding;
let block_cache = Cache::new_hyper_clock_cache(
rocksdb_configs.ledger_db_config.block_cache_size as usize,
/* estimated_entry_charge = */ 0,
);

let ledger_metadata_db_path = Self::metadata_db_path(db_root_path.as_ref(), sharding);
let ledger_metadata_db = Arc::new(Self::open_rocksdb(
ledger_metadata_db_path.clone(),
Expand All @@ -137,7 +133,7 @@ impl LedgerDb {
LEDGER_DB_NAME
},
&rocksdb_configs.ledger_db_config,
&block_cache,
block_cache,
readonly,
)?);

Expand Down Expand Up @@ -187,7 +183,7 @@ impl LedgerDb {
ledger_db_folder.join(EVENT_DB_NAME),
EVENT_DB_NAME,
&rocksdb_configs.ledger_db_config,
&block_cache,
block_cache,
readonly,
)
.unwrap(),
Expand All @@ -203,7 +199,7 @@ impl LedgerDb {
ledger_db_folder.join(PERSISTED_AUXILIARY_INFO_DB_NAME),
PERSISTED_AUXILIARY_INFO_DB_NAME,
&rocksdb_configs.ledger_db_config,
&block_cache,
block_cache,
readonly,
)
.unwrap(),
Expand All @@ -215,7 +211,7 @@ impl LedgerDb {
ledger_db_folder.join(TRANSACTION_ACCUMULATOR_DB_NAME),
TRANSACTION_ACCUMULATOR_DB_NAME,
&rocksdb_configs.ledger_db_config,
&block_cache,
block_cache,
readonly,
)
.unwrap(),
Expand All @@ -227,7 +223,7 @@ impl LedgerDb {
ledger_db_folder.join(TRANSACTION_AUXILIARY_DATA_DB_NAME),
TRANSACTION_AUXILIARY_DATA_DB_NAME,
&rocksdb_configs.ledger_db_config,
&block_cache,
block_cache,
readonly,
)
.unwrap(),
Expand All @@ -239,7 +235,7 @@ impl LedgerDb {
ledger_db_folder.join(TRANSACTION_DB_NAME),
TRANSACTION_DB_NAME,
&rocksdb_configs.ledger_db_config,
&block_cache,
block_cache,
readonly,
)
.unwrap(),
Expand All @@ -251,7 +247,7 @@ impl LedgerDb {
ledger_db_folder.join(TRANSACTION_INFO_DB_NAME),
TRANSACTION_INFO_DB_NAME,
&rocksdb_configs.ledger_db_config,
&block_cache,
block_cache,
readonly,
)
.unwrap(),
Expand All @@ -263,7 +259,7 @@ impl LedgerDb {
ledger_db_folder.join(WRITE_SET_DB_NAME),
WRITE_SET_DB_NAME,
&rocksdb_configs.ledger_db_config,
&block_cache,
block_cache,
readonly,
)
.unwrap(),
Expand Down Expand Up @@ -310,7 +306,12 @@ impl LedgerDb {
enable_storage_sharding: sharding,
..Default::default()
};
let ledger_db = Self::new(db_root_path, rocksdb_configs, /*readonly=*/ false)?;
let ledger_db = Self::new(
db_root_path,
rocksdb_configs,
/*readonly=*/ false,
None,
)?;
let cp_ledger_db_folder = cp_root_path.as_ref().join(LEDGER_DB_FOLDER_NAME);

info!(
Expand Down Expand Up @@ -443,7 +444,7 @@ impl LedgerDb {
path: PathBuf,
name: &str,
db_config: &RocksdbConfig,
block_cache: &Cache,
block_cache: Option<&Cache>,
readonly: bool,
) -> Result<DB> {
let db = if readonly {
Expand Down Expand Up @@ -484,10 +485,9 @@ impl LedgerDb {

fn gen_cfds_by_name(
db_config: &RocksdbConfig,
block_cache: &Cache,
cache: Option<&Cache>,
name: &str,
) -> Vec<ColumnFamilyDescriptor> {
let cache = Some(block_cache);
match name {
LEDGER_DB_NAME => gen_ledger_cfds(db_config, cache),
LEDGER_METADATA_DB_NAME => gen_ledger_metadata_cfds(db_config, cache),
Expand Down
7 changes: 2 additions & 5 deletions storage/aptosdb/src/state_kv_db.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@ impl StateKvDb {
rocksdb_configs: RocksdbConfigs,
readonly: bool,
ledger_db: Arc<DB>,
block_cache: Option<&Cache>,
) -> Result<Self> {
let sharding = rocksdb_configs.enable_storage_sharding;
if !sharding {
Expand All @@ -68,14 +69,10 @@ impl StateKvDb {
});
}

let block_cache = Cache::new_hyper_clock_cache(
rocksdb_configs.state_kv_db_config.block_cache_size as usize,
/* estimated_entry_charge = */ 0,
);
Self::open_sharded(
db_paths,
rocksdb_configs.state_kv_db_config,
Some(&block_cache),
block_cache,
readonly,
)
}
Expand Down
20 changes: 9 additions & 11 deletions storage/aptosdb/src/state_merkle_db.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,7 @@ impl StateMerkleDb {
// TODO(grao): Currently when this value is set to 0 we disable both caches. This is
// hacky, need to revisit.
max_nodes_per_lru_cache_shard: usize,
block_cache: Option<&Cache>,
) -> Result<Self> {
let sharding = rocksdb_configs.enable_storage_sharding;
let state_merkle_db_config = rocksdb_configs.state_merkle_db_config;
Expand All @@ -85,10 +86,6 @@ impl StateMerkleDb {
version_caches.insert(Some(i), VersionedNodeCache::new());
}
let lru_cache = NonZeroUsize::new(max_nodes_per_lru_cache_shard).map(LruNodeCache::new);
let block_cache = Cache::new_hyper_clock_cache(
state_merkle_db_config.block_cache_size as usize,
/* estimated_entry_charge = */ 0,
);

if !sharding {
info!("Sharded state merkle DB is not enabled!");
Expand All @@ -97,7 +94,7 @@ impl StateMerkleDb {
state_merkle_db_path,
STATE_MERKLE_DB_NAME,
&state_merkle_db_config,
&block_cache,
block_cache,
readonly,
)?);
return Ok(Self {
Expand All @@ -112,7 +109,7 @@ impl StateMerkleDb {
Self::open(
db_paths,
state_merkle_db_config,
&block_cache,
block_cache,
readonly,
version_caches,
lru_cache,
Expand Down Expand Up @@ -179,6 +176,7 @@ impl StateMerkleDb {
rocksdb_configs,
/*readonly=*/ false,
/*max_nodes_per_lru_cache_shard=*/ 0,
None,
)?;
let cp_state_merkle_db_path = cp_root_path.as_ref().join(STATE_MERKLE_DB_FOLDER_NAME);

Expand Down Expand Up @@ -559,7 +557,7 @@ impl StateMerkleDb {
fn open(
db_paths: &StorageDirPaths,
state_merkle_db_config: RocksdbConfig,
block_cache: &Cache,
block_cache: Option<&Cache>,
readonly: bool,
version_caches: HashMap<Option<usize>, VersionedNodeCache>,
lru_cache: Option<LruNodeCache>,
Expand Down Expand Up @@ -628,7 +626,7 @@ impl StateMerkleDb {
db_root_path: P,
shard_id: usize,
state_merkle_db_config: &RocksdbConfig,
block_cache: &Cache,
block_cache: Option<&Cache>,
readonly: bool,
) -> Result<DB> {
let db_name = format!("state_merkle_db_shard_{}", shard_id);
Expand All @@ -645,22 +643,22 @@ impl StateMerkleDb {
path: PathBuf,
name: &str,
state_merkle_db_config: &RocksdbConfig,
block_cache: &Cache,
block_cache: Option<&Cache>,
readonly: bool,
) -> Result<DB> {
Ok(if readonly {
DB::open_cf_readonly(
&gen_rocksdb_options(state_merkle_db_config, true),
path,
name,
gen_state_merkle_cfds(state_merkle_db_config, Some(block_cache)),
gen_state_merkle_cfds(state_merkle_db_config, block_cache),
)?
} else {
DB::open_cf(
&gen_rocksdb_options(state_merkle_db_config, false),
path,
name,
gen_state_merkle_cfds(state_merkle_db_config, Some(block_cache)),
gen_state_merkle_cfds(state_merkle_db_config, block_cache),
)?
})
}
Expand Down
Loading