diff --git a/Cargo.lock b/Cargo.lock index 3d2095a84bb0..b86ff564a404 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3867,7 +3867,7 @@ dependencies = [ "iana-time-zone-haiku", "js-sys", "wasm-bindgen", - "windows-core", + "windows-core 0.52.0", ] [[package]] @@ -5166,7 +5166,6 @@ dependencies = [ "linera-views-derive", "linera-witty", "linked-hash-map", - "num_cpus", "prometheus", "rand", "rocksdb", @@ -5174,6 +5173,7 @@ dependencies = [ "serde", "sha3", "static_assertions", + "sysinfo", "tempfile", "test-case", "thiserror 1.0.69", @@ -5699,6 +5699,15 @@ dependencies = [ "sha3", ] +[[package]] +name = "ntapi" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8a3895c6391c39d7fe7ebc444a87eb2991b2a0bc718fdabd071eec617fc68e4" +dependencies = [ + "winapi", +] + [[package]] name = "nu-ansi-term" version = "0.46.0" @@ -8024,6 +8033,20 @@ dependencies = [ "syn 2.0.95", ] +[[package]] +name = "sysinfo" +version = "0.33.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fc858248ea01b66f19d8e8a6d55f41deaf91e9d495246fd01368d99935c6c01" +dependencies = [ + "core-foundation-sys", + "libc", + "memchr", + "ntapi", + "rayon", + "windows", +] + [[package]] name = "system-configuration" version = "0.5.1" @@ -9547,6 +9570,16 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows" +version = "0.57.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12342cb4d8e3b046f3d80effd474a7a02447231330ef77d71daa6fbc40681143" +dependencies = [ + "windows-core 0.57.0", + "windows-targets 0.52.6", +] + [[package]] name = "windows-core" version = "0.52.0" @@ -9556,17 +9589,60 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "windows-core" +version = "0.57.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2ed2439a290666cd67ecce2b0ffaad89c2a56b976b736e6ece670297897832d" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-result 0.1.2", + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-implement" +version = "0.57.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9107ddc059d5b6fbfbffdfa7a7fe3e22a226def0b2608f72e9d552763d3e1ad7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.95", +] + +[[package]] +name = "windows-interface" +version = "0.57.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29bee4b38ea3cde66011baa44dba677c432a78593e202392d1e9070cf2a7fca7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.95", +] + [[package]] name = "windows-registry" version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e400001bb720a623c1c69032f8e3e4cf09984deec740f007dd2b03ec864804b0" dependencies = [ - "windows-result", + "windows-result 0.2.0", "windows-strings", "windows-targets 0.52.6", ] +[[package]] +name = "windows-result" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e383302e8ec8515204254685643de10811af0ed97ea37210dc26fb0032647f8" +dependencies = [ + "windows-targets 0.52.6", +] + [[package]] name = "windows-result" version = "0.2.0" @@ -9582,7 +9658,7 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4cd9b125c486025df0eabcb585e62173c6c9eddcec5d117d3b6e8c30e2ee4d10" dependencies = [ - "windows-result", + "windows-result 0.2.0", "windows-targets 0.52.6", ] diff --git a/Cargo.toml b/Cargo.toml index 27ed1e0172c0..3e78a9ec014d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -123,7 +123,6 @@ lru = "0.12.3" num-bigint = "0.4.3" num-format = "0.4.4" num-traits = "0.2.18" -num_cpus = "1.16.0" octocrab = "0.42.1" oneshot = "0.1.6" pathdiff = "0.2.1" @@ -172,6 +171,7 @@ similar-asserts = "1.5.0" static_assertions = "1.1.0" stdext = "0.3.3" syn = "2.0.52" +sysinfo = "0.33.1" tempfile = "3.10.1" test-case = "3.3.1" test-log = { version = "0.2.15", default-features = false, features = [ diff --git a/examples/Cargo.lock b/examples/Cargo.lock index e27914e689cb..92a84240384c 100644 --- a/examples/Cargo.lock +++ b/examples/Cargo.lock @@ -3397,9 +3397,9 @@ checksum = "884e2677b40cc8c339eaefcb701c32ef1fd2493d71118dc0ca4b6a736c93bd67" [[package]] name = "libc" -version = "0.2.159" +version = "0.2.171" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "561d97a539a36e26a9a5fad1ea11a3039a67714694aaa379433e580854bc3dc5" +checksum = "c19937216e9d3aa9956d9bb8dfc0b0c8beb6058fc4f7a4dc4d850edf86a237d6" [[package]] name = "libm" @@ -3689,12 +3689,12 @@ dependencies = [ "linera-views-derive", "linera-witty", "linked-hash-map", - "num_cpus", "prometheus", "rand", "serde", "sha3", "static_assertions", + "sysinfo", "tempfile", "thiserror 1.0.65", "tokio", @@ -4175,6 +4175,15 @@ dependencies = [ "tokio", ] +[[package]] +name = "ntapi" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8a3895c6391c39d7fe7ebc444a87eb2991b2a0bc718fdabd071eec617fc68e4" +dependencies = [ + "winapi", +] + [[package]] name = "nu-ansi-term" version = "0.46.0" @@ -5854,6 +5863,20 @@ dependencies = [ "walkdir", ] +[[package]] +name = "sysinfo" +version = "0.33.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fc858248ea01b66f19d8e8a6d55f41deaf91e9d495246fd01368d99935c6c01" +dependencies = [ + "core-foundation-sys", + "libc", + "memchr", + "ntapi", + "rayon", + "windows", +] + [[package]] name = "system-configuration" version = "0.5.1" @@ -6737,17 +6760,70 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows" +version = "0.57.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12342cb4d8e3b046f3d80effd474a7a02447231330ef77d71daa6fbc40681143" +dependencies = [ + "windows-core", + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-core" +version = "0.57.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2ed2439a290666cd67ecce2b0ffaad89c2a56b976b736e6ece670297897832d" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-result 0.1.2", + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-implement" +version = "0.57.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9107ddc059d5b6fbfbffdfa7a7fe3e22a226def0b2608f72e9d552763d3e1ad7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.95", +] + +[[package]] +name = "windows-interface" +version = "0.57.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29bee4b38ea3cde66011baa44dba677c432a78593e202392d1e9070cf2a7fca7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.95", +] + [[package]] name = "windows-registry" version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e400001bb720a623c1c69032f8e3e4cf09984deec740f007dd2b03ec864804b0" dependencies = [ - "windows-result", + "windows-result 0.2.0", "windows-strings", "windows-targets 0.52.6", ] +[[package]] +name = "windows-result" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e383302e8ec8515204254685643de10811af0ed97ea37210dc26fb0032647f8" +dependencies = [ + "windows-targets 0.52.6", +] + [[package]] name = "windows-result" version = "0.2.0" @@ -6763,7 +6839,7 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4cd9b125c486025df0eabcb585e62173c6c9eddcec5d117d3b6e8c30e2ee4d10" dependencies = [ - "windows-result", + "windows-result 0.2.0", "windows-targets 0.52.6", ] diff --git a/linera-views/Cargo.toml b/linera-views/Cargo.toml index e7947d54b9ae..466c5829cde6 100644 --- a/linera-views/Cargo.toml +++ b/linera-views/Cargo.toml @@ -45,7 +45,6 @@ linera-base.workspace = true linera-views-derive.workspace = true linera-witty.workspace = true linked-hash-map.workspace = true -num_cpus.workspace = true prometheus.workspace = true rand = { workspace = true, features = ["small_rng"] } rocksdb = { workspace = true, optional = true } @@ -53,6 +52,7 @@ scylla = { workspace = true, optional = true } serde.workspace = true sha3.workspace = true static_assertions.workspace = true +sysinfo.workspace = true tempfile.workspace = true thiserror.workspace = true tokio = { workspace = true, features = ["rt", "sync"] } diff --git a/linera-views/src/backends/rocks_db.rs b/linera-views/src/backends/rocks_db.rs index 7e8748177e5c..1163fff4c1d1 100644 --- a/linera-views/src/backends/rocks_db.rs +++ b/linera-views/src/backends/rocks_db.rs @@ -14,7 +14,9 @@ use std::{ }; use linera_base::ensure; +use rocksdb::{BlockBasedOptions, Cache, DBCompactionStyle}; use serde::{Deserialize, Serialize}; +use sysinfo::{CpuRefreshKind, MemoryRefreshKind, RefreshKind, System}; use tempfile::TempDir; use thiserror::Error; @@ -49,8 +51,9 @@ const MAX_VALUE_SIZE: usize = 3221225072; // 8388608 and so for offset reason we decrease by 400 const MAX_KEY_SIZE: usize = 8388208; -const DB_CACHE_SIZE: usize = 128 * 1024 * 1024; // 128 MiB -const DB_MAX_WRITE_BUFFER_NUMBER: i32 = 8; +const WRITE_BUFFER_SIZE: usize = 64 * 1024 * 1024; // 64 MB +const MAX_WRITE_BUFFER_NUMBER: i32 = 32; +const HYPER_CLOCK_CACHE_BLOCK_SIZE: usize = 8 * 1024; // 8 KB /// The RocksDB client that we use. type DB = rocksdb::DBWithThreadMode; @@ -300,21 +303,48 @@ impl RocksDbStoreInternal { if !std::path::Path::exists(&path_buf) { std::fs::create_dir(path_buf.clone())?; } + let sys = System::new_with_specifics( + RefreshKind::nothing() + .with_cpu(CpuRefreshKind::everything()) + .with_memory(MemoryRefreshKind::nothing().with_ram()), + ); + let num_cpus = sys.cpus().len() as i32; + let total_ram = sys.total_memory() as usize; let mut options = rocksdb::Options::default(); options.create_if_missing(true); options.create_missing_column_families(true); // Flush in-memory buffer to disk more often - options.set_write_buffer_size(DB_CACHE_SIZE); - options.set_max_write_buffer_number(DB_MAX_WRITE_BUFFER_NUMBER); + options.set_write_buffer_size(WRITE_BUFFER_SIZE); + options.set_max_write_buffer_number(MAX_WRITE_BUFFER_NUMBER); options.set_compression_type(rocksdb::DBCompressionType::Lz4); - options.set_level_zero_slowdown_writes_trigger(-1); - options.set_level_zero_stop_writes_trigger(48); - options.set_stats_dump_period_sec(60); - options.enable_statistics(); - options.increase_parallelism(num_cpus::get() as i32); - options.set_max_background_jobs(8); + options.set_level_zero_slowdown_writes_trigger(12); + options.set_level_zero_stop_writes_trigger(20); + // We use half the available CPUs for RocksDB parallelism to allow concurrent operations + // while leaving resources for other application tasks. Using a third of CPUs for background + // jobs (compactions, flushes) balances background maintenance with foreground operations, + // preventing RocksDB from consuming too many system resources, while still keeping good + // performance. + options.increase_parallelism((num_cpus / 2).max(1)); + options.set_max_background_jobs((num_cpus / 3).max(1)); options.set_level_compaction_dynamic_level_bytes(true); + options.set_compaction_style(DBCompactionStyle::Level); + options.set_target_file_size_base(WRITE_BUFFER_SIZE as u64); + + let mut block_options = BlockBasedOptions::default(); + block_options.set_pin_l0_filter_and_index_blocks_in_cache(true); + block_options.set_cache_index_and_filter_blocks(true); + // Allocate 1/4 of total RAM for RocksDB block cache, which is a reasonable balance: + // - Large enough to significantly improve read performance by caching frequently accessed blocks + // - Small enough to leave memory for other system components + // - Follows common practice for database caching in server environments + // - Prevents excessive memory pressure that could lead to swapping or OOM conditions + block_options.set_block_cache(&Cache::new_hyper_clock_cache( + total_ram / 4, + HYPER_CLOCK_CACHE_BLOCK_SIZE, + )); + options.set_block_based_table_factory(&block_options); + let db = DB::open(&options, path_buf)?; let executor = RocksDbStoreExecutor { db: Arc::new(db),