diff --git a/CHANGELOG.md b/CHANGELOG.md index d439fbe30b..96aba8eb8d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,12 @@ ### 2025-11-03 +- Switch to binary fuse filter for added performance on trie layers [#5159](https://github.com/lambdaclass/ethrex/pull/5159) + +### 2025-10-31 + - Avoid unnecessary hash validations [#5167](https://github.com/lambdaclass/ethrex/pull/5167) + - Merge execution with some post-execution validations [#5170](https://github.com/lambdaclass/ethrex/pull/5170) ### 2025-10-31 diff --git a/Cargo.lock b/Cargo.lock index 06f830f89e..1d3d8792ca 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2151,6 +2151,17 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "cuckoofilter" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b810a8449931679f64cd7eef1bbd0fa315801b6d5d9cdc1ace2804d6529eee18" +dependencies = [ + "byteorder", + "fnv", + "rand 0.7.3", +] + [[package]] name = "cust" version = "0.3.2" @@ -3761,13 +3772,13 @@ dependencies = [ "async-trait", "bincode", "bytes", + "cuckoofilter", "ethereum-types 0.15.1", "ethrex-common", "ethrex-rlp", "ethrex-trie", "hex", "hex-literal", - "qfilter", "rayon", "rocksdb", "rustc-hash 2.1.1", @@ -4302,6 +4313,17 @@ dependencies = [ "typenum", ] +[[package]] +name = "getrandom" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce" +dependencies = [ + "cfg-if 1.0.4", + "libc", + "wasi 0.9.0+wasi-snapshot-preview1", +] + [[package]] name = "getrandom" version = "0.2.16" @@ -4311,7 +4333,7 @@ dependencies = [ "cfg-if 1.0.4", "js-sys", "libc", - "wasi", + "wasi 0.11.1+wasi-snapshot-preview1", "wasm-bindgen", ] @@ -6223,7 +6245,7 @@ checksum = "69d83b0086dc8ecf3ce9ae2874b2d1290252e2a30720bea58a5c6639b0092873" dependencies = [ "libc", "log", - "wasi", + "wasi 0.11.1+wasi-snapshot-preview1", "windows-sys 0.61.2", ] @@ -7756,15 +7778,6 @@ dependencies = [ "parking_lot", ] -[[package]] -name = "qfilter" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "746341cd2357c9a4df2d951522b4a8dd1ef553e543119899ad7bf87e938c8fbe" -dependencies = [ - "xxhash-rust", -] - [[package]] name = "quick-error" version = "1.2.3" @@ -7865,6 +7878,19 @@ dependencies = [ "ptr_meta", ] +[[package]] +name = "rand" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03" +dependencies = [ + "getrandom 0.1.16", + "libc", + "rand_chacha 0.2.2", + "rand_core 0.5.1", + "rand_hc", +] + [[package]] name = "rand" version = "0.8.5" @@ -7887,6 +7913,16 @@ dependencies = [ "serde", ] +[[package]] +name = "rand_chacha" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402" +dependencies = [ + "ppv-lite86", + "rand_core 0.5.1", +] + [[package]] name = "rand_chacha" version = "0.3.1" @@ -7907,6 +7943,15 @@ dependencies = [ "rand_core 0.9.3", ] +[[package]] +name = "rand_core" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19" +dependencies = [ + "getrandom 0.1.16", +] + [[package]] name = "rand_core" version = "0.6.4" @@ -7926,6 +7971,15 @@ dependencies = [ "serde", ] +[[package]] +name = "rand_hc" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c" +dependencies = [ + "rand_core 0.5.1", +] + [[package]] name = "rand_xorshift" version = "0.4.0" @@ -11637,6 +11691,12 @@ dependencies = [ "try-lock", ] +[[package]] +name = "wasi" +version = "0.9.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" + [[package]] name = "wasi" version = "0.11.1+wasi-snapshot-preview1" @@ -12241,12 +12301,6 @@ dependencies = [ "tap", ] -[[package]] -name = "xxhash-rust" -version = "0.8.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fdd20c5420375476fbd4394763288da7eb0cc0b8c11deed431a91562af7335d3" - [[package]] name = "yaml-rust2" version = "0.10.4" diff --git a/crates/l2/tee/quote-gen/Cargo.lock b/crates/l2/tee/quote-gen/Cargo.lock index 77b00c85eb..ba71994ee6 100644 --- a/crates/l2/tee/quote-gen/Cargo.lock +++ b/crates/l2/tee/quote-gen/Cargo.lock @@ -2075,12 +2075,14 @@ name = "ethrex-blockchain" version = "5.0.0" dependencies = [ "bytes", - "cfg-if 1.0.3", "ethrex-common", "ethrex-metrics", "ethrex-rlp", "ethrex-storage", + "ethrex-trie", "ethrex-vm", + "hex", + "rustc-hash", "secp256k1", "sha3", "thiserror 2.0.16", @@ -2094,17 +2096,19 @@ name = "ethrex-common" version = "5.0.0" dependencies = [ "bytes", - "c-kzg", "crc32fast", "ethereum-types 0.15.1", + "ethrex-crypto", "ethrex-rlp", "ethrex-trie", "hex", "kzg-rs", "lazy_static", + "libc", "once_cell", "rayon", "rkyv", + "rustc-hash", "secp256k1", "serde 1.0.228", "serde_json", @@ -2163,7 +2167,6 @@ dependencies = [ "axum", "bincode", "bytes", - "cfg-if 1.0.3", "chrono", "clap", "color-eyre", @@ -2284,6 +2287,7 @@ dependencies = [ "malachite", "p256", "ripemd", + "rustc-hash", "secp256k1", "serde 1.0.228", "serde_json", @@ -2366,7 +2370,6 @@ dependencies = [ "axum", "axum-extra", "bytes", - "cfg-if 1.0.3", "envy", "ethereum-types 0.15.1", "ethrex-blockchain", @@ -2440,14 +2443,13 @@ dependencies = [ "ethrex-rlp", "ethrex-trie", "hex", - "qfilter", - "rayon", "rustc-hash", "serde 1.0.228", "serde_json", "sha3", "thiserror 2.0.16", "tracing", + "xorfilter-rs", ] [[package]] @@ -2503,7 +2505,6 @@ version = "5.0.0" dependencies = [ "bincode", "bytes", - "cfg-if 1.0.3", "derive_more 1.0.0", "dyn-clone", "ethereum-types 0.15.1", @@ -2882,6 +2883,7 @@ dependencies = [ "bytes", "ethrex-blockchain", "ethrex-common", + "ethrex-crypto", "ethrex-l2-common", "ethrex-rlp", "ethrex-storage", @@ -4747,15 +4749,6 @@ dependencies = [ "syn 2.0.106", ] -[[package]] -name = "qfilter" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "746341cd2357c9a4df2d951522b4a8dd1ef553e543119899ad7bf87e938c8fbe" -dependencies = [ - "xxhash-rust", -] - [[package]] name = "quote" version = "0.3.15" @@ -7479,10 +7472,10 @@ dependencies = [ ] [[package]] -name = "xxhash-rust" -version = "0.8.15" +name = "xorfilter-rs" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fdd20c5420375476fbd4394763288da7eb0cc0b8c11deed431a91562af7335d3" +checksum = "47f9da296a88b6bc150b896d17770a62d4dc6f63ecf0ed10a9c08a1cb3d12f24" [[package]] name = "yansi" diff --git a/crates/storage/Cargo.toml b/crates/storage/Cargo.toml index c95898bae7..4e357f7804 100644 --- a/crates/storage/Cargo.toml +++ b/crates/storage/Cargo.toml @@ -26,7 +26,7 @@ rocksdb = { workspace = true, optional = true } rustc-hash.workspace = true tokio = { workspace = true, optional = true, features = ["rt"] } bincode = "1.3.3" -qfilter = "0.2.5" +cuckoofilter = "0.5.0" rayon.workspace = true [features] diff --git a/crates/storage/trie_db/layering.rs b/crates/storage/trie_db/layering.rs index ae52f3b729..5b19948da5 100644 --- a/crates/storage/trie_db/layering.rs +++ b/crates/storage/trie_db/layering.rs @@ -1,6 +1,5 @@ use ethrex_common::H256; -use rayon::iter::{ParallelBridge, ParallelIterator}; -use rustc_hash::FxHashMap; +use rustc_hash::{FxHashMap, FxHasher}; use std::sync::Arc; use ethrex_trie::{Nibbles, TrieDB, TrieError}; @@ -12,7 +11,7 @@ struct TrieLayer { id: usize, } -#[derive(Clone, Debug)] +#[derive(Clone, Default)] pub struct TrieLayerCache { /// Monotonically increasing ID for layers, starting at 1. /// TODO: this implementation panics on overflow @@ -26,26 +25,23 @@ pub struct TrieLayerCache { /// In case a bloom filter insert or merge fails, we need to mark the bloom filter as poisoned /// so we never use it again, because if we don't we may be misled into believing a key is not present /// on a diff layer when it is (i.e. a false negative), leading to wrong executions. - bloom: Option, + bloom: Option>>, } -impl Default for TrieLayerCache { - fn default() -> Self { - // Try to create the bloom filter, if it fails use poison mode. - let bloom = Self::create_filter().ok(); - Self { - bloom, - last_id: 0, - layers: Default::default(), - } +impl std::fmt::Debug for TrieLayerCache { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("TrieLayerCache") + .field("last_id", &self.last_id) + .field("layers", &self.layers) + // bloom doesn't implement Debug + .finish_non_exhaustive() } } impl TrieLayerCache { // TODO: tune this - fn create_filter() -> Result { - qfilter::Filter::new_resizeable(1_000_000, 100_000_000, 0.02) - .inspect_err(|e| tracing::warn!("could not create trie layering bloom filter {e}")) + fn create_filter() -> cuckoofilter::CuckooFilter { + cuckoofilter::CuckooFilter::with_capacity(50_000_000_000) } pub fn get(&self, state_root: H256, key: Nibbles) -> Option> { @@ -110,70 +106,38 @@ impl TrieLayerCache { return; } - // add this new bloom to the global one. - if let Some(filter) = &mut self.bloom { - for (p, _) in &key_values { - if let Err(qfilter::Error::CapacityExceeded) = filter.insert(p.as_ref()) { - tracing::warn!("TrieLayerCache: put_batch capacity exceeded"); - self.bloom = None; - break; - } - } - } - let nodes: FxHashMap, Vec> = key_values .into_iter() .map(|(path, value)| (path.into_vec(), value)) .collect(); self.last_id += 1; + + // Note: bloom doesn't need to be edited here since commit will be called after this method, + // which removes the oldest layer and thus rebuilds the bloom. + let entry = TrieLayer { nodes: Arc::new(nodes), parent, id: self.last_id, }; + self.layers.insert(state_root, Arc::new(entry)); } /// Rebuilds the global bloom filter accruing all current existing layers. pub fn rebuild_bloom(&mut self) { - let mut blooms: Vec<_> = self - .layers - .values() - .par_bridge() - .map(|entry| { - let Ok(mut bloom) = Self::create_filter() else { - tracing::warn!("TrieLayerCache: rebuild_bloom could not create filter"); - return None; - }; - for (p, _) in entry.nodes.iter() { - if let Err(qfilter::Error::CapacityExceeded) = bloom.insert(p) { - tracing::warn!("TrieLayerCache: rebuild_bloom capacity exceeded"); - return None; - } - } - Some(bloom) - }) - .collect(); + let mut bloom = Self::create_filter(); - let Some(mut ret) = blooms.pop().flatten() else { - tracing::warn!("TrieLayerCache: rebuild_bloom no valid bloom found"); - self.bloom = None; - return; - }; - for bloom in blooms.iter() { - let Some(bloom) = bloom else { - tracing::warn!("TrieLayerCache: rebuild_bloom no valid bloom found"); - self.bloom = None; - return; - }; - if let Err(qfilter::Error::CapacityExceeded) = ret.merge(false, bloom) { - tracing::warn!("TrieLayerCache: rebuild_bloom capacity exceeded"); + for key in self.layers.values().flat_map(|x| x.nodes.keys()) { + if let Err(e) = bloom.add(key) { + tracing::warn!("TrieLayerCache: rebuild_bloom error: {e}"); self.bloom = None; return; } } - self.bloom = Some(ret); + + self.bloom = Some(Arc::new(bloom)); } pub fn commit(&mut self, state_root: H256) -> Option, Vec)>> {