From 6971453e1b7b093427a6e5cedf03d9ceaf5a2ee4 Mon Sep 17 00:00:00 2001 From: codyx Date: Fri, 31 Jan 2025 14:58:34 +0400 Subject: [PATCH 1/6] Remove duplicate headers fetching and MPT constructions --- crates/fetcher/src/lib.rs | 184 ++++++++++++++++++--------- crates/fetcher/src/proof_keys/evm.rs | 94 +++++++++----- 2 files changed, 186 insertions(+), 92 deletions(-) diff --git a/crates/fetcher/src/lib.rs b/crates/fetcher/src/lib.rs index 439345ae..2605af11 100644 --- a/crates/fetcher/src/lib.rs +++ b/crates/fetcher/src/lib.rs @@ -1,23 +1,36 @@ use std::{ collections::{HashMap, HashSet}, + env, num::ParseIntError, + sync::Arc, }; use alloy::hex::FromHexError; use dry_hint_processor::syscall_handler::{evm, starknet, SyscallHandler}; +use eth_trie_proofs::{tx_receipt_trie::TxReceiptsMptHandler, tx_trie::TxsMptHandler}; use futures::StreamExt; use indexer::models::IndexerError; use indicatif::{MultiProgress, ProgressBar, ProgressStyle}; -use proof_keys::{evm::ProofKeys as EvmProofKeys, starknet::ProofKeys as StarknetProofKeys, ProofKeys}; +use proof_keys::{ + evm::{FlattenedKey, ProofKeys as EvmProofKeys}, + starknet::ProofKeys as StarknetProofKeys, + ProofKeys, +}; +use reqwest::Url; use starknet_types_core::felt::FromStrError; use thiserror::Error; -use types::proofs::{ - evm::{ - account::Account, header::Header as EvmHeader, receipt::Receipt, storage::Storage, transaction::Transaction, Proofs as EvmProofs, +use tokio::sync::Mutex; +use types::{ + proofs::{ + evm::{ + account::Account, header::Header as EvmHeader, receipt::Receipt, storage::Storage, transaction::Transaction, + Proofs as EvmProofs, + }, + header::HeaderMmrMeta, + mmr::MmrMeta, + starknet::{header::Header as StarknetHeader, storage::Storage as StarknetStorage, Proofs as StarknetProofs}, }, - header::HeaderMmrMeta, - mmr::MmrMeta, - starknet::{header::Header as StarknetHeader, storage::Storage as StarknetStorage, Proofs as StarknetProofs}, + RPC_URL_ETHEREUM, }; pub mod proof_keys; @@ -127,7 +140,6 @@ pub struct Fetcher<'a> { #[cfg(feature = "progress_bars")] progress_bars: ProgressBars, } - impl<'a> Fetcher<'a> { pub fn new(proof_keys: &'a ProofKeys) -> Self { Self { @@ -137,26 +149,19 @@ impl<'a> Fetcher<'a> { } } - pub async fn collect_evm_proofs(&self) -> Result { + async fn collect_evm_headers_proofs( + &self, + flattened_keys: &HashSet, + ) -> Result>, FetcherError> { let mut headers_with_mmr = HashMap::default(); - let mut accounts: HashSet = HashSet::default(); - let mut storages: HashSet = HashSet::default(); - let mut receipts: HashSet = HashSet::default(); - let mut transactions: HashSet = HashSet::default(); - - // Collect header proofs let mut header_fut = futures::stream::iter( - self.proof_keys - .evm - .header_keys + flattened_keys .iter() .map(|key| EvmProofKeys::fetch_header_proof(key.chain_id, key.block_number)), - ) - .buffer_unordered(BUFFER_UNORDERED) - .boxed(); + ); while let Some(result) = header_fut.next().await { - let item = result?; + let item = result.await?; headers_with_mmr .entry(item.mmr_meta) .and_modify(|headers: &mut Vec| headers.extend(item.headers.clone())) @@ -166,10 +171,22 @@ impl<'a> Fetcher<'a> { self.progress_bars.evm_header.safe_inc(); } + Ok(headers_with_mmr) + } + + pub async fn collect_evm_proofs(&self) -> Result { + let mut accounts: HashSet = HashSet::default(); + let mut storages: HashSet = HashSet::default(); + let mut receipts: HashSet = HashSet::default(); + let mut transactions: HashSet = HashSet::default(); + + let flattened_keys = self.proof_keys.evm.to_flattened_keys(); + + // Collect required header proofs for all keys + let headers_with_mmr = self.collect_evm_headers_proofs(&flattened_keys).await?; + #[cfg(feature = "progress_bars")] - self.progress_bars - .evm_header - .safe_finish_with_message("ethereum header keys - finished"); + self.progress_bars.evm_header.safe_finish_with_message("evm header keys - finished"); // Collect account proofs let mut account_fut = futures::stream::iter(self.proof_keys.evm.account_keys.iter().map(EvmProofKeys::fetch_account_proof)) @@ -177,12 +194,7 @@ impl<'a> Fetcher<'a> { .boxed(); while let Some(result) = account_fut.next().await { - let (header_with_mmr, account) = result?; - headers_with_mmr - .entry(header_with_mmr.mmr_meta) - .and_modify(|headers: &mut Vec| headers.extend(header_with_mmr.headers.clone())) - .or_insert(header_with_mmr.headers); - accounts.insert(account); + accounts.insert(result?); #[cfg(feature = "progress_bars")] self.progress_bars.evm_account.safe_inc(); @@ -191,7 +203,7 @@ impl<'a> Fetcher<'a> { #[cfg(feature = "progress_bars")] self.progress_bars .evm_account - .safe_finish_with_message("ethereum account keys - finished"); + .safe_finish_with_message("evm account keys - finished"); // Collect storage proofs let mut storage_fut = futures::stream::iter(self.proof_keys.evm.storage_keys.iter().map(EvmProofKeys::fetch_storage_proof)) @@ -199,11 +211,7 @@ impl<'a> Fetcher<'a> { .boxed(); while let Some(result) = storage_fut.next().await { - let (header_with_mmr, account, storage) = result?; - headers_with_mmr - .entry(header_with_mmr.mmr_meta) - .and_modify(|headers: &mut Vec| headers.extend(header_with_mmr.headers.clone())) - .or_insert(header_with_mmr.headers); + let (account, storage) = result?; accounts.insert(account); storages.insert(storage); @@ -214,19 +222,47 @@ impl<'a> Fetcher<'a> { #[cfg(feature = "progress_bars")] self.progress_bars .evm_storage - .safe_finish_with_message("ethereum storage keys - finished"); + .safe_finish_with_message("evm storage keys - finished"); - // Collect ransaction receipts proofs - let mut transaction_receipts_fut = - futures::stream::iter(self.proof_keys.evm.receipt_keys.iter().map(EvmProofKeys::fetch_receipt_proof)) - .buffer_unordered(BUFFER_UNORDERED) - .boxed(); + // For each receipt block, we need to create a tx_receipts_mpt_handler + let mut tx_receipts_mpt_handlers: HashMap> = HashMap::default(); - while let Some(Ok((header_with_mmr, transaction_receipt))) = transaction_receipts_fut.next().await { - headers_with_mmr - .entry(header_with_mmr.mmr_meta) - .and_modify(|headers: &mut Vec| headers.extend(header_with_mmr.headers.clone())) - .or_insert(header_with_mmr.headers); + for key in self.proof_keys.evm.receipt_keys.iter() { + if !tx_receipts_mpt_handlers.contains_key(&key.block_number) { + let tx_receipts_mpt_handler = TxReceiptsMptHandler::new(Url::parse(&env::var(RPC_URL_ETHEREUM).unwrap()).unwrap()) + .map_err(|e| FetcherError::InternalError(e.to_string()))?; + + let mut boxed_handler = Box::new(tx_receipts_mpt_handler); + + // Build the receipt root for the block + boxed_handler + .build_tx_receipts_tree_from_block(key.block_number) + .await + .map_err(|e| FetcherError::InternalError(e.to_string()))?; + + tx_receipts_mpt_handlers.insert(key.block_number, boxed_handler); + } + } + + // Convert HashMap to Arc for sharing + let tx_receipts_mpt_handlers = Arc::new(Mutex::new(tx_receipts_mpt_handlers)); + let tx_receipts_mpt_handlers_clone = tx_receipts_mpt_handlers.clone(); + + let mut transaction_receipts_fut = futures::stream::iter(self.proof_keys.evm.receipt_keys.iter().map({ + move |key| { + let handlers = tx_receipts_mpt_handlers_clone.clone(); + async move { + let mut handlers_guard = handlers.lock().await; + let handler = handlers_guard.get_mut(&key.block_number).unwrap(); + + EvmProofKeys::fetch_receipt_proof(key, handler.as_mut()).await + } + } + })) + .buffer_unordered(BUFFER_UNORDERED) + .boxed(); + + while let Some(Ok(transaction_receipt)) = transaction_receipts_fut.next().await { receipts.insert(transaction_receipt); #[cfg(feature = "progress_bars")] @@ -236,24 +272,46 @@ impl<'a> Fetcher<'a> { #[cfg(feature = "progress_bars")] self.progress_bars .evm_receipts - .safe_finish_with_message("ethereum receipt keys - finished"); + .safe_finish_with_message("evm receipt keys - finished"); - // Collect storage proofs - let mut transaction_keys_fut = futures::stream::iter( - self.proof_keys - .evm - .transaction_keys - .iter() - .map(EvmProofKeys::fetch_transaction_proof), - ) + let mut tx_trie_handlers: HashMap> = HashMap::default(); + + for key in self.proof_keys.evm.transaction_keys.iter() { + if !tx_trie_handlers.contains_key(&key.block_number) { + let tx_trie_handler = TxsMptHandler::new(Url::parse(&env::var(RPC_URL_ETHEREUM).unwrap()).unwrap()) + .map_err(|e| FetcherError::InternalError(e.to_string()))?; + + let mut boxed_handler = Box::new(tx_trie_handler); + + boxed_handler + .build_tx_tree_from_block(key.block_number) + .await + .map_err(|e| FetcherError::InternalError(e.to_string()))?; + + tx_trie_handlers.insert(key.block_number, boxed_handler); + } + } + + // Convert HashMap to Arc for sharing + let tx_trie_handlers = Arc::new(Mutex::new(tx_trie_handlers)); + let tx_trie_handlers_clone = tx_trie_handlers.clone(); + + // Collect transaction proofs + let mut transaction_keys_fut = futures::stream::iter(self.proof_keys.evm.transaction_keys.iter().map({ + move |key| { + let handlers = tx_trie_handlers_clone.clone(); + async move { + let mut handlers_guard = handlers.lock().await; + let handler = handlers_guard.get_mut(&key.block_number).unwrap(); + + EvmProofKeys::fetch_transaction_proof(key, handler.as_mut()).await + } + } + })) .buffer_unordered(BUFFER_UNORDERED) .boxed(); - while let Some(Ok((header_with_mmr, transaction))) = transaction_keys_fut.next().await { - headers_with_mmr - .entry(header_with_mmr.mmr_meta) - .and_modify(|headers: &mut Vec| headers.extend(header_with_mmr.headers.clone())) - .or_insert(header_with_mmr.headers); + while let Some(Ok(transaction)) = transaction_keys_fut.next().await { transactions.insert(transaction); #[cfg(feature = "progress_bars")] @@ -263,7 +321,7 @@ impl<'a> Fetcher<'a> { #[cfg(feature = "progress_bars")] self.progress_bars .evm_transactions - .safe_finish_with_message("ethereum storage keys - finished"); + .safe_finish_with_message("evm storage keys - finished"); Ok(EvmProofs { headers_with_mmr: process_headers(headers_with_mmr), diff --git a/crates/fetcher/src/proof_keys/evm.rs b/crates/fetcher/src/proof_keys/evm.rs index 0d37e555..9d03719d 100644 --- a/crates/fetcher/src/proof_keys/evm.rs +++ b/crates/fetcher/src/proof_keys/evm.rs @@ -31,6 +31,12 @@ pub struct ProofKeys { pub transaction_keys: HashSet, } +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct FlattenedKey { + pub chain_id: u128, + pub block_number: u64, +} + impl ProofKeys { fn normalize_hex(input: &str) -> String { let hex_str = input.trim_start_matches("0x"); @@ -70,20 +76,20 @@ impl ProofKeys { }) } - pub async fn fetch_account_proof(key: &keys::evm::account::Key) -> Result<(HeaderMmrMeta
, Account), FetcherError> { + pub async fn fetch_account_proof(key: &keys::evm::account::Key) -> Result { let provider = RootProvider::>::new_http(Url::parse(&env::var(RPC_URL_ETHEREUM).unwrap()).unwrap()); let value = provider .get_proof(key.address, vec![]) .block_id(key.block_number.into()) .await .map_err(|e| FetcherError::InternalError(e.to_string()))?; - Ok(( - Self::fetch_header_proof(key.chain_id, key.block_number).await?, - Account::new(value.address, vec![MPTProof::new(key.block_number, value.account_proof)]), + Ok(Account::new( + value.address, + vec![MPTProof::new(key.block_number, value.account_proof)], )) } - pub async fn fetch_storage_proof(key: &keys::evm::storage::Key) -> Result<(HeaderMmrMeta
, Account, Storage), FetcherError> { + pub async fn fetch_storage_proof(key: &keys::evm::storage::Key) -> Result<(Account, Storage), FetcherError> { let provider = RootProvider::>::new_http(Url::parse(&env::var(RPC_URL_ETHEREUM).unwrap()).unwrap()); let value = provider .get_proof(key.address, vec![key.storage_slot]) @@ -91,7 +97,6 @@ impl ProofKeys { .await .map_err(|e| FetcherError::InternalError(e.to_string()))?; Ok(( - Self::fetch_header_proof(key.chain_id, key.block_number).await?, Account::new(value.address, vec![MPTProof::new(key.block_number, value.account_proof)]), Storage::new( value.address, @@ -109,10 +114,6 @@ impl ProofKeys { block_number: u64, tx_index: u64, ) -> Result { - tx_receipts_mpt_handler - .build_tx_receipts_tree_from_block(block_number) - .await - .map_err(|e| FetcherError::InternalError(e.to_string()))?; let trie_proof = tx_receipts_mpt_handler .get_proof(tx_index) .map_err(|e| FetcherError::InternalError(e.to_string()))?; @@ -125,16 +126,15 @@ impl ProofKeys { Ok(MPTProof { block_number, proof }) } - pub async fn fetch_receipt_proof(key: &keys::evm::receipt::Key) -> Result<(HeaderMmrMeta
, Receipt), FetcherError> { - let mut tx_receipts_mpt_handler = TxReceiptsMptHandler::new(Url::parse(&env::var(RPC_URL_ETHEREUM).unwrap()).unwrap()) - .map_err(|e| FetcherError::InternalError(e.to_string()))?; - - let header = Self::fetch_header_proof(key.chain_id, key.block_number).await?; + pub async fn fetch_receipt_proof( + key: &keys::evm::receipt::Key, + tx_receipts_mpt_handler: &mut TxReceiptsMptHandler, + ) -> Result { let receipt_mpt_proof = - Self::generate_block_tx_receipt_proof(&mut tx_receipts_mpt_handler, key.block_number, key.transaction_index).await?; - + Self::generate_block_tx_receipt_proof(tx_receipts_mpt_handler, key.block_number, key.transaction_index).await?; let rlp_encoded_key = alloy_rlp::encode(U256::from(key.transaction_index)); - Ok((header, Receipt::new(U256::from_be_slice(&rlp_encoded_key), receipt_mpt_proof))) + + Ok(Receipt::new(U256::from_be_slice(&rlp_encoded_key), receipt_mpt_proof)) } async fn generate_block_tx_proof( @@ -142,10 +142,6 @@ impl ProofKeys { block_number: u64, tx_index: u64, ) -> Result { - tx_trie_handler - .build_tx_tree_from_block(block_number) - .await - .map_err(|e| FetcherError::InternalError(e.to_string()))?; let trie_proof = tx_trie_handler .get_proof(tx_index) .map_err(|e| FetcherError::InternalError(e.to_string()))?; @@ -158,14 +154,54 @@ impl ProofKeys { Ok(MPTProof { block_number, proof }) } - pub async fn fetch_transaction_proof(key: &keys::evm::transaction::Key) -> Result<(HeaderMmrMeta
, Transaction), FetcherError> { - let mut tx_trie_handler = TxsMptHandler::new(Url::parse(&env::var(RPC_URL_ETHEREUM).unwrap()).unwrap()) - .map_err(|e| FetcherError::InternalError(e.to_string()))?; - - let header = Self::fetch_header_proof(key.chain_id, key.block_number).await?; - let tx_proof = Self::generate_block_tx_proof(&mut tx_trie_handler, key.block_number, key.transaction_index).await?; + pub async fn fetch_transaction_proof( + key: &keys::evm::transaction::Key, + tx_trie_handler: &mut TxsMptHandler, + ) -> Result { + let tx_proof = Self::generate_block_tx_proof(tx_trie_handler, key.block_number, key.transaction_index).await?; let rlp_encoded_key = alloy_rlp::encode(U256::from(key.transaction_index)); - Ok((header, Transaction::new(U256::from_be_slice(&rlp_encoded_key), tx_proof))) + Ok(Transaction::new(U256::from_be_slice(&rlp_encoded_key), tx_proof)) + } + + pub fn to_flattened_keys(&self) -> HashSet { + let mut flattened = HashSet::new(); + + for key in &self.header_keys { + flattened.insert(FlattenedKey { + chain_id: key.chain_id, + block_number: key.block_number, + }); + } + + for key in &self.account_keys { + flattened.insert(FlattenedKey { + chain_id: key.chain_id, + block_number: key.block_number, + }); + } + + for key in &self.storage_keys { + flattened.insert(FlattenedKey { + chain_id: key.chain_id, + block_number: key.block_number, + }); + } + + for key in &self.receipt_keys { + flattened.insert(FlattenedKey { + chain_id: key.chain_id, + block_number: key.block_number, + }); + } + + for key in &self.transaction_keys { + flattened.insert(FlattenedKey { + chain_id: key.chain_id, + block_number: key.block_number, + }); + } + + flattened } } From 0a5af705b6fda1df3baf095deaac64eccbac5f66 Mon Sep 17 00:00:00 2001 From: codyx Date: Fri, 31 Jan 2025 15:07:56 +0400 Subject: [PATCH 2/6] clippy fixes --- crates/fetcher/src/lib.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/fetcher/src/lib.rs b/crates/fetcher/src/lib.rs index 2605af11..d2f77664 100644 --- a/crates/fetcher/src/lib.rs +++ b/crates/fetcher/src/lib.rs @@ -228,7 +228,7 @@ impl<'a> Fetcher<'a> { let mut tx_receipts_mpt_handlers: HashMap> = HashMap::default(); for key in self.proof_keys.evm.receipt_keys.iter() { - if !tx_receipts_mpt_handlers.contains_key(&key.block_number) { + if let std::collections::hash_map::Entry::Vacant(e) = tx_receipts_mpt_handlers.entry(key.block_number) { let tx_receipts_mpt_handler = TxReceiptsMptHandler::new(Url::parse(&env::var(RPC_URL_ETHEREUM).unwrap()).unwrap()) .map_err(|e| FetcherError::InternalError(e.to_string()))?; @@ -240,7 +240,7 @@ impl<'a> Fetcher<'a> { .await .map_err(|e| FetcherError::InternalError(e.to_string()))?; - tx_receipts_mpt_handlers.insert(key.block_number, boxed_handler); + e.insert(boxed_handler); } } @@ -277,7 +277,7 @@ impl<'a> Fetcher<'a> { let mut tx_trie_handlers: HashMap> = HashMap::default(); for key in self.proof_keys.evm.transaction_keys.iter() { - if !tx_trie_handlers.contains_key(&key.block_number) { + if let std::collections::hash_map::Entry::Vacant(e) = tx_trie_handlers.entry(key.block_number) { let tx_trie_handler = TxsMptHandler::new(Url::parse(&env::var(RPC_URL_ETHEREUM).unwrap()).unwrap()) .map_err(|e| FetcherError::InternalError(e.to_string()))?; @@ -288,7 +288,7 @@ impl<'a> Fetcher<'a> { .await .map_err(|e| FetcherError::InternalError(e.to_string()))?; - tx_trie_handlers.insert(key.block_number, boxed_handler); + e.insert(boxed_handler); } } From 79e3ccb3d059e6c1b22062aadffdf253cf1ad613 Mon Sep 17 00:00:00 2001 From: codyx Date: Wed, 5 Feb 2025 18:49:25 +0400 Subject: [PATCH 3/6] Optimize future stream --- crates/fetcher/src/lib.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/crates/fetcher/src/lib.rs b/crates/fetcher/src/lib.rs index d2f77664..08aa15e2 100644 --- a/crates/fetcher/src/lib.rs +++ b/crates/fetcher/src/lib.rs @@ -158,10 +158,12 @@ impl<'a> Fetcher<'a> { flattened_keys .iter() .map(|key| EvmProofKeys::fetch_header_proof(key.chain_id, key.block_number)), - ); + ) + .buffer_unordered(BUFFER_UNORDERED) + .boxed(); while let Some(result) = header_fut.next().await { - let item = result.await?; + let item = result?; headers_with_mmr .entry(item.mmr_meta) .and_modify(|headers: &mut Vec| headers.extend(item.headers.clone())) From 5a5f081fb59c7fe50a2dd06c82b7d3a0c27f1eaf Mon Sep 17 00:00:00 2001 From: codyx Date: Wed, 5 Feb 2025 18:49:42 +0400 Subject: [PATCH 4/6] Add fetcher modules and tests --- tests/src/evm.cairo | 1 + tests/src/evm/fetcher_modules.cairo | 117 ++++++++++++++++++++++++++++ tests/src/evm/fetcher_modules.rs | 28 +++++++ tests/src/evm/mod.rs | 1 + 4 files changed, 147 insertions(+) create mode 100644 tests/src/evm/fetcher_modules.cairo create mode 100644 tests/src/evm/fetcher_modules.rs diff --git a/tests/src/evm.cairo b/tests/src/evm.cairo index 13743590..1b52970e 100644 --- a/tests/src/evm.cairo +++ b/tests/src/evm.cairo @@ -1,5 +1,6 @@ pub mod account_modules; pub mod header_modules; +pub mod fetcher_modules; pub mod storage_modules; pub mod receipt_modules; pub mod transaction_modules; diff --git a/tests/src/evm/fetcher_modules.cairo b/tests/src/evm/fetcher_modules.cairo new file mode 100644 index 00000000..1bca4bd2 --- /dev/null +++ b/tests/src/evm/fetcher_modules.cairo @@ -0,0 +1,117 @@ +#[starknet::contract] +mod evm_fetcher_many_keys_same_header { + use hdp_cairo::{ + HDP, evm::header::{HeaderTrait, HeaderKey, HeaderImpl}, + evm::storage::{StorageTrait, StorageKey, StorageImpl}, + evm::account::{AccountTrait, AccountKey, AccountImpl}, + evm::block_tx::{BlockTxTrait, BlockTxKey, BlockTxImpl}, + evm::block_receipt::{BlockReceiptTrait, BlockReceiptKey, BlockReceiptImpl}, + }; + + #[storage] + struct Storage {} + + #[external(v0)] + pub fn main(ref self: ContractState, hdp: HDP) { + // Header + hdp.evm.header_get_parent(HeaderKey { chain_id: 11155111, block_number: 6000000 }); + // Account + hdp + .evm + .account_get_nonce( + AccountKey { + chain_id: 11155111, + block_number: 6000000, + address: 0xc6e2459991BfE27cca6d86722F35da23A1E4Cb97, + }, + ); + // Storage + hdp + .evm + .storage_get_slot( + StorageKey { + chain_id: 11155111, + block_number: 6000000, + address: 0x75cec1db9dceb703200eaa6595f66885c962b920, + storage_slot: 0x1, + }, + ); + // Transaction + hdp + .evm + .block_tx_get_nonce( + BlockTxKey { chain_id: 11155111, block_number: 6000000, transaction_index: 0 }, + ); + // Receipt + hdp + .evm + .block_receipt_get_cumulative_gas_used( + BlockReceiptKey { chain_id: 11155111, block_number: 6000000, transaction_index: 1 }, + ); + } +} + +#[starknet::contract] +mod evm_fetcher_many_keys_same_header_10x { + use hdp_cairo::{ + HDP, evm::header::{HeaderTrait, HeaderKey, HeaderImpl}, + evm::storage::{StorageTrait, StorageKey, StorageImpl}, + evm::account::{AccountTrait, AccountKey, AccountImpl}, + }; + + #[storage] + struct Storage {} + + #[external(v0)] + pub fn main(ref self: ContractState, hdp: HDP) { + for i in 0..10_u256 { + let block_number: felt252 = (6000000_u256 + i).try_into().unwrap(); + // Header + hdp.evm.header_get_parent(HeaderKey { chain_id: 11155111, block_number }); + // Account + hdp + .evm + .account_get_nonce( + AccountKey { + chain_id: 11155111, + block_number, + address: 0xc6e2459991BfE27cca6d86722F35da23A1E4Cb97, + }, + ); + // Storage + hdp + .evm + .storage_get_slot( + StorageKey { + chain_id: 11155111, + block_number, + address: 0x75cec1db9dceb703200eaa6595f66885c962b920, + storage_slot: 0x1, + }, + ); + } + } +} + +#[starknet::contract] +mod evm_fetcher_many_txns_same_header { + use hdp_cairo::{HDP, evm::block_tx::{BlockTxTrait, BlockTxKey, BlockTxImpl}}; + + #[storage] + struct Storage {} + + #[external(v0)] + pub fn main(ref self: ContractState, hdp: HDP) { + let block_number: felt252 = 6000000; + for i in 0..10_u256 { + let transaction_index: felt252 = i.try_into().unwrap(); + // Transaction + hdp + .evm + .block_tx_get_nonce( + BlockTxKey { chain_id: 11155111, block_number, transaction_index }, + ); + } + } +} + diff --git a/tests/src/evm/fetcher_modules.rs b/tests/src/evm/fetcher_modules.rs new file mode 100644 index 00000000..9452efb4 --- /dev/null +++ b/tests/src/evm/fetcher_modules.rs @@ -0,0 +1,28 @@ +use crate::test_utils::run; + +#[tokio::test(flavor = "multi_thread", worker_threads = 1)] +async fn test_modules_evm_fetcher_many_keys_same_header() { + run(serde_json::from_slice(include_bytes!( + "../../../target/dev/modules_evm_fetcher_many_keys_same_header.compiled_contract_class.json" + )) + .unwrap()) + .await +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 1)] +async fn test_modules_evm_fetcher_many_keys_same_header_10x() { + run(serde_json::from_slice(include_bytes!( + "../../../target/dev/modules_evm_fetcher_many_keys_same_header_10x.compiled_contract_class.json" + )) + .unwrap()) + .await +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 1)] +async fn test_modules_evm_fetcher_many_txns_same_header() { + run(serde_json::from_slice(include_bytes!( + "../../../target/dev/modules_evm_fetcher_many_txns_same_header.compiled_contract_class.json" + )) + .unwrap()) + .await +} diff --git a/tests/src/evm/mod.rs b/tests/src/evm/mod.rs index 62a66c90..64edae54 100644 --- a/tests/src/evm/mod.rs +++ b/tests/src/evm/mod.rs @@ -1,4 +1,5 @@ pub mod account_modules; +pub mod fetcher_modules; pub mod header_modules; pub mod receipt_modules; pub mod storage_modules; From 86c1d8ec0557ebb140943c36d342af3ac9f418c2 Mon Sep 17 00:00:00 2001 From: codyx Date: Wed, 5 Feb 2025 18:54:46 +0400 Subject: [PATCH 5/6] chore: Minor formatting and whitespace cleanup --- hdp_cairo/src/debug.cairo | 2 +- hdp_cairo/src/debug/printer.cairo | 8 ++------ tests/src/lib.cairo | 2 +- tests/src/utils.cairo | 2 +- tests/src/utils/debug.cairo | 10 ++++++---- 5 files changed, 11 insertions(+), 13 deletions(-) diff --git a/hdp_cairo/src/debug.cairo b/hdp_cairo/src/debug.cairo index d414462a..32d48df1 100644 --- a/hdp_cairo/src/debug.cairo +++ b/hdp_cairo/src/debug.cairo @@ -1,2 +1,2 @@ mod printer; -pub use printer::{print, print_array}; \ No newline at end of file +pub use printer::{print, print_array}; diff --git a/hdp_cairo/src/debug/printer.cairo b/hdp_cairo/src/debug/printer.cairo index 3af6cbad..c022ba0f 100644 --- a/hdp_cairo/src/debug/printer.cairo +++ b/hdp_cairo/src/debug/printer.cairo @@ -16,10 +16,6 @@ pub fn print, +Drop>(value: T) { } pub fn print_array(array: Array) { - call_contract_syscall( - DEBUG_CONTRACT_ADDRESS.try_into().unwrap(), - PRINT_ARRAY, - array.span() - ) + call_contract_syscall(DEBUG_CONTRACT_ADDRESS.try_into().unwrap(), PRINT_ARRAY, array.span()) .unwrap_syscall(); -} \ No newline at end of file +} diff --git a/tests/src/lib.cairo b/tests/src/lib.cairo index c1762f74..99fa67e9 100644 --- a/tests/src/lib.cairo +++ b/tests/src/lib.cairo @@ -1,3 +1,3 @@ pub mod evm; pub mod starknet; -pub mod utils; \ No newline at end of file +pub mod utils; diff --git a/tests/src/utils.cairo b/tests/src/utils.cairo index 477cc78a..2f365233 100644 --- a/tests/src/utils.cairo +++ b/tests/src/utils.cairo @@ -1 +1 @@ -pub mod debug; \ No newline at end of file +pub mod debug; diff --git a/tests/src/utils/debug.cairo b/tests/src/utils/debug.cairo index c482f1dd..b53c3dcd 100644 --- a/tests/src/utils/debug.cairo +++ b/tests/src/utils/debug.cairo @@ -15,7 +15,11 @@ mod test_debug_print { impl PointDisplay of Display { fn fmt(self: @Point, ref f: Formatter) -> Result<(), Error> { - let str: ByteArray = format!("PointThatIAmMakingQuiteABitLongerToEnsureWeHaveMoreFelts ({}, {})", *self.x, *self.y); + let str: ByteArray = format!( + "PointThatIAmMakingQuiteABitLongerToEnsureWeHaveMoreFelts ({}, {})", + *self.x, + *self.y, + ); f.buffer.append(@str); Result::Ok(()) } @@ -24,11 +28,9 @@ mod test_debug_print { #[external(v0)] pub fn main(ref self: ContractState, hdp: HDP) { let p = Point { x: 1, y: 3 }; - + print(p); print(1); print_array(array![1, 2, 3]); - - } } From fd69cf1f816b752e7f3e38ab0bf3cfde5ac8e297 Mon Sep 17 00:00:00 2001 From: Bartosz Nowak Date: Thu, 6 Feb 2025 08:34:27 +0100 Subject: [PATCH 6/6] simplification and opts of receipts and transactions fetching --- crates/fetcher/src/lib.rs | 95 +++++++++------------------- crates/fetcher/src/proof_keys/evm.rs | 17 ++--- 2 files changed, 36 insertions(+), 76 deletions(-) diff --git a/crates/fetcher/src/lib.rs b/crates/fetcher/src/lib.rs index 08aa15e2..7aca4873 100644 --- a/crates/fetcher/src/lib.rs +++ b/crates/fetcher/src/lib.rs @@ -2,7 +2,6 @@ use std::{ collections::{HashMap, HashSet}, env, num::ParseIntError, - sync::Arc, }; use alloy::hex::FromHexError; @@ -19,7 +18,6 @@ use proof_keys::{ use reqwest::Url; use starknet_types_core::felt::FromStrError; use thiserror::Error; -use tokio::sync::Mutex; use types::{ proofs::{ evm::{ @@ -226,100 +224,67 @@ impl<'a> Fetcher<'a> { .evm_storage .safe_finish_with_message("evm storage keys - finished"); - // For each receipt block, we need to create a tx_receipts_mpt_handler - let mut tx_receipts_mpt_handlers: HashMap> = HashMap::default(); + // For each block, we need to create a mpt_handler + let mut receipt_mpt_handlers: HashMap = HashMap::default(); - for key in self.proof_keys.evm.receipt_keys.iter() { - if let std::collections::hash_map::Entry::Vacant(e) = tx_receipts_mpt_handlers.entry(key.block_number) { - let tx_receipts_mpt_handler = TxReceiptsMptHandler::new(Url::parse(&env::var(RPC_URL_ETHEREUM).unwrap()).unwrap()) + for block_number in self.proof_keys.evm.receipt_keys.iter().map(|key| key.block_number) { + if let std::collections::hash_map::Entry::Vacant(entry) = receipt_mpt_handlers.entry(block_number) { + let mut mpt_handler = TxReceiptsMptHandler::new(Url::parse(&env::var(RPC_URL_ETHEREUM).unwrap()).unwrap()) .map_err(|e| FetcherError::InternalError(e.to_string()))?; - let mut boxed_handler = Box::new(tx_receipts_mpt_handler); - - // Build the receipt root for the block - boxed_handler - .build_tx_receipts_tree_from_block(key.block_number) + mpt_handler + .build_tx_receipts_tree_from_block(block_number) .await .map_err(|e| FetcherError::InternalError(e.to_string()))?; - e.insert(boxed_handler); - } - } - - // Convert HashMap to Arc for sharing - let tx_receipts_mpt_handlers = Arc::new(Mutex::new(tx_receipts_mpt_handlers)); - let tx_receipts_mpt_handlers_clone = tx_receipts_mpt_handlers.clone(); - - let mut transaction_receipts_fut = futures::stream::iter(self.proof_keys.evm.receipt_keys.iter().map({ - move |key| { - let handlers = tx_receipts_mpt_handlers_clone.clone(); - async move { - let mut handlers_guard = handlers.lock().await; - let handler = handlers_guard.get_mut(&key.block_number).unwrap(); - - EvmProofKeys::fetch_receipt_proof(key, handler.as_mut()).await - } + entry.insert(mpt_handler); } - })) - .buffer_unordered(BUFFER_UNORDERED) - .boxed(); - - while let Some(Ok(transaction_receipt)) = transaction_receipts_fut.next().await { - receipts.insert(transaction_receipt); #[cfg(feature = "progress_bars")] self.progress_bars.evm_receipts.safe_inc(); } + receipts.extend( + self.proof_keys + .evm + .receipt_keys + .iter() + .map(|key| EvmProofKeys::compute_receipt_proof(key, receipt_mpt_handlers.get_mut(&key.block_number).unwrap()).unwrap()), + ); + #[cfg(feature = "progress_bars")] self.progress_bars .evm_receipts .safe_finish_with_message("evm receipt keys - finished"); - let mut tx_trie_handlers: HashMap> = HashMap::default(); + // For each tx block, we need to create a mpt_handler + let mut tx_mpt_handlers: HashMap = HashMap::default(); for key in self.proof_keys.evm.transaction_keys.iter() { - if let std::collections::hash_map::Entry::Vacant(e) = tx_trie_handlers.entry(key.block_number) { - let tx_trie_handler = TxsMptHandler::new(Url::parse(&env::var(RPC_URL_ETHEREUM).unwrap()).unwrap()) + if let std::collections::hash_map::Entry::Vacant(entry) = tx_mpt_handlers.entry(key.block_number) { + let mut mpt_handler = TxsMptHandler::new(Url::parse(&env::var(RPC_URL_ETHEREUM).unwrap()).unwrap()) .map_err(|e| FetcherError::InternalError(e.to_string()))?; - let mut boxed_handler = Box::new(tx_trie_handler); - - boxed_handler + mpt_handler .build_tx_tree_from_block(key.block_number) .await .map_err(|e| FetcherError::InternalError(e.to_string()))?; - e.insert(boxed_handler); - } - } - - // Convert HashMap to Arc for sharing - let tx_trie_handlers = Arc::new(Mutex::new(tx_trie_handlers)); - let tx_trie_handlers_clone = tx_trie_handlers.clone(); - - // Collect transaction proofs - let mut transaction_keys_fut = futures::stream::iter(self.proof_keys.evm.transaction_keys.iter().map({ - move |key| { - let handlers = tx_trie_handlers_clone.clone(); - async move { - let mut handlers_guard = handlers.lock().await; - let handler = handlers_guard.get_mut(&key.block_number).unwrap(); - - EvmProofKeys::fetch_transaction_proof(key, handler.as_mut()).await - } + entry.insert(mpt_handler); } - })) - .buffer_unordered(BUFFER_UNORDERED) - .boxed(); - - while let Some(Ok(transaction)) = transaction_keys_fut.next().await { - transactions.insert(transaction); #[cfg(feature = "progress_bars")] self.progress_bars.evm_transactions.safe_inc(); } + transactions.extend( + self.proof_keys + .evm + .transaction_keys + .iter() + .map(|key| EvmProofKeys::compute_transaction_proof(key, tx_mpt_handlers.get_mut(&key.block_number).unwrap()).unwrap()), + ); + #[cfg(feature = "progress_bars")] self.progress_bars .evm_transactions diff --git a/crates/fetcher/src/proof_keys/evm.rs b/crates/fetcher/src/proof_keys/evm.rs index 9d03719d..170b81b5 100644 --- a/crates/fetcher/src/proof_keys/evm.rs +++ b/crates/fetcher/src/proof_keys/evm.rs @@ -109,7 +109,7 @@ impl ProofKeys { )) } - async fn generate_block_tx_receipt_proof( + fn generate_block_tx_receipt_proof( tx_receipts_mpt_handler: &mut TxReceiptsMptHandler, block_number: u64, tx_index: u64, @@ -126,22 +126,17 @@ impl ProofKeys { Ok(MPTProof { block_number, proof }) } - pub async fn fetch_receipt_proof( + pub fn compute_receipt_proof( key: &keys::evm::receipt::Key, tx_receipts_mpt_handler: &mut TxReceiptsMptHandler, ) -> Result { - let receipt_mpt_proof = - Self::generate_block_tx_receipt_proof(tx_receipts_mpt_handler, key.block_number, key.transaction_index).await?; + let receipt_mpt_proof = Self::generate_block_tx_receipt_proof(tx_receipts_mpt_handler, key.block_number, key.transaction_index)?; let rlp_encoded_key = alloy_rlp::encode(U256::from(key.transaction_index)); Ok(Receipt::new(U256::from_be_slice(&rlp_encoded_key), receipt_mpt_proof)) } - async fn generate_block_tx_proof( - tx_trie_handler: &mut TxsMptHandler, - block_number: u64, - tx_index: u64, - ) -> Result { + fn generate_block_tx_proof(tx_trie_handler: &mut TxsMptHandler, block_number: u64, tx_index: u64) -> Result { let trie_proof = tx_trie_handler .get_proof(tx_index) .map_err(|e| FetcherError::InternalError(e.to_string()))?; @@ -154,11 +149,11 @@ impl ProofKeys { Ok(MPTProof { block_number, proof }) } - pub async fn fetch_transaction_proof( + pub fn compute_transaction_proof( key: &keys::evm::transaction::Key, tx_trie_handler: &mut TxsMptHandler, ) -> Result { - let tx_proof = Self::generate_block_tx_proof(tx_trie_handler, key.block_number, key.transaction_index).await?; + let tx_proof = Self::generate_block_tx_proof(tx_trie_handler, key.block_number, key.transaction_index)?; let rlp_encoded_key = alloy_rlp::encode(U256::from(key.transaction_index)); Ok(Transaction::new(U256::from_be_slice(&rlp_encoded_key), tx_proof))