From 8fd2b68acfc79ff51835ff67c4dcf749e15913c9 Mon Sep 17 00:00:00 2001 From: DaniPopes <57450786+DaniPopes@users.noreply.github.com> Date: Wed, 1 Oct 2025 15:56:11 +0200 Subject: [PATCH 1/5] wip --- Cargo.lock | 1 + crates/evm/traces/Cargo.toml | 13 +- crates/evm/traces/src/identifier/mod.rs | 3 + crates/evm/traces/src/identifier/sourcify.rs | 229 +++++++++++++++++++ 4 files changed, 240 insertions(+), 6 deletions(-) create mode 100644 crates/evm/traces/src/identifier/sourcify.rs diff --git a/Cargo.lock b/Cargo.lock index fb9157a4d6025..b7101a8b516f5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4827,6 +4827,7 @@ dependencies = [ "itertools 0.14.0", "memchr", "rayon", + "reqwest", "revm", "revm-inspectors", "serde", diff --git a/crates/evm/traces/Cargo.toml b/crates/evm/traces/Cargo.toml index 527ea422bbd07..4223e73e8037c 100644 --- a/crates/evm/traces/Cargo.toml +++ b/crates/evm/traces/Cargo.toml @@ -35,16 +35,17 @@ revm-inspectors.workspace = true eyre.workspace = true futures.workspace = true itertools.workspace = true -serde.workspace = true +memchr.workspace = true +rayon.workspace = true +reqwest.workspace = true +revm.workspace = true serde_json.workspace = true +serde.workspace = true +solar.workspace = true +tempfile.workspace = true tokio = { workspace = true, features = ["time", "macros"] } tracing.workspace = true -tempfile.workspace = true -rayon.workspace = true -solar.workspace = true -revm.workspace = true yansi.workspace = true -memchr.workspace = true [dev-dependencies] tempfile.workspace = true diff --git a/crates/evm/traces/src/identifier/mod.rs b/crates/evm/traces/src/identifier/mod.rs index 8ad929b7317a0..82a756fcea4ec 100644 --- a/crates/evm/traces/src/identifier/mod.rs +++ b/crates/evm/traces/src/identifier/mod.rs @@ -12,6 +12,9 @@ pub use local::LocalTraceIdentifier; mod etherscan; pub use etherscan::EtherscanIdentifier; +mod sourcify; +pub use sourcify::SourcifyIdentifier; + mod signatures; pub use signatures::{SignaturesCache, SignaturesIdentifier}; diff --git a/crates/evm/traces/src/identifier/sourcify.rs b/crates/evm/traces/src/identifier/sourcify.rs new file mode 100644 index 0000000000000..c43f9e3f38759 --- /dev/null +++ b/crates/evm/traces/src/identifier/sourcify.rs @@ -0,0 +1,229 @@ +use super::{IdentifiedAddress, TraceIdentifier}; +use crate::debug::ContractSources; +use alloy_json_abi::JsonAbi; +use alloy_primitives::Address; +use foundry_common::compile::etherscan_project; +use foundry_config::{Chain, Config}; +use futures::{ + future::join_all, + stream::{FuturesUnordered, Stream, StreamExt}, + task::{Context, Poll}, +}; +use reqwest::StatusCode; +use revm_inspectors::tracing::types::CallTraceNode; +use serde::Deserialize; +use std::{borrow::Cow, collections::BTreeMap, pin::Pin, sync::atomic::Ordering}; +use tokio::time::{Duration, Interval}; + +#[derive(Debug, Clone, Deserialize)] +#[serde(untagged)] +enum SourcifyResponse { + Success(Metadata), + Error(SourcifyErrorResponse), +} + +#[derive(Debug, Clone, Deserialize)] +#[serde(rename_all = "camelCase")] +struct SourcifyErrorResponse { + custom_code: String, + message: String, + error_id: String, +} + +#[derive(Debug, Clone, Deserialize)] +#[serde(rename_all = "camelCase")] +struct Metadata { + #[serde(default)] + abi: Option, + #[serde(default)] + compilation: Option, +} + +#[derive(Debug, Clone, Deserialize)] +#[serde(rename_all = "camelCase")] +struct Compilation { + #[serde(default)] + language: String, + #[serde(default)] + name: String, +} + +/// A trace identifier that tries to identify addresses using Etherscan. +pub struct SourcifyIdentifier { + client: reqwest::Client, + url: String, + contracts: BTreeMap, +} + +impl SourcifyIdentifier { + /// Creates a new Etherscan identifier with the given client + pub fn new(config: &Config, chain: Option) -> eyre::Result> { + if config.offline { + return Ok(None); + } + Ok(Some(Self { + client: reqwest::Client::new(), + url: format!( + "https://sourcify.dev/server/v2/contract/{}", + chain.unwrap_or_default().id(), + ), + contracts: BTreeMap::new(), + })) + } + + fn identify_from_metadata( + &self, + address: Address, + metadata: &Metadata, + ) -> IdentifiedAddress<'static> { + let label = metadata.compilation.as_ref().map(|c| c.name.clone()); + let abi = metadata.abi.clone().map(Cow::Owned); + IdentifiedAddress { address, label: label.clone(), contract: label, abi, artifact_id: None } + } +} + +impl TraceIdentifier for SourcifyIdentifier { + fn identify_addresses(&mut self, nodes: &[&CallTraceNode]) -> Vec> { + if nodes.is_empty() { + return Vec::new(); + } + + trace!(target: "evm::traces::etherscan", "identify {} addresses", nodes.len()); + + let mut identities = Vec::new(); + let mut fetcher = + SourcifyFetcher::new(self.client.clone(), self.url.clone(), Duration::from_secs(1), 5); + + for &node in nodes { + let address = node.trace.address; + if let Some(metadata) = self.contracts.get(&address) { + identities.push(self.identify_from_metadata(address, metadata)); + } else { + fetcher.push(address); + } + } + + let fetched_identities = foundry_common::block_on( + fetcher + .map(|(address, metadata)| { + let addr = self.identify_from_metadata(address, &metadata); + self.contracts.insert(address, metadata); + addr + }) + .collect::>>(), + ); + + identities.extend(fetched_identities); + identities + } +} + +type SourcifyFuture = Pin< + Box)>>, +>; + +/// A rate limit aware Sourcify client. +/// +/// Fetches information about multiple addresses concurrently, while respecting rate limits. +struct SourcifyFetcher { + /// The client + client: reqwest::Client, + /// The URL to fetch the contract metadata from + url: String, + /// The time we wait if we hit the rate limit + timeout: Duration, + /// The interval we are currently waiting for before making a new request + backoff: Option, + /// The maximum amount of requests to send concurrently + concurrency: usize, + /// The addresses we have yet to make requests for + queue: Vec
, + /// The in progress requests + in_progress: FuturesUnordered, +} + +impl SourcifyFetcher { + fn new(client: reqwest::Client, url: String, timeout: Duration, concurrency: usize) -> Self { + Self { + client, + url, + timeout, + backoff: None, + concurrency, + queue: Vec::new(), + in_progress: FuturesUnordered::new(), + } + } + + fn push(&mut self, address: Address) { + self.queue.push(address); + } + + fn queue_next_reqs(&mut self) { + while self.in_progress.len() < self.concurrency { + let Some(addr) = self.queue.pop() else { break }; + let client = self.client.clone(); + let url = self.url.clone(); + self.in_progress.push(Box::pin(async move { + trace!(target: "traces::etherscan", ?addr, "fetching info"); + let res = client.get(format!("{url}/{addr}?fields=abi")).send().await; + let res = match res { + Ok(res) => { + let code = res.status(); + res.json().await.map(|res| (code, res)) + } + Err(e) => Err(e), + }; + (addr, res) + })); + } + } +} + +impl Stream for SourcifyFetcher { + type Item = (Address, Metadata); + + fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + let pin = self.get_mut(); + + loop { + if let Some(mut backoff) = pin.backoff.take() + && backoff.poll_tick(cx).is_pending() + { + pin.backoff = Some(backoff); + return Poll::Pending; + } + + pin.queue_next_reqs(); + + let mut made_progress_this_iter = false; + match pin.in_progress.poll_next_unpin(cx) { + Poll::Pending => {} + Poll::Ready(None) => return Poll::Ready(None), + Poll::Ready(Some((addr, res))) => { + made_progress_this_iter = true; + let (code, res) = match res { + Ok(r) => r, + Err(err) => { + warn!(target: "traces::etherscan", ?err, "could not get sourcify info"); + // continue; + } + }; + if code.as_u16() == 429 { + warn!(target: "traces::sourcify", ?res, "rate limit exceeded on attempt"); + pin.backoff = Some(tokio::time::interval(pin.timeout)); + pin.queue.push(addr); + // continue; + } + if let SourcifyResponse::Success(res) = res { + return Poll::Ready(Some((addr, res))); + } + } + } + + if !made_progress_this_iter { + return Poll::Pending; + } + } + } +} From 1d6a2e08d25906fe594141bb0b14cd4f28d05499 Mon Sep 17 00:00:00 2001 From: DaniPopes <57450786+DaniPopes@users.noreply.github.com> Date: Wed, 1 Oct 2025 21:07:52 +0200 Subject: [PATCH 2/5] unify --- Cargo.lock | 1 + crates/cast/src/debug.rs | 4 +- crates/chisel/src/dispatcher.rs | 2 +- crates/evm/traces/Cargo.toml | 3 +- crates/evm/traces/src/identifier/etherscan.rs | 278 ------------ crates/evm/traces/src/identifier/external.rs | 401 ++++++++++++++++++ crates/evm/traces/src/identifier/mod.rs | 25 +- crates/evm/traces/src/identifier/sourcify.rs | 229 ---------- crates/forge/src/cmd/test/mod.rs | 2 +- crates/script/src/execute.rs | 2 +- 10 files changed, 420 insertions(+), 527 deletions(-) delete mode 100644 crates/evm/traces/src/identifier/etherscan.rs create mode 100644 crates/evm/traces/src/identifier/external.rs delete mode 100644 crates/evm/traces/src/identifier/sourcify.rs diff --git a/Cargo.lock b/Cargo.lock index b7101a8b516f5..22b51194222c8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4816,6 +4816,7 @@ dependencies = [ "alloy-json-abi", "alloy-primitives", "alloy-sol-types", + "async-trait", "eyre", "foundry-block-explorers", "foundry-common", diff --git a/crates/cast/src/debug.rs b/crates/cast/src/debug.rs index fa8fa7a1be636..e8d3fbf2da042 100644 --- a/crates/cast/src/debug.rs +++ b/crates/cast/src/debug.rs @@ -56,7 +56,7 @@ pub(crate) async fn handle_traces( .with_labels(labels.chain(config_labels)) .with_signature_identifier(SignaturesIdentifier::from_config(config)?) .with_label_disabled(disable_label); - let mut identifier = TraceIdentifiers::new().with_etherscan(config, chain)?; + let mut identifier = TraceIdentifiers::new().with_external(config, chain)?; if let Some(contracts) = &known_contracts { builder = builder.with_known_contracts(contracts); identifier = identifier.with_local_and_bytecodes(contracts, contracts_bytecode); @@ -69,7 +69,7 @@ pub(crate) async fn handle_traces( } if decode_internal || debug { - if let Some(ref etherscan_identifier) = identifier.etherscan { + if let Some(ref etherscan_identifier) = identifier.external { sources.merge(etherscan_identifier.get_compiled_contracts().await?); } diff --git a/crates/chisel/src/dispatcher.rs b/crates/chisel/src/dispatcher.rs index a7fbcd335e9fd..0a99825c0b03a 100644 --- a/crates/chisel/src/dispatcher.rs +++ b/crates/chisel/src/dispatcher.rs @@ -181,7 +181,7 @@ impl ChiselDispatcher { )?) .build(); - let mut identifier = TraceIdentifiers::new().with_etherscan( + let mut identifier = TraceIdentifiers::new().with_external( &session_config.foundry_config, session_config.evm_opts.get_remote_chain_id().await, )?; diff --git a/crates/evm/traces/Cargo.toml b/crates/evm/traces/Cargo.toml index 4223e73e8037c..4eed2fcf4f10b 100644 --- a/crates/evm/traces/Cargo.toml +++ b/crates/evm/traces/Cargo.toml @@ -32,6 +32,7 @@ alloy-primitives = { workspace = true, features = [ alloy-sol-types.workspace = true revm-inspectors.workspace = true +async-trait.workspace = true eyre.workspace = true futures.workspace = true itertools.workspace = true @@ -39,7 +40,7 @@ memchr.workspace = true rayon.workspace = true reqwest.workspace = true revm.workspace = true -serde_json.workspace = true +serde_json = { workspace = true, features = ["raw_value"] } serde.workspace = true solar.workspace = true tempfile.workspace = true diff --git a/crates/evm/traces/src/identifier/etherscan.rs b/crates/evm/traces/src/identifier/etherscan.rs deleted file mode 100644 index fa0f8c03dcf77..0000000000000 --- a/crates/evm/traces/src/identifier/etherscan.rs +++ /dev/null @@ -1,278 +0,0 @@ -use super::{IdentifiedAddress, TraceIdentifier}; -use crate::debug::ContractSources; -use alloy_primitives::Address; -use foundry_block_explorers::{ - contract::{ContractMetadata, Metadata}, - errors::EtherscanError, -}; -use foundry_common::compile::etherscan_project; -use foundry_config::{Chain, Config}; -use futures::{ - future::join_all, - stream::{FuturesUnordered, Stream, StreamExt}, - task::{Context, Poll}, -}; -use revm_inspectors::tracing::types::CallTraceNode; -use std::{ - borrow::Cow, - collections::BTreeMap, - pin::Pin, - sync::{ - Arc, - atomic::{AtomicBool, Ordering}, - }, -}; -use tokio::time::{Duration, Interval}; - -/// A trace identifier that tries to identify addresses using Etherscan. -pub struct EtherscanIdentifier { - /// The Etherscan client - client: Arc, - /// Tracks whether the API key provides was marked as invalid - /// - /// After the first [EtherscanError::InvalidApiKey] this will get set to true, so we can - /// prevent any further attempts - invalid_api_key: Arc, - pub contracts: BTreeMap, - pub sources: BTreeMap, -} - -impl EtherscanIdentifier { - /// Creates a new Etherscan identifier with the given client - pub fn new(config: &Config, chain: Option) -> eyre::Result> { - // In offline mode, don't use Etherscan. - if config.offline { - return Ok(None); - } - - let config = match config.get_etherscan_config_with_chain(chain) { - Ok(Some(config)) => config, - Ok(None) => { - warn!(target: "traces::etherscan", "etherscan config not found"); - return Ok(None); - } - Err(err) => { - warn!(?err, "failed to get etherscan config"); - return Ok(None); - } - }; - - trace!(target: "traces::etherscan", chain=?config.chain, url=?config.api_url, "using etherscan identifier"); - Ok(Some(Self { - client: Arc::new(config.into_client()?), - invalid_api_key: Arc::new(AtomicBool::new(false)), - contracts: BTreeMap::new(), - sources: BTreeMap::new(), - })) - } - - /// Goes over the list of contracts we have pulled from the traces, clones their source from - /// Etherscan and compiles them locally, for usage in the debugger. - pub async fn get_compiled_contracts(&self) -> eyre::Result { - let outputs_fut = self - .contracts - .iter() - // filter out vyper files - .filter(|(_, metadata)| !metadata.is_vyper()) - .map(|(address, metadata)| async move { - sh_println!("Compiling: {} {address}", metadata.contract_name)?; - let root = tempfile::tempdir()?; - let root_path = root.path(); - let project = etherscan_project(metadata, root_path)?; - let output = project.compile()?; - - if output.has_compiler_errors() { - eyre::bail!("{output}") - } - - Ok((project, output, root)) - }) - .collect::>(); - - // poll all the futures concurrently - let outputs = join_all(outputs_fut).await; - - let mut sources: ContractSources = Default::default(); - - // construct the map - for res in outputs { - let (project, output, _root) = res?; - sources.insert(&output, project.root(), None)?; - } - - Ok(sources) - } - - fn identify_from_metadata( - &self, - address: Address, - metadata: &Metadata, - ) -> IdentifiedAddress<'static> { - let label = metadata.contract_name.clone(); - let abi = metadata.abi().ok().map(Cow::Owned); - IdentifiedAddress { - address, - label: Some(label.clone()), - contract: Some(label), - abi, - artifact_id: None, - } - } -} - -impl TraceIdentifier for EtherscanIdentifier { - fn identify_addresses(&mut self, nodes: &[&CallTraceNode]) -> Vec> { - if self.invalid_api_key.load(Ordering::Relaxed) || nodes.is_empty() { - return Vec::new(); - } - - trace!(target: "evm::traces::etherscan", "identify {} addresses", nodes.len()); - - let mut identities = Vec::new(); - let mut fetcher = EtherscanFetcher::new( - self.client.clone(), - Duration::from_secs(1), - 5, - Arc::clone(&self.invalid_api_key), - ); - - for &node in nodes { - let address = node.trace.address; - if let Some(metadata) = self.contracts.get(&address) { - identities.push(self.identify_from_metadata(address, metadata)); - } else { - fetcher.push(address); - } - } - - let fetched_identities = foundry_common::block_on( - fetcher - .map(|(address, metadata)| { - let addr = self.identify_from_metadata(address, &metadata); - self.contracts.insert(address, metadata); - addr - }) - .collect::>>(), - ); - - identities.extend(fetched_identities); - identities - } -} - -type EtherscanFuture = - Pin)>>>; - -/// A rate limit aware Etherscan client. -/// -/// Fetches information about multiple addresses concurrently, while respecting rate limits. -struct EtherscanFetcher { - /// The Etherscan client - client: Arc, - /// The time we wait if we hit the rate limit - timeout: Duration, - /// The interval we are currently waiting for before making a new request - backoff: Option, - /// The maximum amount of requests to send concurrently - concurrency: usize, - /// The addresses we have yet to make requests for - queue: Vec
, - /// The in progress requests - in_progress: FuturesUnordered, - /// tracks whether the API key provides was marked as invalid - invalid_api_key: Arc, -} - -impl EtherscanFetcher { - fn new( - client: Arc, - timeout: Duration, - concurrency: usize, - invalid_api_key: Arc, - ) -> Self { - Self { - client, - timeout, - backoff: None, - concurrency, - queue: Vec::new(), - in_progress: FuturesUnordered::new(), - invalid_api_key, - } - } - - fn push(&mut self, address: Address) { - self.queue.push(address); - } - - fn queue_next_reqs(&mut self) { - while self.in_progress.len() < self.concurrency { - let Some(addr) = self.queue.pop() else { break }; - let client = Arc::clone(&self.client); - self.in_progress.push(Box::pin(async move { - trace!(target: "traces::etherscan", ?addr, "fetching info"); - let res = client.contract_source_code(addr).await; - (addr, res) - })); - } - } -} - -impl Stream for EtherscanFetcher { - type Item = (Address, Metadata); - - fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - let pin = self.get_mut(); - - loop { - if let Some(mut backoff) = pin.backoff.take() - && backoff.poll_tick(cx).is_pending() - { - pin.backoff = Some(backoff); - return Poll::Pending; - } - - pin.queue_next_reqs(); - - let mut made_progress_this_iter = false; - match pin.in_progress.poll_next_unpin(cx) { - Poll::Pending => {} - Poll::Ready(None) => return Poll::Ready(None), - Poll::Ready(Some((addr, res))) => { - made_progress_this_iter = true; - match res { - Ok(mut metadata) => { - if let Some(item) = metadata.items.pop() { - return Poll::Ready(Some((addr, item))); - } - } - Err(EtherscanError::RateLimitExceeded) => { - warn!(target: "traces::etherscan", "rate limit exceeded on attempt"); - pin.backoff = Some(tokio::time::interval(pin.timeout)); - pin.queue.push(addr); - } - Err(EtherscanError::InvalidApiKey) => { - warn!(target: "traces::etherscan", "invalid api key"); - // mark key as invalid - pin.invalid_api_key.store(true, Ordering::Relaxed); - return Poll::Ready(None); - } - Err(EtherscanError::BlockedByCloudflare) => { - warn!(target: "traces::etherscan", "blocked by cloudflare"); - // mark key as invalid - pin.invalid_api_key.store(true, Ordering::Relaxed); - return Poll::Ready(None); - } - Err(err) => { - warn!(target: "traces::etherscan", "could not get etherscan info: {:?}", err); - } - } - } - } - - if !made_progress_this_iter { - return Poll::Pending; - } - } - } -} diff --git a/crates/evm/traces/src/identifier/external.rs b/crates/evm/traces/src/identifier/external.rs new file mode 100644 index 0000000000000..e7e3315c340d3 --- /dev/null +++ b/crates/evm/traces/src/identifier/external.rs @@ -0,0 +1,401 @@ +use super::{IdentifiedAddress, TraceIdentifier}; +use crate::debug::ContractSources; +use alloy_primitives::{Address, map::HashMap}; +use foundry_block_explorers::{contract::Metadata, errors::EtherscanError}; +use foundry_common::compile::etherscan_project; +use foundry_config::{Chain, Config}; +use futures::{ + future::join_all, + stream::{FuturesUnordered, Stream, StreamExt}, + task::{Context, Poll}, +}; +use revm_inspectors::tracing::types::CallTraceNode; +use serde::Deserialize; +use std::{ + borrow::Cow, + pin::Pin, + sync::{ + Arc, + atomic::{AtomicBool, Ordering}, + }, +}; +use tokio::time::{Duration, Interval}; + +/// A trace identifier that tries to identify addresses using Etherscan. +pub struct ExternalIdentifier { + fetchers: Vec>, + /// Cached contracts. + contracts: HashMap, +} + +impl ExternalIdentifier { + /// Creates a new external identifier with the given client + pub fn new(config: &Config, mut chain: Option) -> eyre::Result> { + if config.offline { + return Ok(None); + } + + let config = match config.get_etherscan_config_with_chain(chain) { + Ok(Some(config)) => { + trace!(target: "traces::external", chain=?config.chain, url=?config.api_url, "using etherscan identifier"); + chain = config.chain; + Some(config) + } + Ok(None) => { + warn!(target: "traces::external", "etherscan config not found"); + None + } + Err(err) => { + warn!(target: "traces::external", ?err, "failed to get etherscan config"); + None + } + }; + + let mut fetchers = Vec::>::new(); + fetchers.push(Arc::new(SourcifyFetcher::new(chain.unwrap_or_default()))); + if let Some(config) = config { + fetchers.push(Arc::new(EtherscanFetcher::new(config.into_client()?))); + } + + Ok(Some(Self { fetchers, contracts: Default::default() })) + } + + /// Goes over the list of contracts we have pulled from the traces, clones their source from + /// Etherscan and compiles them locally, for usage in the debugger. + pub async fn get_compiled_contracts(&self) -> eyre::Result { + let outputs_fut = self + .contracts + .iter() + // filter out vyper files + .filter(|(_, metadata)| !metadata.is_vyper()) + .map(|(address, metadata)| async move { + sh_println!("Compiling: {} {address}", metadata.contract_name)?; + let root = tempfile::tempdir()?; + let root_path = root.path(); + let project = etherscan_project(metadata, root_path)?; + let output = project.compile()?; + + if output.has_compiler_errors() { + eyre::bail!("{output}") + } + + Ok((project, output, root)) + }) + .collect::>(); + + // poll all the futures concurrently + let outputs = join_all(outputs_fut).await; + + let mut sources: ContractSources = Default::default(); + + // construct the map + for res in outputs { + let (project, output, _root) = res?; + sources.insert(&output, project.root(), None)?; + } + + Ok(sources) + } + + fn identify_from_metadata( + &self, + address: Address, + metadata: &Metadata, + ) -> IdentifiedAddress<'static> { + let label = metadata.contract_name.clone(); + let abi = metadata.abi().ok().map(Cow::Owned); + IdentifiedAddress { + address, + label: Some(label.clone()), + contract: Some(label), + abi, + artifact_id: None, + } + } +} + +impl TraceIdentifier for ExternalIdentifier { + fn identify_addresses(&mut self, nodes: &[&CallTraceNode]) -> Vec> { + if nodes.is_empty() { + return Vec::new(); + } + + trace!(target: "evm::traces::external", "identify {} addresses", nodes.len()); + + let mut identities = Vec::new(); + let mut to_fetch = Vec::new(); + + // Check cache first. + for &node in nodes { + let address = node.trace.address; + if let Some(metadata) = self.contracts.get(&address) { + identities.push(self.identify_from_metadata(address, metadata)); + } else { + to_fetch.push(address); + } + } + + if to_fetch.is_empty() { + return identities; + } + + let fetchers = self + .fetchers + .iter() + .map(|fetcher| ExternalFetcher::new(fetcher.clone(), Duration::from_secs(1), 5)); + let fetched_identities = foundry_common::block_on( + futures::stream::select_all(fetchers) + .map(|(address, metadata)| { + let addr = self.identify_from_metadata(address, &metadata); + self.contracts.insert(address, metadata); + addr + }) + .collect::>>(), + ); + + identities.extend(fetched_identities); + identities + } +} + +type FetchFuture = + Pin, EtherscanError>)>>>; + +/// A rate limit aware fetcher. +/// +/// Fetches information about multiple addresses concurrently, while respecting rate limits. +struct ExternalFetcher { + /// The fetcher + fetcher: Arc, + /// The time we wait if we hit the rate limit + timeout: Duration, + /// The interval we are currently waiting for before making a new request + backoff: Option, + /// The maximum amount of requests to send concurrently + concurrency: usize, + /// The addresses we have yet to make requests for + queue: Vec
, + /// The in progress requests + in_progress: FuturesUnordered, +} + +impl ExternalFetcher { + fn new(fetcher: Arc, timeout: Duration, concurrency: usize) -> Self { + Self { + fetcher, + timeout, + backoff: None, + concurrency, + queue: Vec::new(), + in_progress: FuturesUnordered::new(), + } + } + + fn queue_next_reqs(&mut self) { + while self.in_progress.len() < self.concurrency { + let Some(addr) = self.queue.pop() else { break }; + let fetcher = Arc::clone(&self.fetcher); + self.in_progress.push(Box::pin(async move { + trace!(target: "traces::external", ?addr, "fetching info"); + let res = fetcher.fetch(addr).await; + (addr, res) + })); + } + } +} + +impl Stream for ExternalFetcher { + type Item = (Address, Metadata); + + fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + let pin = self.get_mut(); + + if pin.fetcher.invalid_api_key().load(Ordering::Relaxed) { + return Poll::Ready(None); + } + + loop { + if let Some(mut backoff) = pin.backoff.take() + && backoff.poll_tick(cx).is_pending() + { + pin.backoff = Some(backoff); + return Poll::Pending; + } + + pin.queue_next_reqs(); + + let mut made_progress_this_iter = false; + match pin.in_progress.poll_next_unpin(cx) { + Poll::Pending => {} + Poll::Ready(None) => return Poll::Ready(None), + Poll::Ready(Some((addr, res))) => { + made_progress_this_iter = true; + match res { + Ok(metadata) => { + if let Some(metadata) = metadata { + return Poll::Ready(Some((addr, metadata))); + } + } + Err(EtherscanError::RateLimitExceeded) => { + warn!(target: "traces::external", "rate limit exceeded on attempt"); + pin.backoff = Some(tokio::time::interval(pin.timeout)); + pin.queue.push(addr); + } + Err(EtherscanError::InvalidApiKey) => { + warn!(target: "traces::external", "invalid api key"); + // mark key as invalid + pin.fetcher.invalid_api_key().store(true, Ordering::Relaxed); + return Poll::Ready(None); + } + Err(EtherscanError::BlockedByCloudflare) => { + warn!(target: "traces::external", "blocked by cloudflare"); + // mark key as invalid + pin.fetcher.invalid_api_key().store(true, Ordering::Relaxed); + return Poll::Ready(None); + } + Err(err) => { + warn!(target: "traces::external", ?err, "could not get info"); + } + } + } + } + + if !made_progress_this_iter { + return Poll::Pending; + } + } + } +} + +#[async_trait::async_trait] +trait ExternalFetcherT: Send + Sync { + fn invalid_api_key(&self) -> &AtomicBool; + + async fn fetch(&self, address: Address) -> Result, EtherscanError>; +} + +struct EtherscanFetcher { + client: foundry_block_explorers::Client, + invalid_api_key: AtomicBool, +} + +impl EtherscanFetcher { + fn new(client: foundry_block_explorers::Client) -> Self { + Self { client, invalid_api_key: AtomicBool::new(false) } + } +} + +#[async_trait::async_trait] +impl ExternalFetcherT for EtherscanFetcher { + fn invalid_api_key(&self) -> &AtomicBool { + &self.invalid_api_key + } + + async fn fetch(&self, address: Address) -> Result, EtherscanError> { + self.client.contract_source_code(address).await.map(|mut metadata| metadata.items.pop()) + } +} + +struct SourcifyFetcher { + client: reqwest::Client, + url: String, + invalid_api_key: AtomicBool, +} + +impl SourcifyFetcher { + fn new(chain: Chain) -> Self { + Self { + client: reqwest::Client::new(), + url: format!("https://sourcify.dev/server/v2/contract/{}", chain.id()), + invalid_api_key: AtomicBool::new(false), + } + } +} + +#[async_trait::async_trait] +impl ExternalFetcherT for SourcifyFetcher { + fn invalid_api_key(&self) -> &AtomicBool { + &self.invalid_api_key + } + + async fn fetch(&self, address: Address) -> Result, EtherscanError> { + // abi,metadata,creationBytecode.onchainBytecode,deployment.blockNumber,compilation + let url = format!("{url}/{address}?fields=abi,compilation,proxyResolution", url = self.url); + let response = self.client.get(url).send().await?; + let code = response.status(); + match code.as_u16() { + // Not verified. + 404 => return Ok(None), + // Rate limited. + 429 => return Err(EtherscanError::RateLimitExceeded), + _ => {} + } + let response: SourcifyResponse = response.json().await?; + match response { + SourcifyResponse::Success(metadata) => Ok(Some(metadata.into())), + SourcifyResponse::Error(error) => Err(EtherscanError::Unknown(format!("{error:#?}"))), + } + } +} + +/// Sourcify API response for `/v2/contract/{chainId}/{address}`. +#[derive(Debug, Clone, Deserialize)] +#[serde(untagged)] +enum SourcifyResponse { + Success(SourcifyMetadata), + Error(SourcifyError), +} + +#[derive(Debug, Clone, Deserialize)] +#[serde(rename_all = "camelCase")] +#[expect(dead_code)] // Used in Debug. +struct SourcifyError { + custom_code: String, + message: String, + error_id: String, +} + +#[derive(Debug, Clone, Deserialize)] +#[serde(rename_all = "camelCase")] +struct SourcifyMetadata { + #[serde(default)] + abi: Option>, + #[serde(default)] + compilation: Option, +} + +#[derive(Debug, Clone, Deserialize)] +#[serde(rename_all = "camelCase")] +struct Compilation { + #[serde(default)] + compiler_version: String, + #[serde(default)] + name: String, +} + +impl From for Metadata { + fn from(metadata: SourcifyMetadata) -> Self { + let SourcifyMetadata { abi, compilation } = metadata; + // Defaulted fields may be fetched from sourcify but we don't make use of them. + Metadata { + source_code: foundry_block_explorers::contract::SourceCodeMetadata::Sources( + Default::default(), + ), + abi: Box::::from(abi.unwrap_or_default()).into(), + contract_name: compilation.as_ref().map(|c| c.name.clone()).unwrap_or_default(), + compiler_version: compilation + .as_ref() + .map(|c| c.compiler_version.clone()) + .unwrap_or_default(), + optimization_used: 0, + runs: 0, + constructor_arguments: Default::default(), + evm_version: String::new(), + library: String::new(), + license_type: String::new(), + proxy: 0, + implementation: None, + swarm_source: String::new(), + } + } +} diff --git a/crates/evm/traces/src/identifier/mod.rs b/crates/evm/traces/src/identifier/mod.rs index 82a756fcea4ec..c612e502fabf5 100644 --- a/crates/evm/traces/src/identifier/mod.rs +++ b/crates/evm/traces/src/identifier/mod.rs @@ -9,11 +9,8 @@ use std::borrow::Cow; mod local; pub use local::LocalTraceIdentifier; -mod etherscan; -pub use etherscan::EtherscanIdentifier; - -mod sourcify; -pub use sourcify::SourcifyIdentifier; +mod external; +pub use external::ExternalIdentifier; mod signatures; pub use signatures::{SignaturesCache, SignaturesIdentifier}; @@ -44,8 +41,8 @@ pub trait TraceIdentifier { pub struct TraceIdentifiers<'a> { /// The local trace identifier. pub local: Option>, - /// The optional Etherscan trace identifier. - pub etherscan: Option, + /// The optional external trace identifier. + pub external: Option, } impl Default for TraceIdentifiers<'_> { @@ -63,8 +60,8 @@ impl TraceIdentifier for TraceIdentifiers<'_> { return identities; } } - if let Some(etherscan) = &mut self.etherscan { - identities.extend(etherscan.identify_addresses(nodes)); + if let Some(external) = &mut self.external { + identities.extend(external.identify_addresses(nodes)); } identities } @@ -73,7 +70,7 @@ impl TraceIdentifier for TraceIdentifiers<'_> { impl<'a> TraceIdentifiers<'a> { /// Creates a new, empty instance. pub const fn new() -> Self { - Self { local: None, etherscan: None } + Self { local: None, external: None } } /// Sets the local identifier. @@ -93,14 +90,14 @@ impl<'a> TraceIdentifiers<'a> { self } - /// Sets the etherscan identifier. - pub fn with_etherscan(mut self, config: &Config, chain: Option) -> eyre::Result { - self.etherscan = EtherscanIdentifier::new(config, chain)?; + /// Sets the external identifier. + pub fn with_external(mut self, config: &Config, chain: Option) -> eyre::Result { + self.external = ExternalIdentifier::new(config, chain)?; Ok(self) } /// Returns `true` if there are no set identifiers. pub fn is_empty(&self) -> bool { - self.local.is_none() && self.etherscan.is_none() + self.local.is_none() && self.external.is_none() } } diff --git a/crates/evm/traces/src/identifier/sourcify.rs b/crates/evm/traces/src/identifier/sourcify.rs deleted file mode 100644 index c43f9e3f38759..0000000000000 --- a/crates/evm/traces/src/identifier/sourcify.rs +++ /dev/null @@ -1,229 +0,0 @@ -use super::{IdentifiedAddress, TraceIdentifier}; -use crate::debug::ContractSources; -use alloy_json_abi::JsonAbi; -use alloy_primitives::Address; -use foundry_common::compile::etherscan_project; -use foundry_config::{Chain, Config}; -use futures::{ - future::join_all, - stream::{FuturesUnordered, Stream, StreamExt}, - task::{Context, Poll}, -}; -use reqwest::StatusCode; -use revm_inspectors::tracing::types::CallTraceNode; -use serde::Deserialize; -use std::{borrow::Cow, collections::BTreeMap, pin::Pin, sync::atomic::Ordering}; -use tokio::time::{Duration, Interval}; - -#[derive(Debug, Clone, Deserialize)] -#[serde(untagged)] -enum SourcifyResponse { - Success(Metadata), - Error(SourcifyErrorResponse), -} - -#[derive(Debug, Clone, Deserialize)] -#[serde(rename_all = "camelCase")] -struct SourcifyErrorResponse { - custom_code: String, - message: String, - error_id: String, -} - -#[derive(Debug, Clone, Deserialize)] -#[serde(rename_all = "camelCase")] -struct Metadata { - #[serde(default)] - abi: Option, - #[serde(default)] - compilation: Option, -} - -#[derive(Debug, Clone, Deserialize)] -#[serde(rename_all = "camelCase")] -struct Compilation { - #[serde(default)] - language: String, - #[serde(default)] - name: String, -} - -/// A trace identifier that tries to identify addresses using Etherscan. -pub struct SourcifyIdentifier { - client: reqwest::Client, - url: String, - contracts: BTreeMap, -} - -impl SourcifyIdentifier { - /// Creates a new Etherscan identifier with the given client - pub fn new(config: &Config, chain: Option) -> eyre::Result> { - if config.offline { - return Ok(None); - } - Ok(Some(Self { - client: reqwest::Client::new(), - url: format!( - "https://sourcify.dev/server/v2/contract/{}", - chain.unwrap_or_default().id(), - ), - contracts: BTreeMap::new(), - })) - } - - fn identify_from_metadata( - &self, - address: Address, - metadata: &Metadata, - ) -> IdentifiedAddress<'static> { - let label = metadata.compilation.as_ref().map(|c| c.name.clone()); - let abi = metadata.abi.clone().map(Cow::Owned); - IdentifiedAddress { address, label: label.clone(), contract: label, abi, artifact_id: None } - } -} - -impl TraceIdentifier for SourcifyIdentifier { - fn identify_addresses(&mut self, nodes: &[&CallTraceNode]) -> Vec> { - if nodes.is_empty() { - return Vec::new(); - } - - trace!(target: "evm::traces::etherscan", "identify {} addresses", nodes.len()); - - let mut identities = Vec::new(); - let mut fetcher = - SourcifyFetcher::new(self.client.clone(), self.url.clone(), Duration::from_secs(1), 5); - - for &node in nodes { - let address = node.trace.address; - if let Some(metadata) = self.contracts.get(&address) { - identities.push(self.identify_from_metadata(address, metadata)); - } else { - fetcher.push(address); - } - } - - let fetched_identities = foundry_common::block_on( - fetcher - .map(|(address, metadata)| { - let addr = self.identify_from_metadata(address, &metadata); - self.contracts.insert(address, metadata); - addr - }) - .collect::>>(), - ); - - identities.extend(fetched_identities); - identities - } -} - -type SourcifyFuture = Pin< - Box)>>, ->; - -/// A rate limit aware Sourcify client. -/// -/// Fetches information about multiple addresses concurrently, while respecting rate limits. -struct SourcifyFetcher { - /// The client - client: reqwest::Client, - /// The URL to fetch the contract metadata from - url: String, - /// The time we wait if we hit the rate limit - timeout: Duration, - /// The interval we are currently waiting for before making a new request - backoff: Option, - /// The maximum amount of requests to send concurrently - concurrency: usize, - /// The addresses we have yet to make requests for - queue: Vec
, - /// The in progress requests - in_progress: FuturesUnordered, -} - -impl SourcifyFetcher { - fn new(client: reqwest::Client, url: String, timeout: Duration, concurrency: usize) -> Self { - Self { - client, - url, - timeout, - backoff: None, - concurrency, - queue: Vec::new(), - in_progress: FuturesUnordered::new(), - } - } - - fn push(&mut self, address: Address) { - self.queue.push(address); - } - - fn queue_next_reqs(&mut self) { - while self.in_progress.len() < self.concurrency { - let Some(addr) = self.queue.pop() else { break }; - let client = self.client.clone(); - let url = self.url.clone(); - self.in_progress.push(Box::pin(async move { - trace!(target: "traces::etherscan", ?addr, "fetching info"); - let res = client.get(format!("{url}/{addr}?fields=abi")).send().await; - let res = match res { - Ok(res) => { - let code = res.status(); - res.json().await.map(|res| (code, res)) - } - Err(e) => Err(e), - }; - (addr, res) - })); - } - } -} - -impl Stream for SourcifyFetcher { - type Item = (Address, Metadata); - - fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - let pin = self.get_mut(); - - loop { - if let Some(mut backoff) = pin.backoff.take() - && backoff.poll_tick(cx).is_pending() - { - pin.backoff = Some(backoff); - return Poll::Pending; - } - - pin.queue_next_reqs(); - - let mut made_progress_this_iter = false; - match pin.in_progress.poll_next_unpin(cx) { - Poll::Pending => {} - Poll::Ready(None) => return Poll::Ready(None), - Poll::Ready(Some((addr, res))) => { - made_progress_this_iter = true; - let (code, res) = match res { - Ok(r) => r, - Err(err) => { - warn!(target: "traces::etherscan", ?err, "could not get sourcify info"); - // continue; - } - }; - if code.as_u16() == 429 { - warn!(target: "traces::sourcify", ?res, "rate limit exceeded on attempt"); - pin.backoff = Some(tokio::time::interval(pin.timeout)); - pin.queue.push(addr); - // continue; - } - if let SourcifyResponse::Success(res) = res { - return Poll::Ready(Some((addr, res))); - } - } - } - - if !made_progress_this_iter { - return Poll::Pending; - } - } - } -} diff --git a/crates/forge/src/cmd/test/mod.rs b/crates/forge/src/cmd/test/mod.rs index 7c8a47da1bad9..b06f39ede46ee 100644 --- a/crates/forge/src/cmd/test/mod.rs +++ b/crates/forge/src/cmd/test/mod.rs @@ -528,7 +528,7 @@ impl TestArgs { // Avoid using etherscan for gas report as we decode more traces and this will be // expensive. if !self.gas_report { - identifier = identifier.with_etherscan(&config, remote_chain_id)?; + identifier = identifier.with_external(&config, remote_chain_id)?; } // Build the trace decoder. diff --git a/crates/script/src/execute.rs b/crates/script/src/execute.rs index 06a665b86823a..e9bcb8e813700 100644 --- a/crates/script/src/execute.rs +++ b/crates/script/src/execute.rs @@ -335,7 +335,7 @@ impl ExecutedState { .with_label_disabled(self.args.disable_labels) .build(); - let mut identifier = TraceIdentifiers::new().with_local(known_contracts).with_etherscan( + let mut identifier = TraceIdentifiers::new().with_local(known_contracts).with_external( &self.script_config.config, self.script_config.evm_opts.get_remote_chain_id().await, )?; From 11c3dfa9f15253b1f3784bc70aba4e1baeae18b6 Mon Sep 17 00:00:00 2001 From: DaniPopes <57450786+DaniPopes@users.noreply.github.com> Date: Wed, 1 Oct 2025 21:24:12 +0200 Subject: [PATCH 3/5] update overriding --- crates/evm/traces/src/identifier/external.rs | 48 +++++++++++++++----- 1 file changed, 37 insertions(+), 11 deletions(-) diff --git a/crates/evm/traces/src/identifier/external.rs b/crates/evm/traces/src/identifier/external.rs index e7e3315c340d3..7c162e8cad9d7 100644 --- a/crates/evm/traces/src/identifier/external.rs +++ b/crates/evm/traces/src/identifier/external.rs @@ -1,6 +1,9 @@ use super::{IdentifiedAddress, TraceIdentifier}; use crate::debug::ContractSources; -use alloy_primitives::{Address, map::HashMap}; +use alloy_primitives::{ + Address, + map::{Entry, HashMap}, +}; use foundry_block_explorers::{contract::Metadata, errors::EtherscanError}; use foundry_common::compile::etherscan_project; use foundry_config::{Chain, Config}; @@ -25,7 +28,7 @@ use tokio::time::{Duration, Interval}; pub struct ExternalIdentifier { fetchers: Vec>, /// Cached contracts. - contracts: HashMap, + contracts: HashMap, } impl ExternalIdentifier { @@ -67,8 +70,8 @@ impl ExternalIdentifier { .contracts .iter() // filter out vyper files - .filter(|(_, metadata)| !metadata.is_vyper()) - .map(|(address, metadata)| async move { + .filter(|(_, (_, metadata))| !metadata.is_vyper()) + .map(|(address, (_, metadata))| async move { sh_println!("Compiling: {} {address}", metadata.contract_name)?; let root = tempfile::tempdir()?; let root_path = root.path(); @@ -128,7 +131,7 @@ impl TraceIdentifier for ExternalIdentifier { // Check cache first. for &node in nodes { let address = node.trace.address; - if let Some(metadata) = self.contracts.get(&address) { + if let Some((_, metadata)) = self.contracts.get(&address) { identities.push(self.identify_from_metadata(address, metadata)); } else { to_fetch.push(address); @@ -145,9 +148,18 @@ impl TraceIdentifier for ExternalIdentifier { .map(|fetcher| ExternalFetcher::new(fetcher.clone(), Duration::from_secs(1), 5)); let fetched_identities = foundry_common::block_on( futures::stream::select_all(fetchers) - .map(|(address, metadata)| { - let addr = self.identify_from_metadata(address, &metadata); - self.contracts.insert(address, metadata); + .map(|(address, value)| { + let addr = self.identify_from_metadata(address, &value.1); + match self.contracts.entry(address) { + Entry::Occupied(mut occupied_entry) => { + if !matches!(occupied_entry.get().0, FetcherKind::Etherscan) { + occupied_entry.insert(value); + } + } + Entry::Vacant(vacant_entry) => { + vacant_entry.insert(value); + } + } addr }) .collect::>>(), @@ -205,7 +217,7 @@ impl ExternalFetcher { } impl Stream for ExternalFetcher { - type Item = (Address, Metadata); + type Item = (Address, (FetcherKind, Metadata)); fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { let pin = self.get_mut(); @@ -233,7 +245,7 @@ impl Stream for ExternalFetcher { match res { Ok(metadata) => { if let Some(metadata) = metadata { - return Poll::Ready(Some((addr, metadata))); + return Poll::Ready(Some((addr, (pin.fetcher.kind(), metadata)))); } } Err(EtherscanError::RateLimitExceeded) => { @@ -267,10 +279,16 @@ impl Stream for ExternalFetcher { } } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum FetcherKind { + Etherscan, + Sourcify, +} + #[async_trait::async_trait] trait ExternalFetcherT: Send + Sync { + fn kind(&self) -> FetcherKind; fn invalid_api_key(&self) -> &AtomicBool; - async fn fetch(&self, address: Address) -> Result, EtherscanError>; } @@ -287,6 +305,10 @@ impl EtherscanFetcher { #[async_trait::async_trait] impl ExternalFetcherT for EtherscanFetcher { + fn kind(&self) -> FetcherKind { + FetcherKind::Etherscan + } + fn invalid_api_key(&self) -> &AtomicBool { &self.invalid_api_key } @@ -314,6 +336,10 @@ impl SourcifyFetcher { #[async_trait::async_trait] impl ExternalFetcherT for SourcifyFetcher { + fn kind(&self) -> FetcherKind { + FetcherKind::Sourcify + } + fn invalid_api_key(&self) -> &AtomicBool { &self.invalid_api_key } From 211e2125b0e7ee6cf2ad73103dfa97afd63c293a Mon Sep 17 00:00:00 2001 From: DaniPopes <57450786+DaniPopes@users.noreply.github.com> Date: Wed, 1 Oct 2025 21:50:18 +0200 Subject: [PATCH 4/5] logs, config --- crates/evm/traces/src/identifier/external.rs | 50 +++++++++++++++----- crates/evm/traces/src/identifier/mod.rs | 1 + 2 files changed, 39 insertions(+), 12 deletions(-) diff --git a/crates/evm/traces/src/identifier/external.rs b/crates/evm/traces/src/identifier/external.rs index 7c162e8cad9d7..09568d2b44bcd 100644 --- a/crates/evm/traces/src/identifier/external.rs +++ b/crates/evm/traces/src/identifier/external.rs @@ -40,7 +40,6 @@ impl ExternalIdentifier { let config = match config.get_etherscan_config_with_chain(chain) { Ok(Some(config)) => { - trace!(target: "traces::external", chain=?config.chain, url=?config.api_url, "using etherscan identifier"); chain = config.chain; Some(config) } @@ -55,10 +54,18 @@ impl ExternalIdentifier { }; let mut fetchers = Vec::>::new(); - fetchers.push(Arc::new(SourcifyFetcher::new(chain.unwrap_or_default()))); + if let Some(chain) = chain { + trace!(target: "traces::external", ?chain, "using sourcify identifier"); + fetchers.push(Arc::new(SourcifyFetcher::new(chain))); + } if let Some(config) = config { + trace!(target: "traces::external", chain=?config.chain, url=?config.api_url, "using etherscan identifier"); fetchers.push(Arc::new(EtherscanFetcher::new(config.into_client()?))); } + if fetchers.is_empty() { + trace!(target: "traces::external", "no fetchers enabled"); + return Ok(None); + } Ok(Some(Self { fetchers, contracts: Default::default() })) } @@ -141,11 +148,10 @@ impl TraceIdentifier for ExternalIdentifier { if to_fetch.is_empty() { return identities; } + trace!(target: "evm::traces::external", "fetching {} addresses", to_fetch.len()); - let fetchers = self - .fetchers - .iter() - .map(|fetcher| ExternalFetcher::new(fetcher.clone(), Duration::from_secs(1), 5)); + let fetchers = + self.fetchers.iter().map(|fetcher| ExternalFetcher::new(fetcher.clone(), &to_fetch)); let fetched_identities = foundry_common::block_on( futures::stream::select_all(fetchers) .map(|(address, value)| { @@ -164,6 +170,7 @@ impl TraceIdentifier for ExternalIdentifier { }) .collect::>>(), ); + trace!(target: "traces::external", "fetched {} addresses", fetched_identities.len()); identities.extend(fetched_identities); identities @@ -192,13 +199,13 @@ struct ExternalFetcher { } impl ExternalFetcher { - fn new(fetcher: Arc, timeout: Duration, concurrency: usize) -> Self { + fn new(fetcher: Arc, to_fetch: &[Address]) -> Self { Self { - fetcher, - timeout, + timeout: fetcher.timeout(), backoff: None, - concurrency, - queue: Vec::new(), + concurrency: fetcher.concurrency(), + fetcher, + queue: to_fetch.to_vec(), in_progress: FuturesUnordered::new(), } } @@ -288,6 +295,8 @@ enum FetcherKind { #[async_trait::async_trait] trait ExternalFetcherT: Send + Sync { fn kind(&self) -> FetcherKind; + fn timeout(&self) -> Duration; + fn concurrency(&self) -> usize; fn invalid_api_key(&self) -> &AtomicBool; async fn fetch(&self, address: Address) -> Result, EtherscanError>; } @@ -309,6 +318,14 @@ impl ExternalFetcherT for EtherscanFetcher { FetcherKind::Etherscan } + fn timeout(&self) -> Duration { + Duration::from_secs(1) + } + + fn concurrency(&self) -> usize { + 5 + } + fn invalid_api_key(&self) -> &AtomicBool { &self.invalid_api_key } @@ -340,6 +357,14 @@ impl ExternalFetcherT for SourcifyFetcher { FetcherKind::Sourcify } + fn timeout(&self) -> Duration { + Duration::from_secs(1) + } + + fn concurrency(&self) -> usize { + 5 + } + fn invalid_api_key(&self) -> &AtomicBool { &self.invalid_api_key } @@ -349,6 +374,8 @@ impl ExternalFetcherT for SourcifyFetcher { let url = format!("{url}/{address}?fields=abi,compilation,proxyResolution", url = self.url); let response = self.client.get(url).send().await?; let code = response.status(); + let response: SourcifyResponse = response.json().await?; + trace!(target: "traces::external", "Sourcify response: {response:#?}"); match code.as_u16() { // Not verified. 404 => return Ok(None), @@ -356,7 +383,6 @@ impl ExternalFetcherT for SourcifyFetcher { 429 => return Err(EtherscanError::RateLimitExceeded), _ => {} } - let response: SourcifyResponse = response.json().await?; match response { SourcifyResponse::Success(metadata) => Ok(Some(metadata.into())), SourcifyResponse::Error(error) => Err(EtherscanError::Unknown(format!("{error:#?}"))), diff --git a/crates/evm/traces/src/identifier/mod.rs b/crates/evm/traces/src/identifier/mod.rs index c612e502fabf5..2fa57fca3d1d0 100644 --- a/crates/evm/traces/src/identifier/mod.rs +++ b/crates/evm/traces/src/identifier/mod.rs @@ -16,6 +16,7 @@ mod signatures; pub use signatures::{SignaturesCache, SignaturesIdentifier}; /// An address identified by a [`TraceIdentifier`]. +#[derive(Debug)] pub struct IdentifiedAddress<'a> { /// The address. pub address: Address, From 7932572a8d165f85b7a6b90969f51588144d27a3 Mon Sep 17 00:00:00 2001 From: DaniPopes <57450786+DaniPopes@users.noreply.github.com> Date: Wed, 1 Oct 2025 22:44:44 +0200 Subject: [PATCH 5/5] chore: more logs, cache None responses --- crates/evm/traces/src/identifier/external.rs | 73 ++++++++++++-------- crates/evm/traces/src/identifier/mod.rs | 4 ++ crates/forge/src/cmd/test/mod.rs | 2 +- 3 files changed, 51 insertions(+), 28 deletions(-) diff --git a/crates/evm/traces/src/identifier/external.rs b/crates/evm/traces/src/identifier/external.rs index 09568d2b44bcd..3ebb8d1929810 100644 --- a/crates/evm/traces/src/identifier/external.rs +++ b/crates/evm/traces/src/identifier/external.rs @@ -28,7 +28,7 @@ use tokio::time::{Duration, Interval}; pub struct ExternalIdentifier { fetchers: Vec>, /// Cached contracts. - contracts: HashMap, + contracts: HashMap)>, } impl ExternalIdentifier { @@ -44,26 +44,26 @@ impl ExternalIdentifier { Some(config) } Ok(None) => { - warn!(target: "traces::external", "etherscan config not found"); + warn!(target: "evm::traces::external", "etherscan config not found"); None } Err(err) => { - warn!(target: "traces::external", ?err, "failed to get etherscan config"); + warn!(target: "evm::traces::external", ?err, "failed to get etherscan config"); None } }; let mut fetchers = Vec::>::new(); if let Some(chain) = chain { - trace!(target: "traces::external", ?chain, "using sourcify identifier"); + debug!(target: "evm::traces::external", ?chain, "using sourcify identifier"); fetchers.push(Arc::new(SourcifyFetcher::new(chain))); } if let Some(config) = config { - trace!(target: "traces::external", chain=?config.chain, url=?config.api_url, "using etherscan identifier"); + debug!(target: "evm::traces::external", chain=?config.chain, url=?config.api_url, "using etherscan identifier"); fetchers.push(Arc::new(EtherscanFetcher::new(config.into_client()?))); } if fetchers.is_empty() { - trace!(target: "traces::external", "no fetchers enabled"); + debug!(target: "evm::traces::external", "no fetchers enabled"); return Ok(None); } @@ -77,8 +77,11 @@ impl ExternalIdentifier { .contracts .iter() // filter out vyper files - .filter(|(_, (_, metadata))| !metadata.is_vyper()) + .filter(|(_, (_, metadata))| { + metadata.as_ref().is_some_and(|metadata| !metadata.is_vyper()) + }) .map(|(address, (_, metadata))| async move { + let metadata = metadata.as_ref().unwrap(); sh_println!("Compiling: {} {address}", metadata.contract_name)?; let root = tempfile::tempdir()?; let root_path = root.path(); @@ -139,7 +142,11 @@ impl TraceIdentifier for ExternalIdentifier { for &node in nodes { let address = node.trace.address; if let Some((_, metadata)) = self.contracts.get(&address) { - identities.push(self.identify_from_metadata(address, metadata)); + if let Some(metadata) = metadata { + identities.push(self.identify_from_metadata(address, metadata)); + } else { + // Do nothing. We know that this contract was not verified. + } } else { to_fetch.push(address); } @@ -154,11 +161,19 @@ impl TraceIdentifier for ExternalIdentifier { self.fetchers.iter().map(|fetcher| ExternalFetcher::new(fetcher.clone(), &to_fetch)); let fetched_identities = foundry_common::block_on( futures::stream::select_all(fetchers) - .map(|(address, value)| { - let addr = self.identify_from_metadata(address, &value.1); + .filter_map(|(address, value)| { + let addr = value + .1 + .as_ref() + .map(|metadata| self.identify_from_metadata(address, metadata)); match self.contracts.entry(address) { Entry::Occupied(mut occupied_entry) => { - if !matches!(occupied_entry.get().0, FetcherKind::Etherscan) { + // Override if: + // - new is from Etherscan and old is not + // - new is Some and old is None, meaning verified only in one source + if !matches!(occupied_entry.get().0, FetcherKind::Etherscan) + || value.1.is_none() + { occupied_entry.insert(value); } } @@ -166,11 +181,11 @@ impl TraceIdentifier for ExternalIdentifier { vacant_entry.insert(value); } } - addr + async move { addr } }) .collect::>>(), ); - trace!(target: "traces::external", "fetched {} addresses", fetched_identities.len()); + trace!(target: "evm::traces::external", "fetched {} addresses: {fetched_identities:#?}", fetched_identities.len()); identities.extend(fetched_identities); identities @@ -215,7 +230,7 @@ impl ExternalFetcher { let Some(addr) = self.queue.pop() else { break }; let fetcher = Arc::clone(&self.fetcher); self.in_progress.push(Box::pin(async move { - trace!(target: "traces::external", ?addr, "fetching info"); + trace!(target: "evm::traces::external", ?addr, "fetching info"); let res = fetcher.fetch(addr).await; (addr, res) })); @@ -224,11 +239,15 @@ impl ExternalFetcher { } impl Stream for ExternalFetcher { - type Item = (Address, (FetcherKind, Metadata)); + type Item = (Address, (FetcherKind, Option)); fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { let pin = self.get_mut(); + let _guard = + info_span!("evm::traces::external", kind=?pin.fetcher.kind(), "ExternalFetcher") + .entered(); + if pin.fetcher.invalid_api_key().load(Ordering::Relaxed) { return Poll::Ready(None); } @@ -251,29 +270,30 @@ impl Stream for ExternalFetcher { made_progress_this_iter = true; match res { Ok(metadata) => { - if let Some(metadata) = metadata { - return Poll::Ready(Some((addr, (pin.fetcher.kind(), metadata)))); - } + return Poll::Ready(Some((addr, (pin.fetcher.kind(), metadata)))); + } + Err(EtherscanError::ContractCodeNotVerified(_)) => { + return Poll::Ready(Some((addr, (pin.fetcher.kind(), None)))); } Err(EtherscanError::RateLimitExceeded) => { - warn!(target: "traces::external", "rate limit exceeded on attempt"); + warn!(target: "evm::traces::external", "rate limit exceeded on attempt"); pin.backoff = Some(tokio::time::interval(pin.timeout)); pin.queue.push(addr); } Err(EtherscanError::InvalidApiKey) => { - warn!(target: "traces::external", "invalid api key"); + warn!(target: "evm::traces::external", "invalid api key"); // mark key as invalid pin.fetcher.invalid_api_key().store(true, Ordering::Relaxed); return Poll::Ready(None); } Err(EtherscanError::BlockedByCloudflare) => { - warn!(target: "traces::external", "blocked by cloudflare"); + warn!(target: "evm::traces::external", "blocked by cloudflare"); // mark key as invalid pin.fetcher.invalid_api_key().store(true, Ordering::Relaxed); return Poll::Ready(None); } Err(err) => { - warn!(target: "traces::external", ?err, "could not get info"); + warn!(target: "evm::traces::external", ?err, "could not get info"); } } } @@ -370,16 +390,15 @@ impl ExternalFetcherT for SourcifyFetcher { } async fn fetch(&self, address: Address) -> Result, EtherscanError> { - // abi,metadata,creationBytecode.onchainBytecode,deployment.blockNumber,compilation - let url = format!("{url}/{address}?fields=abi,compilation,proxyResolution", url = self.url); + let url = format!("{url}/{address}?fields=abi,compilation", url = self.url); let response = self.client.get(url).send().await?; let code = response.status(); let response: SourcifyResponse = response.json().await?; - trace!(target: "traces::external", "Sourcify response: {response:#?}"); + trace!(target: "evm::traces::external", "Sourcify response for {address}: {response:#?}"); match code.as_u16() { // Not verified. - 404 => return Ok(None), - // Rate limited. + 404 => return Err(EtherscanError::ContractCodeNotVerified(address)), + // Too many requests. 429 => return Err(EtherscanError::RateLimitExceeded), _ => {} } diff --git a/crates/evm/traces/src/identifier/mod.rs b/crates/evm/traces/src/identifier/mod.rs index 2fa57fca3d1d0..521c42010f25b 100644 --- a/crates/evm/traces/src/identifier/mod.rs +++ b/crates/evm/traces/src/identifier/mod.rs @@ -54,6 +54,10 @@ impl Default for TraceIdentifiers<'_> { impl TraceIdentifier for TraceIdentifiers<'_> { fn identify_addresses(&mut self, nodes: &[&CallTraceNode]) -> Vec> { + if nodes.is_empty() { + return Vec::new(); + } + let mut identities = Vec::with_capacity(nodes.len()); if let Some(local) = &mut self.local { identities.extend(local.identify_addresses(nodes)); diff --git a/crates/forge/src/cmd/test/mod.rs b/crates/forge/src/cmd/test/mod.rs index b06f39ede46ee..1065a516fcb37 100644 --- a/crates/forge/src/cmd/test/mod.rs +++ b/crates/forge/src/cmd/test/mod.rs @@ -525,7 +525,7 @@ impl TestArgs { // Set up trace identifiers. let mut identifier = TraceIdentifiers::new().with_local(&known_contracts); - // Avoid using etherscan for gas report as we decode more traces and this will be + // Avoid using external identifiers for gas report as we decode more traces and this will be // expensive. if !self.gas_report { identifier = identifier.with_external(&config, remote_chain_id)?;