From a7648916d36c7ed7ab6658d246013489804b85f3 Mon Sep 17 00:00:00 2001 From: grandizzy Date: Mon, 17 Nov 2025 12:08:32 +0200 Subject: [PATCH 1/2] feat(invariant): assert all invariants --- crates/evm/evm/src/executors/corpus.rs | 1 + .../evm/evm/src/executors/invariant/error.rs | 44 +++- crates/evm/evm/src/executors/invariant/mod.rs | 79 ++++---- .../evm/evm/src/executors/invariant/replay.rs | 43 +++- .../evm/evm/src/executors/invariant/result.rs | 167 +++++++++------- .../evm/evm/src/executors/invariant/shrink.rs | 2 +- crates/evm/fuzz/src/invariant/mod.rs | 6 +- crates/forge/src/runner.rs | 188 +++++++++++------- 8 files changed, 335 insertions(+), 195 deletions(-) diff --git a/crates/evm/evm/src/executors/corpus.rs b/crates/evm/evm/src/executors/corpus.rs index 4577c9788d708..8cee2f063f5eb 100644 --- a/crates/evm/evm/src/executors/corpus.rs +++ b/crates/evm/evm/src/executors/corpus.rs @@ -100,6 +100,7 @@ pub(crate) struct CorpusMetrics { impl fmt::Display for CorpusMetrics { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { writeln!(f)?; + writeln!(f, " Edge coverage metrics:")?; writeln!(f, " - cumulative edges seen: {}", self.cumulative_edges_seen)?; writeln!(f, " - cumulative features seen: {}", self.cumulative_features_seen)?; writeln!(f, " - corpus count: {}", self.corpus_count)?; diff --git a/crates/evm/evm/src/executors/invariant/error.rs b/crates/evm/evm/src/executors/invariant/error.rs index 9f48e9da82cbd..cb476a89f1530 100644 --- a/crates/evm/evm/src/executors/invariant/error.rs +++ b/crates/evm/evm/src/executors/invariant/error.rs @@ -1,10 +1,11 @@ use super::InvariantContract; use crate::executors::RawCallResult; +use alloy_json_abi::Function; use alloy_primitives::{Address, Bytes}; -use foundry_config::InvariantConfig; use foundry_evm_core::decode::RevertDecoder; use foundry_evm_fuzz::{BasicTxDetails, Reason, invariant::FuzzRunIdentifiedContracts}; use proptest::test_runner::TestError; +use std::{collections::HashMap, fmt}; /// Stores information about failures and reverts of the invariant tests. #[derive(Clone, Default)] @@ -14,7 +15,7 @@ pub struct InvariantFailures { /// The latest revert reason of a run. pub revert_reason: Option, /// Maps a broken invariant to its specific error. - pub error: Option, + pub errors: HashMap, } impl InvariantFailures { @@ -22,8 +23,32 @@ impl InvariantFailures { Self::default() } - pub fn into_inner(self) -> (usize, Option) { - (self.reverts, self.error) + pub fn into_inner(self) -> (usize, HashMap) { + (self.reverts, self.errors) + } + + pub fn record_failure(&mut self, invariant: &Function, failure: InvariantFuzzError) { + self.errors.insert(invariant.name.clone(), failure); + } + + pub fn has_failure(&self, invariant: &Function) -> bool { + self.errors.contains_key(&invariant.name) + } + + pub fn get_failure(&self, invariant: &Function) -> Option<&InvariantFuzzError> { + self.errors.get(&invariant.name) + } + + pub fn can_continue(&self, invariants: usize) -> bool { + self.errors.len() < invariants + } +} + +impl fmt::Display for InvariantFailures { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + writeln!(f)?; + writeln!(f, " ❌ Failures: {}", self.errors.len())?; + Ok(()) } } @@ -70,10 +95,11 @@ pub struct FailedInvariantCaseData { impl FailedInvariantCaseData { pub fn new( invariant_contract: &InvariantContract<'_>, - invariant_config: &InvariantConfig, + shrink_run_limit: u32, + fail_on_revert: bool, targeted_contracts: &FuzzRunIdentifiedContracts, calldata: &[BasicTxDetails], - call_result: RawCallResult, + call_result: &RawCallResult, inner_sequence: &[Option], ) -> Self { // Collect abis of fuzzed and invariant contracts to decode custom error. @@ -82,7 +108,7 @@ impl FailedInvariantCaseData { .with_abi(invariant_contract.abi) .decode(call_result.result.as_ref(), call_result.exit_reason); - let func = invariant_contract.invariant_function; + let func = invariant_contract.invariant_fn; debug_assert!(func.inputs.is_empty()); let origin = func.name.as_str(); Self { @@ -95,8 +121,8 @@ impl FailedInvariantCaseData { addr: invariant_contract.address, calldata: func.selector().to_vec().into(), inner_sequence: inner_sequence.to_vec(), - shrink_run_limit: invariant_config.shrink_run_limit, - fail_on_revert: invariant_config.fail_on_revert, + shrink_run_limit, + fail_on_revert, } } } diff --git a/crates/evm/evm/src/executors/invariant/mod.rs b/crates/evm/evm/src/executors/invariant/mod.rs index 11be0b72adc96..4f19b168ae7bd 100644 --- a/crates/evm/evm/src/executors/invariant/mod.rs +++ b/crates/evm/evm/src/executors/invariant/mod.rs @@ -2,6 +2,7 @@ use crate::{ executors::{Executor, RawCallResult}, inspectors::Fuzzer, }; +use alloy_json_abi::Function; use alloy_primitives::{ Address, Bytes, FixedBytes, Selector, U256, map::{AddressMap, HashMap}, @@ -28,7 +29,7 @@ use foundry_evm_traces::{CallTraceArena, SparsedTraceArena}; use indicatif::ProgressBar; use parking_lot::RwLock; use proptest::{strategy::Strategy, test_runner::TestRunner}; -use result::{assert_after_invariant, assert_invariants, can_continue}; +use result::{assert_after_invariant, can_continue}; use revm::state::Account; use std::{ collections::{HashMap as Map, btree_map::Entry}, @@ -41,7 +42,7 @@ pub use error::{InvariantFailures, InvariantFuzzError}; use foundry_evm_coverage::HitMaps; mod replay; -pub use replay::{replay_error, replay_run}; +pub use replay::{generate_counterexample, replay_error, replay_run}; mod result; use foundry_common::{TestFunctionExt, sh_println}; @@ -52,6 +53,7 @@ use serde_json::json; mod shrink; use crate::executors::{ DURATION_BETWEEN_METRICS_REPORT, EarlyExit, EvmError, FuzzTestTimer, corpus::CorpusManager, + invariant::result::invariant_preflight_check, }; pub use shrink::check_sequence; @@ -130,8 +132,6 @@ struct InvariantTestData { last_run_inputs: Vec, // Additional traces for gas report. gas_report_traces: Vec>, - // Last call results of the invariant test. - last_call_results: Option, // Line coverage information collected from all fuzzed calls. line_coverage: Option, // Metrics for each fuzzed selector. @@ -160,19 +160,13 @@ impl InvariantTest { fuzz_state: EvmFuzzState, targeted_contracts: FuzzRunIdentifiedContracts, failures: InvariantFailures, - last_call_results: Option, branch_runner: TestRunner, ) -> Self { - let mut fuzz_cases = vec![]; - if last_call_results.is_none() { - fuzz_cases.push(FuzzedCases::new(vec![])); - } let test_data = InvariantTestData { - fuzz_cases, + fuzz_cases: vec![], failures, last_run_inputs: vec![], gas_report_traces: vec![], - last_call_results, line_coverage: None, metrics: Map::default(), branch_runner, @@ -186,18 +180,13 @@ impl InvariantTest { } /// Whether invariant test has errors or not. - fn has_errors(&self) -> bool { - self.test_data.failures.error.is_some() + fn has_errors(&self, invariant: &Function) -> bool { + self.test_data.failures.has_failure(invariant) } /// Set invariant test error. - fn set_error(&mut self, error: InvariantFuzzError) { - self.test_data.failures.error = Some(error); - } - - /// Set last invariant test call results. - fn set_last_call_results(&mut self, call_result: Option) { - self.test_data.last_call_results = call_result; + fn set_error(&mut self, invariant: &Function, error: InvariantFuzzError) { + self.test_data.failures.record_failure(invariant, error); } /// Set last invariant run call sequence. @@ -336,7 +325,7 @@ impl<'a> InvariantExecutor<'a> { early_exit: &EarlyExit, ) -> Result { // Throw an error to abort test run if the invariant function accepts input params - if !invariant_contract.invariant_function.inputs.is_empty() { + if !invariant_contract.invariant_fn.inputs.is_empty() { return Err(eyre!("Invariant test function should have no inputs")); } @@ -421,9 +410,10 @@ impl<'a> InvariantExecutor<'a> { current_run.inputs.pop(); current_run.rejects += 1; if current_run.rejects > self.config.max_assume_rejects { - invariant_test.set_error(InvariantFuzzError::MaxAssumeRejects( - self.config.max_assume_rejects, - )); + invariant_test.set_error( + invariant_contract.invariant_fn, + InvariantFuzzError::MaxAssumeRejects(self.config.max_assume_rejects), + ); break 'stop; } } else { @@ -468,7 +458,7 @@ impl<'a> InvariantExecutor<'a> { }); // Determine if test can continue or should exit. - let result = can_continue( + let can_continue = can_continue( &invariant_contract, &mut invariant_test, &mut current_run, @@ -477,15 +467,13 @@ impl<'a> InvariantExecutor<'a> { &state_changeset, ) .map_err(|e| eyre!(e.to_string()))?; - if !result.can_continue || current_run.depth == self.config.depth - 1 { + if !can_continue || current_run.depth == self.config.depth - 1 { invariant_test.set_last_run_inputs(¤t_run.inputs); } // If test cannot continue then stop current run and exit test suite. - if !result.can_continue { + if !can_continue { break 'stop; } - - invariant_test.set_last_call_results(result.call_result); current_run.depth += 1; } @@ -501,7 +489,9 @@ impl<'a> InvariantExecutor<'a> { corpus_manager.process_inputs(¤t_run.inputs, current_run.new_coverage); // Call `afterInvariant` only if it is declared and test didn't fail already. - if invariant_contract.call_after_invariant && !invariant_test.has_errors() { + if invariant_contract.call_after_invariant + && !invariant_test.has_errors(invariant_contract.invariant_fn) + { assert_after_invariant( &invariant_contract, &mut invariant_test, @@ -516,10 +506,18 @@ impl<'a> InvariantExecutor<'a> { if let Some(progress) = progress { // If running with progress then increment completed runs. progress.inc(1); - // Display metrics in progress bar. + + let failures = &invariant_test.test_data.failures; + let mut parts = Vec::new(); + // Add failures if present + if !failures.errors.is_empty() { + parts.push(format!("{failures}")); + } + // Add edge coverage metrics if enabled if edge_coverage_enabled { - progress.set_message(format!("{}", &corpus_manager.metrics)); + parts.push(format!("{}", corpus_manager.metrics)); } + progress.set_message(parts.join("")); } else if edge_coverage_enabled && last_metrics_report.elapsed() > DURATION_BETWEEN_METRICS_REPORT { @@ -528,7 +526,7 @@ impl<'a> InvariantExecutor<'a> { "timestamp": SystemTime::now() .duration_since(UNIX_EPOCH)? .as_secs(), - "invariant": invariant_contract.invariant_function.name, + "invariant": invariant_contract.invariant_fn.name, "metrics": &corpus_manager.metrics, }); let _ = sh_println!("{}", serde_json::to_string(&metrics)?); @@ -543,7 +541,7 @@ impl<'a> InvariantExecutor<'a> { let result = invariant_test.test_data; Ok(InvariantFuzzTestResult { - error: result.failures.error, + errors: result.failures.errors, cases: result.fuzz_cases, reverts: result.failures.reverts, last_run_inputs: result.last_run_inputs, @@ -627,7 +625,7 @@ impl<'a> InvariantExecutor<'a> { // already know if we can early exit the invariant run. // This does not count as a fuzz run. It will just register the revert. let mut failures = InvariantFailures::new(); - let last_call_results = assert_invariants( + invariant_preflight_check( invariant_contract, &self.config, &targeted_contracts, @@ -635,7 +633,7 @@ impl<'a> InvariantExecutor<'a> { &[], &mut failures, )?; - if let Some(error) = failures.error { + if let Some(error) = failures.get_failure(invariant_contract.invariant_fn) { return Err(eyre!(error.revert_reason().unwrap_or_default())); } @@ -646,13 +644,8 @@ impl<'a> InvariantExecutor<'a> { None, Some(&targeted_contracts), )?; - let invariant_test = InvariantTest::new( - fuzz_state, - targeted_contracts, - failures, - last_call_results, - self.runner.clone(), - ); + let invariant_test = + InvariantTest::new(fuzz_state, targeted_contracts, failures, self.runner.clone()); Ok((invariant_test, corpus_manager)) } diff --git a/crates/evm/evm/src/executors/invariant/replay.rs b/crates/evm/evm/src/executors/invariant/replay.rs index 50276ca10cf48..65c4b5e62122d 100644 --- a/crates/evm/evm/src/executors/invariant/replay.rs +++ b/crates/evm/evm/src/executors/invariant/replay.rs @@ -70,7 +70,7 @@ pub fn replay_run( let (invariant_result, invariant_success) = call_invariant_function( &executor, invariant_contract.address, - invariant_contract.invariant_function.abi_encode_input(&[])?.into(), + invariant_contract.invariant_fn.abi_encode_input(&[])?.into(), )?; traces.push((TraceKind::Execution, invariant_result.traces.clone().unwrap())); logs.extend(invariant_result.logs); @@ -92,6 +92,47 @@ pub fn replay_run( Ok(counterexample_sequence) } +pub fn generate_counterexample( + mut executor: Executor, + known_contracts: &ContractsByArtifact, + mut ided_contracts: ContractsByAddress, + inputs: &[BasicTxDetails], + show_solidity: bool, +) -> Result> { + // We want traces for a failed case. + if executor.inspector().tracer.is_none() { + executor.set_tracing(TraceMode::Call); + } + + let mut counterexample_sequence = vec![]; + + // Replay each call from the sequence, collect logs, traces and coverage. + for tx in inputs { + let call_result = executor.transact_raw( + tx.sender, + tx.call_details.target, + tx.call_details.calldata.clone(), + U256::ZERO, + )?; + + // Identify newly generated contracts, if they exist. + ided_contracts + .extend(load_contracts(call_result.traces.iter().map(|a| &a.arena), known_contracts)); + + // Create counter example to be used in failed case. + counterexample_sequence.push(BaseCounterExample::from_invariant_call( + tx.sender, + tx.call_details.target, + &tx.call_details.calldata, + &ided_contracts, + call_result.traces, + show_solidity, + )); + } + + Ok(counterexample_sequence) +} + /// Replays the error case, shrinks the failing sequence and collects all necessary traces. #[expect(clippy::too_many_arguments)] pub fn replay_error( diff --git a/crates/evm/evm/src/executors/invariant/result.rs b/crates/evm/evm/src/executors/invariant/result.rs index 611f9fb6bfbc8..e057eb77220ea 100644 --- a/crates/evm/evm/src/executors/invariant/result.rs +++ b/crates/evm/evm/src/executors/invariant/result.rs @@ -18,7 +18,8 @@ use std::{borrow::Cow, collections::HashMap}; /// The outcome of an invariant fuzz test #[derive(Debug)] pub struct InvariantFuzzTestResult { - pub error: Option, + /// Errors recorded per invariant. + pub errors: HashMap, /// Every successful fuzz test case pub cases: Vec, /// Number of reverted fuzz calls @@ -36,61 +37,93 @@ pub struct InvariantFuzzTestResult { pub failed_corpus_replays: usize, } -/// Enriched results of an invariant run check. -/// -/// Contains the success condition and call results of the last run -pub(crate) struct RichInvariantResults { - pub(crate) can_continue: bool, - pub(crate) call_result: Option, -} - -impl RichInvariantResults { - fn new(can_continue: bool, call_result: Option) -> Self { - Self { can_continue, call_result } - } -} - /// Given the executor state, asserts that no invariant has been broken. Otherwise, it fills the /// external `invariant_failures.failed_invariant` map and returns a generic error. /// Either returns the call result if successful, or nothing if there was an error. -pub(crate) fn assert_invariants( +pub(crate) fn invariant_preflight_check( invariant_contract: &InvariantContract<'_>, invariant_config: &InvariantConfig, targeted_contracts: &FuzzRunIdentifiedContracts, executor: &Executor, calldata: &[BasicTxDetails], invariant_failures: &mut InvariantFailures, -) -> Result> { - let mut inner_sequence = vec![]; - - if let Some(fuzzer) = &executor.inspector().fuzzer - && let Some(call_generator) = &fuzzer.call_generator - { - inner_sequence.extend(call_generator.last_sequence.read().iter().cloned()); - } - +) -> Result<()> { let (call_result, success) = call_invariant_function( executor, invariant_contract.address, - invariant_contract.invariant_function.abi_encode_input(&[])?.into(), + invariant_contract.invariant_fn.abi_encode_input(&[])?.into(), )?; if !success { // We only care about invariants which we haven't broken yet. - if invariant_failures.error.is_none() { - let case_data = FailedInvariantCaseData::new( + invariant_failures.record_failure( + invariant_contract.invariant_fn, + InvariantFuzzError::BrokenInvariant(FailedInvariantCaseData::new( invariant_contract, - invariant_config, + invariant_config.shrink_run_limit, + invariant_config.fail_on_revert, targeted_contracts, calldata, - call_result, - &inner_sequence, + &call_result, + &invariant_inner_sequence(executor), + )), + ); + } + + Ok(()) +} + +/// Given the executor state, asserts that no invariant has been broken. Otherwise, it fills the +/// external `invariant_failures.failed_invariant` map and returns a generic error. +/// Either returns the call result if successful, or nothing if there was an error. +pub(crate) fn assert_invariants( + invariant_contract: &InvariantContract<'_>, + invariant_config: &InvariantConfig, + targeted_contracts: &FuzzRunIdentifiedContracts, + executor: &Executor, + calldata: &[BasicTxDetails], + invariant_failures: &mut InvariantFailures, +) -> Result<()> { + let inner_sequence = invariant_inner_sequence(executor); + // We only care about invariants which we haven't broken yet. + for (invariant, fail_on_revert) in &invariant_contract.invariant_fns { + // We only care about invariants which we haven't broken yet. + if invariant_failures.has_failure(invariant) { + continue; + } + + let (call_result, success) = call_invariant_function( + executor, + invariant_contract.address, + invariant.abi_encode_input(&[])?.into(), + )?; + if !success { + invariant_failures.record_failure( + invariant, + InvariantFuzzError::BrokenInvariant(FailedInvariantCaseData::new( + invariant_contract, + invariant_config.shrink_run_limit, + *fail_on_revert, + targeted_contracts, + calldata, + &call_result, + &inner_sequence, + )), ); - invariant_failures.error = Some(InvariantFuzzError::BrokenInvariant(case_data)); - return Ok(None); } } - Ok(Some(call_result)) + Ok(()) +} + +/// Helper function to initialize invariant inner sequence. +fn invariant_inner_sequence(executor: &Executor) -> Vec> { + let mut seq = vec![]; + if let Some(fuzzer) = &executor.inspector().fuzzer + && let Some(call_generator) = &fuzzer.call_generator + { + seq.extend(call_generator.last_sequence.read().iter().cloned()); + } + seq } /// Returns if invariant test can continue and last successful call result of the invariant test @@ -102,9 +135,7 @@ pub(crate) fn can_continue( invariant_config: &InvariantConfig, call_result: RawCallResult, state_changeset: &StateChangeset, -) -> Result { - let mut call_results = None; - +) -> Result { let handlers_succeeded = || { invariant_test.targeted_contracts.targets.lock().keys().all(|address| { invariant_run.executor.is_success( @@ -116,48 +147,46 @@ pub(crate) fn can_continue( }) }; + let failures = &mut invariant_test.test_data.failures; // Assert invariants if the call did not revert and the handlers did not fail. if !call_result.reverted && handlers_succeeded() { if let Some(traces) = call_result.traces { invariant_run.run_traces.push(traces); } - - call_results = assert_invariants( + assert_invariants( invariant_contract, invariant_config, &invariant_test.targeted_contracts, &invariant_run.executor, &invariant_run.inputs, - &mut invariant_test.test_data.failures, + failures, )?; - if call_results.is_none() { - return Ok(RichInvariantResults::new(false, None)); - } } else { // Increase the amount of reverts. - let invariant_data = &mut invariant_test.test_data; - invariant_data.failures.reverts += 1; - // If fail on revert is set, we must return immediately. - if invariant_config.fail_on_revert { - let case_data = FailedInvariantCaseData::new( - invariant_contract, - invariant_config, - &invariant_test.targeted_contracts, - &invariant_run.inputs, - call_result, - &[], - ); - invariant_data.failures.revert_reason = Some(case_data.revert_reason.clone()); - invariant_data.failures.error = Some(InvariantFuzzError::Revert(case_data)); - - return Ok(RichInvariantResults::new(false, None)); - } else if call_result.reverted { - // If we don't fail test on revert then remove last reverted call from inputs. - // This improves shrinking performance as irrelevant calls won't be checked again. - invariant_run.inputs.pop(); + failures.reverts += 1; + // If fail on revert is set, record invariant failure. + for (invariant, fail_on_revert) in &invariant_contract.invariant_fns { + if *fail_on_revert { + let case_data = FailedInvariantCaseData::new( + invariant_contract, + invariant_config.shrink_run_limit, + *fail_on_revert, + &invariant_test.targeted_contracts, + &invariant_run.inputs, + &call_result, + &[], + ); + failures + .errors + .insert(invariant.name.clone(), InvariantFuzzError::Revert(case_data)); + } } + // Remove last reverted call from inputs. + // This improves shrinking performance as irrelevant calls won't be checked again. + invariant_run.inputs.pop(); } - Ok(RichInvariantResults::new(true, call_results)) + // Stop execution if all invariants are broken. + Ok(failures.can_continue(invariant_contract.invariant_fns.len())) } /// Given the executor state, asserts conditions within `afterInvariant` function. @@ -174,13 +203,17 @@ pub(crate) fn assert_after_invariant( if !success { let case_data = FailedInvariantCaseData::new( invariant_contract, - invariant_config, + invariant_config.shrink_run_limit, + invariant_config.fail_on_revert, &invariant_test.targeted_contracts, &invariant_run.inputs, - call_result, + &call_result, &[], ); - invariant_test.set_error(InvariantFuzzError::BrokenInvariant(case_data)); + invariant_test.set_error( + invariant_contract.invariant_fn, + InvariantFuzzError::BrokenInvariant(case_data), + ); } Ok(success) } diff --git a/crates/evm/evm/src/executors/invariant/shrink.rs b/crates/evm/evm/src/executors/invariant/shrink.rs index 2788348b06193..105509e277634 100644 --- a/crates/evm/evm/src/executors/invariant/shrink.rs +++ b/crates/evm/evm/src/executors/invariant/shrink.rs @@ -50,7 +50,7 @@ pub(crate) fn shrink_sequence( } let target_address = invariant_contract.address; - let calldata: Bytes = invariant_contract.invariant_function.selector().to_vec().into(); + let calldata: Bytes = invariant_contract.invariant_fn.selector().to_vec().into(); // Special case test: the invariant is *unsatisfiable* - it took 0 calls to // break the invariant -- consider emitting a warning. let (_, success) = call_invariant_function(executor, target_address, calldata.clone())?; diff --git a/crates/evm/fuzz/src/invariant/mod.rs b/crates/evm/fuzz/src/invariant/mod.rs index a773e8f04bb27..34bc69c3a78d2 100644 --- a/crates/evm/fuzz/src/invariant/mod.rs +++ b/crates/evm/fuzz/src/invariant/mod.rs @@ -257,8 +257,10 @@ impl TargetedContract { pub struct InvariantContract<'a> { /// Address of the test contract. pub address: Address, - /// Invariant function present in the test contract. - pub invariant_function: &'a Function, + /// Invariant function. + pub invariant_fn: &'a Function, + /// All invariant functions present in the test contract and their fail on revert config. + pub invariant_fns: Vec<(&'a Function, bool)>, /// If true, `afterInvariant` function is called after each invariant run. pub call_after_invariant: bool, /// ABI of the test contract. diff --git a/crates/forge/src/runner.rs b/crates/forge/src/runner.rs index 721291ecb3e34..4776fa05ca168 100644 --- a/crates/forge/src/runner.rs +++ b/crates/forge/src/runner.rs @@ -22,7 +22,8 @@ use foundry_evm::{ CallResult, EvmError, Executor, ITest, RawCallResult, fuzz::FuzzedExecutor, invariant::{ - InvariantExecutor, InvariantFuzzError, check_sequence, replay_error, replay_run, + InvariantExecutor, InvariantFuzzError, check_sequence, generate_counterexample, + replay_error, replay_run, }, }, fuzz::{ @@ -339,7 +340,9 @@ impl<'a> ContractRunner<'a> { // Invariant testing requires tracing to figure out what contracts were created. // We also want to disable `debug` for setup since we won't be using those traces. - let has_invariants = self.contract.abi.functions().any(|func| func.is_invariant_test()); + let invariant_fns: Vec<_> = + self.contract.abi.functions().filter(|func| func.is_invariant_test()).collect(); + let has_invariants = !invariant_fns.is_empty(); let prev_tracer = self.executor.inspector_mut().tracer.take(); if prev_tracer.is_some() || has_invariants { @@ -436,6 +439,7 @@ impl<'a> ContractRunner<'a> { let mut res = FunctionRunner::new(&self, &setup).run( func, + invariant_fns.clone(), kind, call_after_invariant, identified_contracts.as_ref(), @@ -513,10 +517,22 @@ impl<'a> FunctionRunner<'a> { fn run( mut self, func: &Function, + invariants: Vec<&Function>, kind: TestFunctionKind, call_after_invariant: bool, identified_contracts: Option<&ContractsByAddress>, ) -> TestResult { + let fail_on_revert_for = |f: &Function| { + if self.inline_config.contains_function(self.cr.name, &f.name) + && let Ok(config) = self.cr.inline_config(Some(f)) + { + return config.invariant.fail_on_revert; + } + self.config.invariant.fail_on_revert + }; + let invariant_fns: Vec<_> = + invariants.into_iter().map(|f| (f, fail_on_revert_for(f))).collect(); + if let Err(e) = self.apply_function_inline_config(func) { self.result.single_fail(Some(e.to_string())); return self.result; @@ -530,6 +546,7 @@ impl<'a> FunctionRunner<'a> { let test_bytecode = &self.cr.contract.bytecode; self.run_invariant_test( func, + invariant_fns, call_after_invariant, identified_contracts.unwrap(), test_bytecode, @@ -713,6 +730,7 @@ impl<'a> FunctionRunner<'a> { fn run_invariant_test( mut self, func: &Function, + invariants: Vec<(&Function, bool)>, call_after_invariant: bool, identified_contracts: &ContractsByAddress, test_bytecode: &Bytes, @@ -756,7 +774,8 @@ impl<'a> FunctionRunner<'a> { ); let invariant_contract = InvariantContract { address: self.address, - invariant_function: func, + invariant_fn: func, + invariant_fns: invariants, call_after_invariant, abi: &self.cr.contract.abi, }; @@ -793,7 +812,7 @@ impl<'a> FunctionRunner<'a> { &txes, (0..min(txes.len(), invariant_config.depth as usize)).collect(), invariant_contract.address, - invariant_contract.invariant_function.selector().to_vec().into(), + invariant_contract.invariant_fn.selector().to_vec().into(), invariant_config.fail_on_revert, invariant_contract.call_after_invariant, ) && !success @@ -845,7 +864,7 @@ impl<'a> FunctionRunner<'a> { self.result.invariant_replay_fail( replayed_entirely, - &invariant_contract.invariant_function.name, + &invariant_contract.invariant_fn.name, call_sequence, ); return self.result; @@ -869,84 +888,109 @@ impl<'a> FunctionRunner<'a> { self.result.merge_coverages(invariant_result.line_coverage); let mut counterexample = None; - let success = invariant_result.error.is_none(); - let reason = invariant_result.error.as_ref().and_then(|err| err.revert_reason()); - - match invariant_result.error { - // If invariants were broken, replay the error to collect logs and traces - Some(error) => match error { - InvariantFuzzError::BrokenInvariant(case_data) - | InvariantFuzzError::Revert(case_data) => { - // Replay error to create counterexample and to collect logs, traces and - // coverage. - match case_data.test_error { - TestError::Abort(_) => {} - TestError::Fail(_, ref calls) => { - match replay_error( - evm.config(), - self.clone_executor(), - calls, - Some(case_data.inner_sequence), - &invariant_contract, - &self.cr.mcr.known_contracts, - identified_contracts.clone(), - &mut self.result.logs, - &mut self.result.traces, - &mut self.result.line_coverage, - &mut self.result.deprecated_cheatcodes, - progress.as_ref(), - &self.tcfg.early_exit, - ) { - Ok(call_sequence) => { - if !call_sequence.is_empty() { - // Persist error in invariant failure dir. - record_invariant_failure( - failure_dir.as_path(), - failure_file.as_path(), - &call_sequence, - test_bytecode, - ); - - let original_seq_len = if let TestError::Fail(_, calls) = - &case_data.test_error - { - calls.len() - } else { - call_sequence.len() - }; - - counterexample = Some(CounterExample::Sequence( - original_seq_len, - call_sequence, - )) - } - } - Err(err) => { - error!(%err, "Failed to replay invariant error"); - } - } - } - }; - } - InvariantFuzzError::MaxAssumeRejects(_) => {} - }, + let success = invariant_result.errors.is_empty(); + let reason = invariant_result + .errors + .get(&invariant_contract.invariant_fn.name) + .and_then(|err| err.revert_reason()); + if success { // If invariants ran successfully, replay the last run to collect logs and // traces. - _ => { - if let Err(err) = replay_run( - &invariant_contract, + if let Err(err) = replay_run( + &invariant_contract, + self.clone_executor(), + &self.cr.mcr.known_contracts, + identified_contracts.clone(), + &mut self.result.logs, + &mut self.result.traces, + &mut self.result.line_coverage, + &mut self.result.deprecated_cheatcodes, + &invariant_result.last_run_inputs, + show_solidity, + ) { + error!(%err, "Failed to replay last invariant run"); + } + } else { + // check if main invariant was broken and replay error + if let Some(error) = invariant_result.errors.get(&invariant_contract.invariant_fn.name) + && let InvariantFuzzError::BrokenInvariant(case_data) + | InvariantFuzzError::Revert(case_data) = error + && let TestError::Fail(_, ref calls) = case_data.test_error + { + match replay_error( + evm.config(), self.clone_executor(), + calls, + Some(case_data.inner_sequence.clone()), + &invariant_contract, &self.cr.mcr.known_contracts, identified_contracts.clone(), &mut self.result.logs, &mut self.result.traces, &mut self.result.line_coverage, &mut self.result.deprecated_cheatcodes, - &invariant_result.last_run_inputs, - show_solidity, + progress.as_ref(), + &self.tcfg.early_exit, ) { - error!(%err, "Failed to replay last invariant run"); + Ok(call_sequence) => { + if !call_sequence.is_empty() { + // Persist error in invariant failure dir. + record_invariant_failure( + failure_dir.as_path(), + failure_file.as_path(), + &call_sequence, + test_bytecode, + ); + + let original_seq_len = + if let TestError::Fail(_, calls) = &case_data.test_error { + calls.len() + } else { + call_sequence.len() + }; + + counterexample = + Some(CounterExample::Sequence(original_seq_len, call_sequence)) + } + } + Err(err) => { + error!(%err, "Failed to replay invariant error"); + } + } + } + + for (invariant, _) in invariant_contract.invariant_fns { + if invariant == invariant_contract.invariant_fn { + continue; + } + + // Generate counterexamples for other invariants broken. + if let Some(error) = invariant_result.errors.get(&invariant.name) + && let InvariantFuzzError::BrokenInvariant(case_data) + | InvariantFuzzError::Revert(case_data) = error + && let TestError::Fail(_, ref calls) = case_data.test_error + { + match generate_counterexample( + self.clone_executor(), + &self.cr.mcr.known_contracts, + identified_contracts.clone(), + calls, + show_solidity, + ) { + Ok(call_sequence) => { + // Persist error in invariant failure dir. + record_invariant_failure( + failure_dir.as_path(), + canonicalized(failure_dir.join(invariant.name.clone())).as_path(), + &call_sequence, + test_bytecode, + ); + } + Err(err) => { + error!(%err, "Failed to generate and record invariant counterexample"); + } + } } } } From a398def45179835e7497205709c7e11c94ebc0b0 Mon Sep 17 00:00:00 2001 From: grandizzy Date: Tue, 18 Nov 2025 07:48:47 +0200 Subject: [PATCH 2/2] Tests and Nits --- crates/config/src/invariant.rs | 3 + crates/forge/src/result.rs | 11 +++ crates/forge/src/runner.rs | 27 ++++++- crates/forge/tests/cli/config.rs | 4 +- .../forge/tests/cli/test_cmd/invariant/mod.rs | 80 +++++++++++++++++++ .../SimpleContractTestNonVerbose.json | 1 + .../fixtures/SimpleContractTestVerbose.json | 1 + 7 files changed, 122 insertions(+), 5 deletions(-) diff --git a/crates/config/src/invariant.rs b/crates/config/src/invariant.rs index 9a089180e1234..e4aa2e44a1a52 100644 --- a/crates/config/src/invariant.rs +++ b/crates/config/src/invariant.rs @@ -37,6 +37,8 @@ pub struct InvariantConfig { pub timeout: Option, /// Display counterexample as solidity calls. pub show_solidity: bool, + /// Continue invariant run until all invariants declared in current test suite breaks. + pub continuous_run: bool, } impl Default for InvariantConfig { @@ -55,6 +57,7 @@ impl Default for InvariantConfig { show_metrics: true, timeout: None, show_solidity: false, + continuous_run: false, } } } diff --git a/crates/forge/src/result.rs b/crates/forge/src/result.rs index 7a0f42846f4c1..c3f85c8f7d554 100644 --- a/crates/forge/src/result.rs +++ b/crates/forge/src/result.rs @@ -396,6 +396,9 @@ pub struct TestResult { /// still be successful (i.e self.success == true) when it's expected to fail. pub reason: Option, + /// This field will be populated if there are additional invariant broken besides the main one. + pub other_failures: Vec, + /// Minimal reproduction test case for failing test pub counterexample: Option, @@ -484,6 +487,12 @@ impl fmt::Display for TestResult { } else { s.push(']'); } + if !self.other_failures.is_empty() { + writeln!(s).unwrap(); + for failure in &self.other_failures { + writeln!(s, "{failure}").unwrap(); + } + } s.red().wrap().fmt(f) } } @@ -679,6 +688,7 @@ impl TestResult { gas_report_traces: Vec>, success: bool, reason: Option, + other_failures: Vec, counterexample: Option, cases: Vec, reverts: usize, @@ -697,6 +707,7 @@ impl TestResult { false => TestStatus::Failure, }; self.reason = reason; + self.other_failures = other_failures; self.counterexample = counterexample; self.gas_report_traces = gas_report_traces; } diff --git a/crates/forge/src/runner.rs b/crates/forge/src/runner.rs index 4776fa05ca168..a517fb2c015e1 100644 --- a/crates/forge/src/runner.rs +++ b/crates/forge/src/runner.rs @@ -772,10 +772,19 @@ impl<'a> FunctionRunner<'a> { identified_contracts, &self.cr.mcr.known_contracts, ); + + // Filter out additional invariants to test if we already have a persisted failure. let invariant_contract = InvariantContract { address: self.address, invariant_fn: func, - invariant_fns: invariants, + invariant_fns: invariants + .into_iter() + .filter(|(invariant_fn, _)| { + *invariant_fn == func + || (invariant_config.continuous_run + && !canonicalized(failure_dir.join(invariant_fn.name.clone())).exists()) + }) + .collect(), call_after_invariant, abi: &self.cr.contract.abi, }; @@ -893,6 +902,7 @@ impl<'a> FunctionRunner<'a> { .errors .get(&invariant_contract.invariant_fn.name) .and_then(|err| err.revert_reason()); + let mut other_failures = vec![]; if success { // If invariants ran successfully, replay the last run to collect logs and @@ -965,12 +975,20 @@ impl<'a> FunctionRunner<'a> { continue; } - // Generate counterexamples for other invariants broken. - if let Some(error) = invariant_result.errors.get(&invariant.name) + // Generate counterexamples for broken invariant, if there is no failure persisted + // already. + let persisted_failure = canonicalized(failure_dir.join(invariant.name.clone())); + if !persisted_failure.exists() + && let Some(error) = invariant_result.errors.get(&invariant.name) && let InvariantFuzzError::BrokenInvariant(case_data) | InvariantFuzzError::Revert(case_data) = error && let TestError::Fail(_, ref calls) = case_data.test_error { + other_failures.push(format!( + "{}: {}", + invariant.name, + error.revert_reason().unwrap_or_default() + )); match generate_counterexample( self.clone_executor(), &self.cr.mcr.known_contracts, @@ -982,7 +1000,7 @@ impl<'a> FunctionRunner<'a> { // Persist error in invariant failure dir. record_invariant_failure( failure_dir.as_path(), - canonicalized(failure_dir.join(invariant.name.clone())).as_path(), + persisted_failure.as_path(), &call_sequence, test_bytecode, ); @@ -999,6 +1017,7 @@ impl<'a> FunctionRunner<'a> { invariant_result.gas_report_traces, success, reason, + other_failures, counterexample, invariant_result.cases, invariant_result.reverts, diff --git a/crates/forge/tests/cli/config.rs b/crates/forge/tests/cli/config.rs index 6af175ae15a6d..92ded6a4b58ce 100644 --- a/crates/forge/tests/cli/config.rs +++ b/crates/forge/tests/cli/config.rs @@ -197,6 +197,7 @@ show_edge_coverage = false failure_persist_dir = "cache/invariant" show_metrics = true show_solidity = false +continuous_run = false [labels] @@ -1272,7 +1273,8 @@ forgetest_init!(test_default_config, |prj, cmd| { "failure_persist_dir": "cache/invariant", "show_metrics": true, "timeout": null, - "show_solidity": false + "show_solidity": false, + "continuous_run": false }, "ffi": false, "allow_internal_expect_revert": false, diff --git a/crates/forge/tests/cli/test_cmd/invariant/mod.rs b/crates/forge/tests/cli/test_cmd/invariant/mod.rs index 6f2f93363e293..f8007c9fdd533 100644 --- a/crates/forge/tests/cli/test_cmd/invariant/mod.rs +++ b/crates/forge/tests/cli/test_cmd/invariant/mod.rs @@ -991,3 +991,83 @@ Ran 3 test suites [ELAPSED]: 6 tests passed, 0 failed, 0 skipped (6 total tests) prj.root().join("fuzz_corpus").join("Counter2Test").join("testFuzz_SetNumber").exists() ); }); + +forgetest_init!(continous_run, |prj, cmd| { + prj.update_config(|config| { + config.invariant.runs = 10; + config.invariant.depth = 100; + config.invariant.continuous_run = true; + }); + prj.add_source( + "Counter.sol", + r#" +contract Counter { + uint256 public cond; + + function work(uint256 x) public { + if (x % 2 != 0 && x < 9000) { + cond++; + } else { + revert(); + } + } +} + "#, + ); + prj.add_test( + "CounterTest.t.sol", + r#" +import {Test} from "forge-std/Test.sol"; +import {Counter} from "../src/Counter.sol"; + +contract CounterTest is Test { + Counter public counter; + + function setUp() public { + counter = new Counter(); + } + + function invariant_cond1() public view { + require(counter.cond() < 10, "condition 1 met"); + } + + function invariant_cond2() public view { + require(counter.cond() < 15, "condition 2 met"); + } + + function invariant_cond3() public view { + require(counter.cond() < 5, "condition 3 met"); + } + + function invariant_cond4() public view { + require(counter.cond() < 111111, "condition 4 met"); + } + + /// forge-config: default.invariant.fail-on-revert = true + function invariant_cond5() public view { + require(counter.cond() < 111111, "condition 5 met"); + } +} + "#, + ); + + // Check that running single `invariant_cond3` test continue to run until it breaks all other + // invariants. + cmd.args(["test", "--mt", "invariant_cond3"]).assert_failure().stdout_eq(str![[r#" +[COMPILING_FILES] with [SOLC_VERSION] +[SOLC_VERSION] [ELAPSED] +Compiler run successful! + +Ran 1 test for test/CounterTest.t.sol:CounterTest +[FAIL: condition 3 met] + [Sequence] (original: 5, shrunk: 5) +... + +invariant_cond1: condition 1 met +invariant_cond2: condition 2 met +invariant_cond5: EvmError: Revert + invariant_cond3() (runs: 10, calls: 1000, reverts: [..]) +... + +"#]]); +}); diff --git a/crates/forge/tests/fixtures/SimpleContractTestNonVerbose.json b/crates/forge/tests/fixtures/SimpleContractTestNonVerbose.json index d8e517219b74f..6c8d55b8e13b1 100644 --- a/crates/forge/tests/fixtures/SimpleContractTestNonVerbose.json +++ b/crates/forge/tests/fixtures/SimpleContractTestNonVerbose.json @@ -5,6 +5,7 @@ "test()": { "status": "Success", "reason": null, + "other_failures": [], "counterexample": null, "logs": [], "decoded_logs": [], diff --git a/crates/forge/tests/fixtures/SimpleContractTestVerbose.json b/crates/forge/tests/fixtures/SimpleContractTestVerbose.json index 96fb5dffe1fa4..aba6765deb38b 100644 --- a/crates/forge/tests/fixtures/SimpleContractTestVerbose.json +++ b/crates/forge/tests/fixtures/SimpleContractTestVerbose.json @@ -5,6 +5,7 @@ "test()": { "status": "Success", "reason": null, + "other_failures": [], "counterexample": null, "logs": [ {