diff --git a/crates/evm/evm/src/executors/fuzz/mod.rs b/crates/evm/evm/src/executors/fuzz/mod.rs index 72fe796be7b7c..4efbe692e4ab7 100644 --- a/crates/evm/evm/src/executors/fuzz/mod.rs +++ b/crates/evm/evm/src/executors/fuzz/mod.rs @@ -1,4 +1,6 @@ -use crate::executors::{DURATION_BETWEEN_METRICS_REPORT, Executor, FuzzTestTimer, RawCallResult}; +use crate::executors::{ + DURATION_BETWEEN_METRICS_REPORT, Executor, FailFast, FuzzTestTimer, RawCallResult, +}; use alloy_dyn_abi::JsonAbiExt; use alloy_json_abi::Function; use alloy_primitives::{Address, Bytes, Log, U256, map::HashMap}; @@ -90,6 +92,7 @@ impl FuzzedExecutor { /// test case. /// /// Returns a list of all the consumed gas and calldata of every fuzz case. + #[allow(clippy::too_many_arguments)] pub fn fuzz( &mut self, func: &Function, @@ -98,6 +101,7 @@ impl FuzzedExecutor { address: Address, rd: &RevertDecoder, progress: Option<&ProgressBar>, + fail_fast: &FailFast, ) -> Result { // Stores the fuzz test execution data. let mut test_data = FuzzTestData::default(); @@ -128,6 +132,10 @@ impl FuzzedExecutor { let mut last_metrics_report = Instant::now(); let max_runs = self.config.runs; let continue_campaign = |runs: u32| { + if fail_fast.should_stop() { + return false; + } + if timer.is_enabled() { !timer.is_timed_out() } else { runs < max_runs } }; diff --git a/crates/evm/evm/src/executors/invariant/mod.rs b/crates/evm/evm/src/executors/invariant/mod.rs index 84ff2a29d00a9..e8000c87d8531 100644 --- a/crates/evm/evm/src/executors/invariant/mod.rs +++ b/crates/evm/evm/src/executors/invariant/mod.rs @@ -49,7 +49,7 @@ use serde_json::json; mod shrink; use crate::executors::{ - DURATION_BETWEEN_METRICS_REPORT, EvmError, FuzzTestTimer, corpus::CorpusManager, + DURATION_BETWEEN_METRICS_REPORT, EvmError, FailFast, FuzzTestTimer, corpus::CorpusManager, }; pub use shrink::check_sequence; @@ -327,6 +327,7 @@ impl<'a> InvariantExecutor<'a> { fuzz_fixtures: &FuzzFixtures, deployed_libs: &[Address], progress: Option<&ProgressBar>, + fail_fast: &FailFast, ) -> Result { // Throw an error to abort test run if the invariant function accepts input params if !invariant_contract.invariant_function.inputs.is_empty() { @@ -341,6 +342,10 @@ impl<'a> InvariantExecutor<'a> { let timer = FuzzTestTimer::new(self.config.timeout); let mut last_metrics_report = Instant::now(); let continue_campaign = |runs: u32| { + if fail_fast.should_stop() { + return false; + } + if timer.is_enabled() { !timer.is_timed_out() } else { runs < self.config.runs } }; diff --git a/crates/evm/evm/src/executors/mod.rs b/crates/evm/evm/src/executors/mod.rs index 6e794855f32fa..d0d29f33ed04c 100644 --- a/crates/evm/evm/src/executors/mod.rs +++ b/crates/evm/evm/src/executors/mod.rs @@ -44,6 +44,10 @@ use revm::{ }; use std::{ borrow::Cow, + sync::{ + Arc, + atomic::{AtomicBool, Ordering}, + }, time::{Duration, Instant}, }; @@ -1109,3 +1113,34 @@ impl FuzzTestTimer { self.inner.is_some_and(|(start, duration)| start.elapsed() > duration) } } + +/// Helper struct to enable fail fast behavior: when one test fails, all other tests stop early. +#[derive(Clone)] +pub struct FailFast { + /// Shared atomic flag set to `true` when a failure occurs. + /// None if fail-fast is disabled. + inner: Option>, +} + +impl FailFast { + pub fn new(fail_fast: bool) -> Self { + Self { inner: fail_fast.then_some(Arc::new(AtomicBool::new(false))) } + } + + /// Returns `true` if fail-fast is enabled. + pub fn is_enabled(&self) -> bool { + self.inner.is_some() + } + + /// Sets the failure flag. Used by other tests to stop early. + pub fn record_fail(&self) { + if let Some(fail_fast) = &self.inner { + fail_fast.store(true, Ordering::Relaxed); + } + } + + /// Whether a failure has been recorded and test should stop. + pub fn should_stop(&self) -> bool { + self.inner.as_ref().map(|flag| flag.load(Ordering::Relaxed)).unwrap_or(false) + } +} diff --git a/crates/forge/src/cmd/test/mod.rs b/crates/forge/src/cmd/test/mod.rs index 60e7646814bb6..27ab00cf2b52c 100644 --- a/crates/forge/src/cmd/test/mod.rs +++ b/crates/forge/src/cmd/test/mod.rs @@ -355,6 +355,7 @@ impl TestArgs { .sender(evm_opts.sender) .with_fork(evm_opts.get_fork(&config, env.clone())) .enable_isolation(evm_opts.isolate) + .fail_fast(self.fail_fast) .odyssey(evm_opts.odyssey) .build::(project_root, &output, env, evm_opts)?; diff --git a/crates/forge/src/multi_runner.rs b/crates/forge/src/multi_runner.rs index 590e95f1e23c2..4c0f4756e6e66 100644 --- a/crates/forge/src/multi_runner.rs +++ b/crates/forge/src/multi_runner.rs @@ -18,7 +18,7 @@ use foundry_evm::{ Env, backend::Backend, decode::RevertDecoder, - executors::{Executor, ExecutorBuilder}, + executors::{Executor, ExecutorBuilder, FailFast}, fork::CreateFork, inspectors::CheatsConfig, opts::EvmOpts, @@ -296,6 +296,8 @@ pub struct TestRunnerConfig { pub isolation: bool, /// Whether to enable Odyssey features. pub odyssey: bool, + /// Whether to exit early on test failure. + pub fail_fast: FailFast, } impl TestRunnerConfig { @@ -404,6 +406,8 @@ pub struct MultiContractRunnerBuilder { pub isolation: bool, /// Whether to enable Odyssey features. pub odyssey: bool, + /// Whether to exit early on test failure. + pub fail_fast: bool, } impl MultiContractRunnerBuilder { @@ -419,6 +423,7 @@ impl MultiContractRunnerBuilder { isolation: Default::default(), decode_internal: Default::default(), odyssey: Default::default(), + fail_fast: false, } } @@ -457,6 +462,11 @@ impl MultiContractRunnerBuilder { self } + pub fn fail_fast(mut self, fail_fast: bool) -> Self { + self.fail_fast = fail_fast; + self + } + pub fn enable_isolation(mut self, enable: bool) -> Self { self.isolation = enable; self @@ -538,15 +548,14 @@ impl MultiContractRunnerBuilder { env, spec_id: self.evm_spec.unwrap_or_else(|| self.config.evm_spec_id()), sender: self.sender.unwrap_or(self.config.sender), - line_coverage: self.line_coverage, debug: self.debug, decode_internal: self.decode_internal, inline_config: Arc::new(InlineConfig::new_parsed(output, &self.config)?), isolation: self.isolation, odyssey: self.odyssey, - config: self.config, + fail_fast: FailFast::new(self.fail_fast), }, }) } diff --git a/crates/forge/src/runner.rs b/crates/forge/src/runner.rs index f584545a08748..b65d458e8c6ae 100644 --- a/crates/forge/src/runner.rs +++ b/crates/forge/src/runner.rs @@ -401,9 +401,16 @@ impl<'a> ContractRunner<'a> { return SuiteResult::new(start.elapsed(), [(instances, fail)].into(), warnings); } + let fail_fast = &self.tcfg.fail_fast; + let test_results = functions .par_iter() .map(|&func| { + // Early exit if we're running with fail-fast and a test already failed. + if fail_fast.should_stop() { + return (func.signature(), TestResult::setup_result(setup.clone())); + } + let start = Instant::now(); let _guard = self.tokio_handle.enter(); @@ -432,6 +439,11 @@ impl<'a> ContractRunner<'a> { ); res.duration = start.elapsed(); + // Set fail fast flag if current test failed. + if res.status.is_failure() { + fail_fast.record_fail(); + } + (sig, res) }) .collect::>(); @@ -629,6 +641,10 @@ impl<'a> FunctionRunner<'a> { ); for i in 0..fixtures_len { + if self.tcfg.fail_fast.should_stop() { + return self.result; + } + // Increment progress bar. if let Some(progress) = progress.as_ref() { progress.inc(1); @@ -800,6 +816,7 @@ impl<'a> FunctionRunner<'a> { &self.setup.fuzz_fixtures, &self.setup.deployed_libs, progress.as_ref(), + &self.tcfg.fail_fast, ) { Ok(x) => x, Err(e) => { @@ -950,6 +967,7 @@ impl<'a> FunctionRunner<'a> { self.address, &self.cr.mcr.revert_decoder, progress.as_ref(), + &self.tcfg.fail_fast, ) { Ok(x) => x, Err(e) => { diff --git a/crates/forge/tests/cli/config.rs b/crates/forge/tests/cli/config.rs index e2e55265f6924..e03408a6f4e74 100644 --- a/crates/forge/tests/cli/config.rs +++ b/crates/forge/tests/cli/config.rs @@ -1878,3 +1878,47 @@ forgetest_init!(test_exclude_lints_config, |prj, cmd| { }); cmd.args(["lint"]).assert_success().stdout_eq(str![""]); }); + +// +forgetest_init!(test_fail_fast_config, |prj, cmd| { + prj.update_config(|config| { + // Set large timeout for fuzzed tests so test campaign won't stop if fail fast not passed. + config.fuzz.timeout = Some(3600); + config.invariant.timeout = Some(3600); + }); + prj.add_test( + "AnotherCounterTest.sol", + r#" +import {Test} from "forge-std/Test.sol"; + +contract InvariantHandler is Test { + function fuzz_selector(uint256 x) public { + } +} + +contract AnotherCounterTest is Test { + uint256[] public fixtureAmount = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]; + + function setUp() public { + new InvariantHandler(); + } + // This failure should stop all other tests. + function test_Failure() public pure { + require(false); + } + + function testFuzz_SetNumber(uint256 x) public { + } + + function invariant_SetNumber() public { + } + + function table_SetNumber(uint256 amount) public { + require(amount < 100); + } +} +"#, + ) + .unwrap(); + cmd.args(["test", "--fail-fast"]).assert_failure(); +});