Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 9 additions & 1 deletion crates/evm/evm/src/executors/fuzz/mod.rs
Original file line number Diff line number Diff line change
@@ -1,4 +1,6 @@
use crate::executors::{DURATION_BETWEEN_METRICS_REPORT, Executor, FuzzTestTimer, RawCallResult};
use crate::executors::{
DURATION_BETWEEN_METRICS_REPORT, Executor, FailFast, FuzzTestTimer, RawCallResult,
};
use alloy_dyn_abi::JsonAbiExt;
use alloy_json_abi::Function;
use alloy_primitives::{Address, Bytes, Log, U256, map::HashMap};
Expand Down Expand Up @@ -90,6 +92,7 @@ impl FuzzedExecutor {
/// test case.
///
/// Returns a list of all the consumed gas and calldata of every fuzz case.
#[allow(clippy::too_many_arguments)]
pub fn fuzz(
&mut self,
func: &Function,
Expand All @@ -98,6 +101,7 @@ impl FuzzedExecutor {
address: Address,
rd: &RevertDecoder,
progress: Option<&ProgressBar>,
fail_fast: &FailFast,
) -> Result<FuzzTestResult> {
// Stores the fuzz test execution data.
let mut test_data = FuzzTestData::default();
Expand Down Expand Up @@ -128,6 +132,10 @@ impl FuzzedExecutor {
let mut last_metrics_report = Instant::now();
let max_runs = self.config.runs;
let continue_campaign = |runs: u32| {
if fail_fast.should_stop() {
return false;
}

if timer.is_enabled() { !timer.is_timed_out() } else { runs < max_runs }
};

Expand Down
7 changes: 6 additions & 1 deletion crates/evm/evm/src/executors/invariant/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ use serde_json::json;

mod shrink;
use crate::executors::{
DURATION_BETWEEN_METRICS_REPORT, EvmError, FuzzTestTimer, corpus::CorpusManager,
DURATION_BETWEEN_METRICS_REPORT, EvmError, FailFast, FuzzTestTimer, corpus::CorpusManager,
};
pub use shrink::check_sequence;

Expand Down Expand Up @@ -327,6 +327,7 @@ impl<'a> InvariantExecutor<'a> {
fuzz_fixtures: &FuzzFixtures,
deployed_libs: &[Address],
progress: Option<&ProgressBar>,
fail_fast: &FailFast,
) -> Result<InvariantFuzzTestResult> {
// Throw an error to abort test run if the invariant function accepts input params
if !invariant_contract.invariant_function.inputs.is_empty() {
Expand All @@ -341,6 +342,10 @@ impl<'a> InvariantExecutor<'a> {
let timer = FuzzTestTimer::new(self.config.timeout);
let mut last_metrics_report = Instant::now();
let continue_campaign = |runs: u32| {
if fail_fast.should_stop() {
return false;
}

if timer.is_enabled() { !timer.is_timed_out() } else { runs < self.config.runs }
};

Expand Down
35 changes: 35 additions & 0 deletions crates/evm/evm/src/executors/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,10 @@ use revm::{
};
use std::{
borrow::Cow,
sync::{
Arc,
atomic::{AtomicBool, Ordering},
},
time::{Duration, Instant},
};

Expand Down Expand Up @@ -1109,3 +1113,34 @@ impl FuzzTestTimer {
self.inner.is_some_and(|(start, duration)| start.elapsed() > duration)
}
}

/// Helper struct to enable fail fast behavior: when one test fails, all other tests stop early.
#[derive(Clone)]
pub struct FailFast {
/// Shared atomic flag set to `true` when a failure occurs.
/// None if fail-fast is disabled.
inner: Option<Arc<AtomicBool>>,
}

impl FailFast {
pub fn new(fail_fast: bool) -> Self {
Self { inner: fail_fast.then_some(Arc::new(AtomicBool::new(false))) }
}

/// Returns `true` if fail-fast is enabled.
pub fn is_enabled(&self) -> bool {
self.inner.is_some()
}

/// Sets the failure flag. Used by other tests to stop early.
pub fn record_fail(&self) {
if let Some(fail_fast) = &self.inner {
fail_fast.store(true, Ordering::Relaxed);
}
}

/// Whether a failure has been recorded and test should stop.
pub fn should_stop(&self) -> bool {
self.inner.as_ref().map(|flag| flag.load(Ordering::Relaxed)).unwrap_or(false)
}
}
1 change: 1 addition & 0 deletions crates/forge/src/cmd/test/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -355,6 +355,7 @@ impl TestArgs {
.sender(evm_opts.sender)
.with_fork(evm_opts.get_fork(&config, env.clone()))
.enable_isolation(evm_opts.isolate)
.fail_fast(self.fail_fast)
.odyssey(evm_opts.odyssey)
.build::<MultiCompiler>(project_root, &output, env, evm_opts)?;

Expand Down
15 changes: 12 additions & 3 deletions crates/forge/src/multi_runner.rs
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ use foundry_evm::{
Env,
backend::Backend,
decode::RevertDecoder,
executors::{Executor, ExecutorBuilder},
executors::{Executor, ExecutorBuilder, FailFast},
fork::CreateFork,
inspectors::CheatsConfig,
opts::EvmOpts,
Expand Down Expand Up @@ -296,6 +296,8 @@ pub struct TestRunnerConfig {
pub isolation: bool,
/// Whether to enable Odyssey features.
pub odyssey: bool,
/// Whether to exit early on test failure.
pub fail_fast: FailFast,
}

impl TestRunnerConfig {
Expand Down Expand Up @@ -404,6 +406,8 @@ pub struct MultiContractRunnerBuilder {
pub isolation: bool,
/// Whether to enable Odyssey features.
pub odyssey: bool,
/// Whether to exit early on test failure.
pub fail_fast: bool,
}

impl MultiContractRunnerBuilder {
Expand All @@ -419,6 +423,7 @@ impl MultiContractRunnerBuilder {
isolation: Default::default(),
decode_internal: Default::default(),
odyssey: Default::default(),
fail_fast: false,
}
}

Expand Down Expand Up @@ -457,6 +462,11 @@ impl MultiContractRunnerBuilder {
self
}

pub fn fail_fast(mut self, fail_fast: bool) -> Self {
self.fail_fast = fail_fast;
self
}

pub fn enable_isolation(mut self, enable: bool) -> Self {
self.isolation = enable;
self
Expand Down Expand Up @@ -538,15 +548,14 @@ impl MultiContractRunnerBuilder {
env,
spec_id: self.evm_spec.unwrap_or_else(|| self.config.evm_spec_id()),
sender: self.sender.unwrap_or(self.config.sender),

line_coverage: self.line_coverage,
debug: self.debug,
decode_internal: self.decode_internal,
inline_config: Arc::new(InlineConfig::new_parsed(output, &self.config)?),
isolation: self.isolation,
odyssey: self.odyssey,

config: self.config,
fail_fast: FailFast::new(self.fail_fast),
},
})
}
Expand Down
18 changes: 18 additions & 0 deletions crates/forge/src/runner.rs
Original file line number Diff line number Diff line change
Expand Up @@ -401,9 +401,16 @@ impl<'a> ContractRunner<'a> {
return SuiteResult::new(start.elapsed(), [(instances, fail)].into(), warnings);
}

let fail_fast = &self.tcfg.fail_fast;

let test_results = functions
.par_iter()
.map(|&func| {
// Early exit if we're running with fail-fast and a test already failed.
if fail_fast.should_stop() {
return (func.signature(), TestResult::setup_result(setup.clone()));
}

let start = Instant::now();

let _guard = self.tokio_handle.enter();
Expand Down Expand Up @@ -432,6 +439,11 @@ impl<'a> ContractRunner<'a> {
);
res.duration = start.elapsed();

// Set fail fast flag if current test failed.
if res.status.is_failure() {
fail_fast.record_fail();
}

(sig, res)
})
.collect::<BTreeMap<_, _>>();
Expand Down Expand Up @@ -629,6 +641,10 @@ impl<'a> FunctionRunner<'a> {
);

for i in 0..fixtures_len {
if self.tcfg.fail_fast.should_stop() {
return self.result;
}

// Increment progress bar.
if let Some(progress) = progress.as_ref() {
progress.inc(1);
Expand Down Expand Up @@ -800,6 +816,7 @@ impl<'a> FunctionRunner<'a> {
&self.setup.fuzz_fixtures,
&self.setup.deployed_libs,
progress.as_ref(),
&self.tcfg.fail_fast,
) {
Ok(x) => x,
Err(e) => {
Expand Down Expand Up @@ -950,6 +967,7 @@ impl<'a> FunctionRunner<'a> {
self.address,
&self.cr.mcr.revert_decoder,
progress.as_ref(),
&self.tcfg.fail_fast,
) {
Ok(x) => x,
Err(e) => {
Expand Down
44 changes: 44 additions & 0 deletions crates/forge/tests/cli/config.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1878,3 +1878,47 @@ forgetest_init!(test_exclude_lints_config, |prj, cmd| {
});
cmd.args(["lint"]).assert_success().stdout_eq(str![""]);
});

// <https://github.com/foundry-rs/foundry/issues/6529>
forgetest_init!(test_fail_fast_config, |prj, cmd| {
prj.update_config(|config| {
// Set large timeout for fuzzed tests so test campaign won't stop if fail fast not passed.
config.fuzz.timeout = Some(3600);
config.invariant.timeout = Some(3600);
});
prj.add_test(
"AnotherCounterTest.sol",
r#"
import {Test} from "forge-std/Test.sol";

contract InvariantHandler is Test {
function fuzz_selector(uint256 x) public {
}
}

contract AnotherCounterTest is Test {
uint256[] public fixtureAmount = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10];

function setUp() public {
new InvariantHandler();
}
// This failure should stop all other tests.
function test_Failure() public pure {
require(false);
}

function testFuzz_SetNumber(uint256 x) public {
}

function invariant_SetNumber() public {
}

function table_SetNumber(uint256 amount) public {
require(amount < 100);
}
}
"#,
)
.unwrap();
cmd.args(["test", "--fail-fast"]).assert_failure();
});
Loading