feat: add timeouts to fuzz testing (foundry-rs#9394)

* feat: add timeouts to fuzz testing

Adds --fuzz-timeout-secs to fuzz tests which will cause a property
test to timeout after a certain number of seconds. Also adds
--fuzz-allow-timeouts so that timeouts are optionally not
considered to be failures.

* simplify timeout implementation

* use u32 for timeout

* switch back to failing for timeouts

* clippy

* Nits:
- move logic to interrupt invariant test in depth loop
- add and reuse start_timer fn and TEST_TIMEOUT constant
- add fuzz and invariant tests
- fix failing test

* Fix fmt

* Changes after review: introduce FuzzTestTimer

---------

Co-authored-by: grandizzy <[email protected]>

Author: 2 people

crates/config/src/fuzz.rs

@@ -28,6 +28,8 @@ pub struct FuzzConfig {
     pub failure_persist_file: Option<String>,
     /// show `console.log` in fuzz test, defaults to `false`
     pub show_logs: bool,
+    /// Optional timeout (in seconds) for each property test
+    pub timeout: Option<u32>,
 }
 
 impl Default for FuzzConfig {
@@ -41,6 +43,7 @@ impl Default for FuzzConfig {
             failure_persist_dir: None,
             failure_persist_file: None,
             show_logs: false,
+            timeout: None,
         }
     }
 }

crates/config/src/invariant.rs

@@ -30,6 +30,8 @@ pub struct InvariantConfig {
     pub failure_persist_dir: Option<PathBuf>,
     /// Whether to collect and display fuzzed selectors metrics.
     pub show_metrics: bool,
+    /// Optional timeout (in seconds) for each invariant test.
+    pub timeout: Option<u32>,
 }
 
 impl Default for InvariantConfig {
@@ -45,6 +47,7 @@ impl Default for InvariantConfig {
             gas_report_samples: 256,
             failure_persist_dir: None,
             show_metrics: false,
+            timeout: None,
         }
     }
 }
@@ -63,6 +66,7 @@ impl InvariantConfig {
             gas_report_samples: 256,
             failure_persist_dir: Some(cache_dir),
             show_metrics: false,
+            timeout: None,
         }
     }
 

crates/evm/core/src/constants.rs

@@ -37,6 +37,9 @@ pub const MAGIC_ASSUME: &[u8] = b"FOUNDRY::ASSUME";
 /// Magic return value returned by the `skip` cheatcode. Optionally appended with a reason.
 pub const MAGIC_SKIP: &[u8] = b"FOUNDRY::SKIP";
 
+/// Test timeout return value.
+pub const TEST_TIMEOUT: &str = "FOUNDRY::TEST_TIMEOUT";
+
 /// The address that deploys the default CREATE2 deployer contract.
 pub const DEFAULT_CREATE2_DEPLOYER_DEPLOYER: Address =
     address!("3fAB184622Dc19b6109349B94811493BF2a45362");

crates/evm/evm/src/executors/fuzz/mod.rs

@@ -1,12 +1,12 @@
-use crate::executors::{Executor, RawCallResult};
+use crate::executors::{Executor, FuzzTestTimer, RawCallResult};
 use alloy_dyn_abi::JsonAbiExt;
 use alloy_json_abi::Function;
 use alloy_primitives::{map::HashMap, Address, Bytes, Log, U256};
 use eyre::Result;
 use foundry_common::evm::Breakpoints;
 use foundry_config::FuzzConfig;
 use foundry_evm_core::{
-    constants::MAGIC_ASSUME,
+    constants::{MAGIC_ASSUME, TEST_TIMEOUT},
     decode::{RevertDecoder, SkipReason},
 };
 use foundry_evm_coverage::HitMaps;
@@ -98,7 +98,15 @@ impl FuzzedExecutor {
         let max_traces_to_collect = std::cmp::max(1, self.config.gas_report_samples) as usize;
         let show_logs = self.config.show_logs;
 
+        // Start timer for this fuzz test.
+        let timer = FuzzTestTimer::new(self.config.timeout);
+
         let run_result = self.runner.clone().run(&strategy, |calldata| {
+            // Check if the timeout has been reached.
+            if timer.is_timed_out() {
+                return Err(TestCaseError::fail(TEST_TIMEOUT));
+            }
+
             let fuzz_res = self.single_fuzz(address, should_fail, calldata)?;
 
             // If running with progress then increment current run.
@@ -193,17 +201,21 @@ impl FuzzedExecutor {
             }
             Err(TestError::Fail(reason, _)) => {
                 let reason = reason.to_string();
-                result.reason = (!reason.is_empty()).then_some(reason);
-
-                let args = if let Some(data) = calldata.get(4..) {
-                    func.abi_decode_input(data, false).unwrap_or_default()
+                if reason == TEST_TIMEOUT {
+                    // If the reason is a timeout, we consider the fuzz test successful.
+                    result.success = true;
                 } else {
-                    vec![]
-                };
+                    result.reason = (!reason.is_empty()).then_some(reason);
+                    let args = if let Some(data) = calldata.get(4..) {
+                        func.abi_decode_input(data, false).unwrap_or_default()
+                    } else {
+                        vec![]
+                    };
 
-                result.counterexample = Some(CounterExample::Single(
-                    BaseCounterExample::from_fuzz_call(calldata, args, call.traces),
-                ));
+                    result.counterexample = Some(CounterExample::Single(
+                        BaseCounterExample::from_fuzz_call(calldata, args, call.traces),
+                    ));
+                }
             }
         }
 

crates/evm/evm/src/executors/invariant/mod.rs

@@ -10,6 +10,7 @@ use foundry_config::InvariantConfig;
 use foundry_evm_core::{
     constants::{
         CALLER, CHEATCODE_ADDRESS, DEFAULT_CREATE2_DEPLOYER, HARDHAT_CONSOLE_ADDRESS, MAGIC_ASSUME,
+        TEST_TIMEOUT,
     },
     precompiles::PRECOMPILES,
 };
@@ -49,7 +50,7 @@ pub use result::InvariantFuzzTestResult;
 use serde::{Deserialize, Serialize};
 
 mod shrink;
-use crate::executors::EvmError;
+use crate::executors::{EvmError, FuzzTestTimer};
 pub use shrink::check_sequence;
 
 sol! {
@@ -332,6 +333,9 @@ impl<'a> InvariantExecutor<'a> {
         let (invariant_test, invariant_strategy) =
             self.prepare_test(&invariant_contract, fuzz_fixtures)?;
 
+        // Start timer for this invariant test.
+        let timer = FuzzTestTimer::new(self.config.timeout);
+
         let _ = self.runner.run(&invariant_strategy, |first_input| {
             // Create current invariant run data.
             let mut current_run = InvariantTestRun::new(
@@ -347,6 +351,15 @@ impl<'a> InvariantExecutor<'a> {
             }
 
             while current_run.depth < self.config.depth {
+                // Check if the timeout has been reached.
+                if timer.is_timed_out() {
+                    // Since we never record a revert here the test is still considered
+                    // successful even though it timed out. We *want*
+                    // this behavior for now, so that's ok, but
+                    // future developers should be aware of this.
+                    return Err(TestCaseError::fail(TEST_TIMEOUT));
+                }
+
                 let tx = current_run.inputs.last().ok_or_else(|| {
                     TestCaseError::fail("No input generated to call fuzzed target.")
                 })?;

crates/evm/evm/src/executors/mod.rs

@@ -35,7 +35,10 @@ use revm::{
         ResultAndState, SignedAuthorization, SpecId, TxEnv, TxKind,
     },
 };
-use std::borrow::Cow;
+use std::{
+    borrow::Cow,
+    time::{Duration, Instant},
+};
 
 mod builder;
 pub use builder::ExecutorBuilder;
@@ -952,3 +955,20 @@ fn convert_executed_result(
         chisel_state,
     })
 }
+
+/// Timer for a fuzz test.
+pub struct FuzzTestTimer {
+    /// Inner fuzz test timer - (test start time, test duration).
+    inner: Option<(Instant, Duration)>,
+}
+
+impl FuzzTestTimer {
+    pub fn new(timeout: Option<u32>) -> Self {
+        Self { inner: timeout.map(|timeout| (Instant::now(), Duration::from_secs(timeout.into()))) }
+    }
+
+    /// Whether the current fuzz test timed out and should be stopped.
+    pub fn is_timed_out(&self) -> bool {
+        self.inner.is_some_and(|(start, duration)| start.elapsed() > duration)
+    }
+}

crates/forge/bin/cmd/test/mod.rs

@@ -145,6 +145,10 @@ pub struct TestArgs {
     #[arg(long, env = "FOUNDRY_FUZZ_RUNS", value_name = "RUNS")]
     pub fuzz_runs: Option<u64>,
 
+    /// Timeout for each fuzz run in seconds.
+    #[arg(long, env = "FOUNDRY_FUZZ_TIMEOUT", value_name = "TIMEOUT")]
+    pub fuzz_timeout: Option<u64>,
+
     /// File to rerun fuzz failures from.
     #[arg(long)]
     pub fuzz_input_file: Option<String>,
@@ -864,6 +868,9 @@ impl Provider for TestArgs {
         if let Some(fuzz_runs) = self.fuzz_runs {
             fuzz_dict.insert("runs".to_string(), fuzz_runs.into());
         }
+        if let Some(fuzz_timeout) = self.fuzz_timeout {
+            fuzz_dict.insert("timeout".to_string(), fuzz_timeout.into());
+        }
         if let Some(fuzz_input_file) = self.fuzz_input_file.clone() {
             fuzz_dict.insert("failure_persist_file".to_string(), fuzz_input_file.into());
         }

crates/forge/tests/it/fuzz.rs

@@ -240,3 +240,38 @@ contract InlineMaxRejectsTest is Test {
 ...
 "#]]);
 });
+
+// Tests that test timeout config is properly applied.
+// If test doesn't timeout after one second, then test will fail with `rejected too many inputs`.
+forgetest_init!(test_fuzz_timeout, |prj, cmd| {
+    prj.wipe_contracts();
+
+    prj.add_test(
+        "Contract.t.sol",
+        r#"
+import {Test} from "forge-std/Test.sol";
+
+contract FuzzTimeoutTest is Test {
+    /// forge-config: default.fuzz.max-test-rejects = 10000
+    /// forge-config: default.fuzz.timeout = 1
+    function test_fuzz_bound(uint256 a) public pure {
+        vm.assume(a == 0);
+    }
+}
+   "#,
+    )
+    .unwrap();
+
+    cmd.args(["test"]).assert_success().stdout_eq(str![[r#"
+[COMPILING_FILES] with [SOLC_VERSION]
+[SOLC_VERSION] [ELAPSED]
+Compiler run successful!
+
+Ran 1 test for test/Contract.t.sol:FuzzTimeoutTest
+[PASS] test_fuzz_bound(uint256) (runs: [..], [AVG_GAS])
+Suite result: ok. 1 passed; 0 failed; 0 skipped; [ELAPSED]
+
+Ran 1 test suite [ELAPSED]: 1 tests passed, 0 failed, 0 skipped (1 total tests)
+
+"#]]);
+});

crates/forge/tests/it/invariant.rs

@@ -949,3 +949,53 @@ Ran 2 tests for test/SelectorMetricsTest.t.sol:CounterTest
 ...
 "#]]);
 });
+
+// Tests that invariant exists with success after configured timeout.
+forgetest_init!(should_apply_configured_timeout, |prj, cmd| {
+    // Add initial test that breaks invariant.
+    prj.add_test(
+        "TimeoutTest.t.sol",
+        r#"
+import {Test} from "forge-std/Test.sol";
+
+contract TimeoutHandler is Test {
+    uint256 public count;
+
+    function increment() public {
+        count++;
+    }
+}
+
+contract TimeoutTest is Test {
+    TimeoutHandler handler;
+
+    function setUp() public {
+        handler = new TimeoutHandler();
+    }
+
+    /// forge-config: default.invariant.runs = 10000
+    /// forge-config: default.invariant.depth = 20000
+    /// forge-config: default.invariant.timeout = 1
+    function invariant_counter_timeout() public view {
+        // Invariant will fail if more than 10000 increments.
+        // Make sure test timeouts after one second and remaining runs are canceled.
+        require(handler.count() < 10000);
+    }
+}
+     "#,
+    )
+    .unwrap();
+
+    cmd.args(["test", "--mt", "invariant_counter_timeout"]).assert_success().stdout_eq(str![[r#"
+[COMPILING_FILES] with [SOLC_VERSION]
+[SOLC_VERSION] [ELAPSED]
+Compiler run successful!
+
+Ran 1 test for test/TimeoutTest.t.sol:TimeoutTest
+[PASS] invariant_counter_timeout() (runs: 0, calls: 0, reverts: 0)
+Suite result: ok. 1 passed; 0 failed; 0 skipped; [ELAPSED]
+
+Ran 1 test suite [ELAPSED]: 1 tests passed, 0 failed, 0 skipped (1 total tests)
+
+"#]]);
+});

crates/forge/tests/it/test_helpers.rs

@@ -121,6 +121,7 @@ impl ForgeTestProfile {
             failure_persist_dir: Some(tempfile::tempdir().unwrap().into_path()),
             failure_persist_file: Some("testfailure".to_string()),
             show_logs: false,
+            timeout: None,
         };
         config.invariant = InvariantConfig {
             runs: 256,
@@ -145,6 +146,7 @@ impl ForgeTestProfile {
                     .into_path(),
             ),
             show_metrics: false,
+            timeout: None,
         };
 
         config.sanitized()