diff --git a/crates/evm/fuzz/src/lib.rs b/crates/evm/fuzz/src/lib.rs index 7910de970af6d..f8e52b2fd17b8 100644 --- a/crates/evm/fuzz/src/lib.rs +++ b/crates/evm/fuzz/src/lib.rs @@ -199,7 +199,7 @@ impl fmt::Display for BaseCounterExample { } /// The outcome of a fuzz test -#[derive(Debug)] +#[derive(Debug, Default)] pub struct FuzzTestResult { /// we keep this for the debugger pub first_case: FuzzCase, diff --git a/crates/forge/src/result.rs b/crates/forge/src/result.rs index 694d26782fd4e..f0f6f45ec8a4b 100644 --- a/crates/forge/src/result.rs +++ b/crates/forge/src/result.rs @@ -684,6 +684,33 @@ impl TestResult { self.gas_report_traces = gas_report_traces; } + /// Returns the result for a table test. Merges table test execution results (logs, labeled + /// addresses, traces and coverages) in initial setup results. + pub fn table_result(&mut self, result: FuzzTestResult) { + self.kind = TestKind::Table { + median_gas: result.median_gas(false), + mean_gas: result.mean_gas(false), + runs: result.gas_by_case.len(), + }; + + // Record logs, labels, traces and merge coverages. + extend!(self, result, TraceKind::Execution); + + self.status = if result.skipped { + TestStatus::Skipped + } else if result.success { + TestStatus::Success + } else { + TestStatus::Failure + }; + self.reason = result.reason; + self.counterexample = result.counterexample; + self.duration = Duration::default(); + self.gas_report_traces = result.gas_report_traces.into_iter().map(|t| vec![t]).collect(); + self.breakpoints = result.breakpoints.unwrap_or_default(); + self.deprecated_cheatcodes = result.deprecated_cheatcodes; + } + /// Returns `true` if this is the result of a fuzz test pub fn is_fuzz(&self) -> bool { matches!(self.kind, TestKind::Fuzz { .. }) @@ -724,6 +751,11 @@ pub enum TestKindReport { metrics: Map, failed_corpus_replays: usize, }, + Table { + runs: usize, + mean_gas: u64, + median_gas: u64, + }, } impl fmt::Display for TestKindReport { @@ -752,6 +784,9 @@ impl fmt::Display for TestKindReport { write!(f, "(runs: {runs}, calls: {calls}, reverts: {reverts})") } } + Self::Table { runs, mean_gas, median_gas } => { + write!(f, "(runs: {runs}, μ: {mean_gas}, ~: {median_gas})") + } } } } @@ -762,7 +797,7 @@ impl TestKindReport { match *self { Self::Unit { gas } => gas, // We use the median for comparisons - Self::Fuzz { median_gas, .. } => median_gas, + Self::Fuzz { median_gas, .. } | Self::Table { median_gas, .. } => median_gas, // We return 0 since it's not applicable Self::Invariant { .. } => 0, } @@ -791,6 +826,8 @@ pub enum TestKind { metrics: Map, failed_corpus_replays: usize, }, + /// A table test. + Table { runs: usize, mean_gas: u64, median_gas: u64 }, } impl Default for TestKind { @@ -821,6 +858,9 @@ impl TestKind { failed_corpus_replays: *failed_corpus_replays, } } + Self::Table { runs, mean_gas, median_gas } => { + TestKindReport::Table { runs: *runs, mean_gas: *mean_gas, median_gas: *median_gas } + } } } } diff --git a/crates/forge/src/runner.rs b/crates/forge/src/runner.rs index f94e4c2d980c2..14fbbf66e5592 100644 --- a/crates/forge/src/runner.rs +++ b/crates/forge/src/runner.rs @@ -2,7 +2,8 @@ use crate::{ MultiContractRunner, TestFilter, - fuzz::BaseCounterExample, + coverage::HitMaps, + fuzz::{BaseCounterExample, FuzzTestResult}, multi_runner::{TestContract, TestRunnerConfig}, progress::{TestsProgress, start_fuzz_progress}, result::{SuiteResult, TestResult, TestSetup}, @@ -640,6 +641,8 @@ impl<'a> FunctionRunner<'a> { fixtures_len as u32, ); + let mut result = FuzzTestResult::default(); + for i in 0..fixtures_len { if self.tcfg.fail_fast.should_stop() { return self.result; @@ -671,24 +674,33 @@ impl<'a> FunctionRunner<'a> { } }; + result.gas_by_case.push((raw_call_result.gas_used, raw_call_result.stipend)); + result.logs.extend(raw_call_result.logs.clone()); + result.labels.extend(raw_call_result.labels.clone()); + HitMaps::merge_opt(&mut result.line_coverage, raw_call_result.line_coverage.clone()); + let is_success = self.executor.is_raw_call_mut_success(self.address, &mut raw_call_result, false); // Record counterexample if test fails. if !is_success { - self.result.counterexample = + result.counterexample = Some(CounterExample::Single(BaseCounterExample::from_fuzz_call( Bytes::from(func.abi_encode_input(&args).unwrap()), args, raw_call_result.traces.clone(), ))); - self.result.single_result(false, reason, raw_call_result); + result.reason = reason; + result.traces = raw_call_result.traces; + self.result.table_result(result); return self.result; } // If it's the last iteration and all other runs succeeded, then use last call result // for logs and traces. if i == fixtures_len - 1 { - self.result.single_result(true, None, raw_call_result); + result.success = true; + result.traces = raw_call_result.traces; + self.result.table_result(result); return self.result; } } diff --git a/crates/forge/tests/it/table.rs b/crates/forge/tests/it/table.rs index fcff0466ad86b..21039581c50ab 100644 --- a/crates/forge/tests/it/table.rs +++ b/crates/forge/tests/it/table.rs @@ -69,25 +69,25 @@ Compiler run successful! Ran 8 tests for test/CounterTable.t.sol:CounterTableTest [FAIL: 2 fixtures defined for diffSwap (expected 10)] tableMultipleParamsDifferentFixturesFail(uint256,bool) ([GAS]) -[FAIL: Cannot swap; counterexample: calldata=0x717892ca00000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000001 args=[1, true]] tableMultipleParamsFail(uint256,bool) ([GAS]) +[FAIL: Cannot swap; counterexample: calldata=0x717892ca00000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000001 args=[1, true]] tableMultipleParamsFail(uint256,bool) (runs: 1, [AVG_GAS]) Traces: [..] CounterTableTest::tableMultipleParamsFail(1, true) └─ ← [Revert] Cannot swap [FAIL: No fixture defined for param noSwap] tableMultipleParamsNoParamFail(uint256,bool) ([GAS]) -[PASS] tableMultipleParamsPass(uint256,bool) ([GAS]) +[PASS] tableMultipleParamsPass(uint256,bool) (runs: 10, [AVG_GAS]) Traces: [..] CounterTableTest::tableMultipleParamsPass(10, true) ├─ [..] Counter::increment() │ └─ ← [Stop] └─ ← [Stop] -[FAIL: Amount cannot be 10; counterexample: calldata=0x44fa2375000000000000000000000000000000000000000000000000000000000000000a args=[10]] tableSingleParamFail(uint256) ([GAS]) +[FAIL: Amount cannot be 10; counterexample: calldata=0x44fa2375000000000000000000000000000000000000000000000000000000000000000a args=[10]] tableSingleParamFail(uint256) (runs: 10, [AVG_GAS]) Traces: [..] CounterTableTest::tableSingleParamFail(10) └─ ← [Revert] Amount cannot be 10 -[PASS] tableSingleParamPass(uint256) ([GAS]) +[PASS] tableSingleParamPass(uint256) (runs: 10, [AVG_GAS]) Traces: [..] CounterTableTest::tableSingleParamPass(10) ├─ [..] Counter::increment() @@ -103,9 +103,9 @@ Ran 1 test suite [ELAPSED]: 2 tests passed, 6 failed, 0 skipped (8 total tests) Failing tests: Encountered 6 failing tests in test/CounterTable.t.sol:CounterTableTest [FAIL: 2 fixtures defined for diffSwap (expected 10)] tableMultipleParamsDifferentFixturesFail(uint256,bool) ([GAS]) -[FAIL: Cannot swap; counterexample: calldata=0x717892ca00000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000001 args=[1, true]] tableMultipleParamsFail(uint256,bool) ([GAS]) +[FAIL: Cannot swap; counterexample: calldata=0x717892ca00000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000001 args=[1, true]] tableMultipleParamsFail(uint256,bool) (runs: 1, [AVG_GAS]) [FAIL: No fixture defined for param noSwap] tableMultipleParamsNoParamFail(uint256,bool) ([GAS]) -[FAIL: Amount cannot be 10; counterexample: calldata=0x44fa2375000000000000000000000000000000000000000000000000000000000000000a args=[10]] tableSingleParamFail(uint256) ([GAS]) +[FAIL: Amount cannot be 10; counterexample: calldata=0x44fa2375000000000000000000000000000000000000000000000000000000000000000a args=[10]] tableSingleParamFail(uint256) (runs: 10, [AVG_GAS]) [FAIL: Table test should have at least one parameter] tableWithNoParamFail() ([GAS]) [FAIL: Table test should have at least one fixture] tableWithParamNoFixtureFail(uint256) ([GAS]) @@ -113,3 +113,107 @@ Encountered a total of 6 failing tests, 2 tests succeeded "#]]); }); + +// Table tests should show logs and contribute to coverage. +// +forgetest_init!(should_show_logs_and_add_coverage, |prj, cmd| { + prj.wipe_contracts(); + prj.add_source( + "Counter.sol", + r#" +contract Counter { + uint256 public number; + + function setNumber(uint256 a, uint256 b) public { + if (a == 1) { + number = b + 1; + } else if (a == 2) { + number = b + 2; + } else if (a == 3) { + number = b + 3; + } else { + number = a + b; + } + } +} + "#, + ); + prj.add_test( + "CounterTest.t.sol", + r#" +import "forge-std/Test.sol"; +import {Counter} from "../src/Counter.sol"; + +contract CounterTest is Test { + struct TestCase { + uint256 a; + uint256 b; + uint256 expected; + } + + Counter public counter; + + function setUp() public { + counter = new Counter(); + } + + function fixtureNumbers() public pure returns (TestCase[] memory) { + TestCase[] memory entries = new TestCase[](4); + entries[0] = TestCase(1, 5, 6); + entries[1] = TestCase(2, 10, 12); + entries[2] = TestCase(3, 11, 14); + entries[3] = TestCase(4, 11, 15); + return entries; + } + + function tableSetNumberTest(TestCase memory numbers) public { + console.log("expected", numbers.expected); + counter.setNumber(numbers.a, numbers.b); + require(counter.number() == numbers.expected, "test failed"); + } +} + "#, + ); + + cmd.args(["test", "-vvv"]).assert_success().stdout_eq(str![[r#" +[COMPILING_FILES] with [SOLC_VERSION] +[SOLC_VERSION] [ELAPSED] +Compiler run successful! + +Ran 1 test for test/CounterTest.t.sol:CounterTest +[PASS] tableSetNumberTest((uint256,uint256,uint256)) (runs: 4, [AVG_GAS]) +Logs: + expected 6 + expected 12 + expected 14 + expected 15 + +Suite result: ok. 1 passed; 0 failed; 0 skipped; [ELAPSED] + +Ran 1 test suite [ELAPSED]: 1 tests passed, 0 failed, 0 skipped (1 total tests) + +"#]]); + + cmd.forge_fuse().args(["coverage"]).assert_success().stdout_eq(str![[r#" +[COMPILING_FILES] with [SOLC_VERSION] +[SOLC_VERSION] [ELAPSED] +Compiler run successful! +Analysing contracts... +Running tests... + +Ran 1 test for test/CounterTest.t.sol:CounterTest +[PASS] tableSetNumberTest((uint256,uint256,uint256)) (runs: 4, [AVG_GAS]) +Suite result: ok. 1 passed; 0 failed; 0 skipped; [ELAPSED] + +Ran 1 test suite [ELAPSED]: 1 tests passed, 0 failed, 0 skipped (1 total tests) + +╭-----------------+---------------+---------------+---------------+---------------╮ +| File | % Lines | % Statements | % Branches | % Funcs | ++=================================================================================+ +| src/Counter.sol | 100.00% (8/8) | 100.00% (7/7) | 100.00% (6/6) | 100.00% (1/1) | +|-----------------+---------------+---------------+---------------+---------------| +| Total | 100.00% (8/8) | 100.00% (7/7) | 100.00% (6/6) | 100.00% (1/1) | +╰-----------------+---------------+---------------+---------------+---------------╯ + +"#]]); +});