Tests and Nits

Author: grandizzy

crates/config/src/invariant.rs

@@ -37,6 +37,8 @@ pub struct InvariantConfig {
     pub timeout: Option<u32>,
     /// Display counterexample as solidity calls.
     pub show_solidity: bool,
+    /// Continue invariant run until all invariants declared in current test suite breaks.
+    pub continuous_run: bool,
 }
 
 impl Default for InvariantConfig {
@@ -55,6 +57,7 @@ impl Default for InvariantConfig {
             show_metrics: true,
             timeout: None,
             show_solidity: false,
+            continuous_run: false,
         }
     }
 }

crates/forge/src/result.rs

@@ -396,6 +396,9 @@ pub struct TestResult {
     /// still be successful (i.e self.success == true) when it's expected to fail.
     pub reason: Option<String>,
 
+    /// This field will be populated if there are additional invariant broken besides the main one.
+    pub other_failures: Vec<String>,
+
     /// Minimal reproduction test case for failing test
     pub counterexample: Option<CounterExample>,
 
@@ -484,6 +487,12 @@ impl fmt::Display for TestResult {
                 } else {
                     s.push(']');
                 }
+                if !self.other_failures.is_empty() {
+                    writeln!(s).unwrap();
+                    for failure in &self.other_failures {
+                        writeln!(s, "{failure}").unwrap();
+                    }
+                }
                 s.red().wrap().fmt(f)
             }
         }
@@ -679,6 +688,7 @@ impl TestResult {
         gas_report_traces: Vec<Vec<CallTraceArena>>,
         success: bool,
         reason: Option<String>,
+        other_failures: Vec<String>,
         counterexample: Option<CounterExample>,
         cases: Vec<FuzzedCases>,
         reverts: usize,
@@ -697,6 +707,7 @@ impl TestResult {
             false => TestStatus::Failure,
         };
         self.reason = reason;
+        self.other_failures = other_failures;
         self.counterexample = counterexample;
         self.gas_report_traces = gas_report_traces;
     }

crates/forge/src/runner.rs

@@ -772,10 +772,19 @@ impl<'a> FunctionRunner<'a> {
             identified_contracts,
             &self.cr.mcr.known_contracts,
         );
+
+        // Filter out additional invariants to test if we already have a persisted failure.
         let invariant_contract = InvariantContract {
             address: self.address,
             invariant_fn: func,
-            invariant_fns: invariants,
+            invariant_fns: invariants
+                .into_iter()
+                .filter(|(invariant_fn, _)| {
+                    *invariant_fn == func
+                        || (invariant_config.continuous_run
+                            && !canonicalized(failure_dir.join(invariant_fn.name.clone())).exists())
+                })
+                .collect(),
             call_after_invariant,
             abi: &self.cr.contract.abi,
         };
@@ -893,6 +902,7 @@ impl<'a> FunctionRunner<'a> {
             .errors
             .get(&invariant_contract.invariant_fn.name)
             .and_then(|err| err.revert_reason());
+        let mut other_failures = vec![];
 
         if success {
             // If invariants ran successfully, replay the last run to collect logs and
@@ -965,12 +975,20 @@ impl<'a> FunctionRunner<'a> {
                     continue;
                 }
 
-                // Generate counterexamples for other invariants broken.
-                if let Some(error) = invariant_result.errors.get(&invariant.name)
+                // Generate counterexamples for broken invariant, if there is no failure persisted
+                // already.
+                let persisted_failure = canonicalized(failure_dir.join(invariant.name.clone()));
+                if !persisted_failure.exists()
+                    && let Some(error) = invariant_result.errors.get(&invariant.name)
                     && let InvariantFuzzError::BrokenInvariant(case_data)
                     | InvariantFuzzError::Revert(case_data) = error
                     && let TestError::Fail(_, ref calls) = case_data.test_error
                 {
+                    other_failures.push(format!(
+                        "{}: {}",
+                        invariant.name,
+                        error.revert_reason().unwrap_or_default()
+                    ));
                     match generate_counterexample(
                         self.clone_executor(),
                         &self.cr.mcr.known_contracts,
@@ -982,7 +1000,7 @@ impl<'a> FunctionRunner<'a> {
                             // Persist error in invariant failure dir.
                             record_invariant_failure(
                                 failure_dir.as_path(),
-                                canonicalized(failure_dir.join(invariant.name.clone())).as_path(),
+                                persisted_failure.as_path(),
                                 &call_sequence,
                                 test_bytecode,
                             );
@@ -999,6 +1017,7 @@ impl<'a> FunctionRunner<'a> {
             invariant_result.gas_report_traces,
             success,
             reason,
+            other_failures,
             counterexample,
             invariant_result.cases,
             invariant_result.reverts,

crates/forge/tests/cli/config.rs

@@ -197,6 +197,7 @@ show_edge_coverage = false
 failure_persist_dir = "cache/invariant"
 show_metrics = true
 show_solidity = false
+continuous_run = false
 
 [labels]
 
@@ -1272,7 +1273,8 @@ forgetest_init!(test_default_config, |prj, cmd| {
     "failure_persist_dir": "cache/invariant",
     "show_metrics": true,
     "timeout": null,
-    "show_solidity": false
+    "show_solidity": false,
+    "continuous_run": false
   },
   "ffi": false,
   "allow_internal_expect_revert": false,

crates/forge/tests/cli/test_cmd/invariant/mod.rs

@@ -991,3 +991,83 @@ Ran 3 test suites [ELAPSED]: 6 tests passed, 0 failed, 0 skipped (6 total tests)
         prj.root().join("fuzz_corpus").join("Counter2Test").join("testFuzz_SetNumber").exists()
     );
 });
+
+forgetest_init!(continous_run, |prj, cmd| {
+    prj.update_config(|config| {
+        config.invariant.runs = 10;
+        config.invariant.depth = 100;
+        config.invariant.continuous_run = true;
+    });
+    prj.add_source(
+        "Counter.sol",
+        r#"
+contract Counter {
+    uint256 public cond;
+
+    function work(uint256 x) public {
+        if (x % 2 != 0 && x < 9000) {
+            cond++;
+        } else {
+            revert();
+        }
+    }
+}
+   "#,
+    );
+    prj.add_test(
+        "CounterTest.t.sol",
+        r#"
+import {Test} from "forge-std/Test.sol";
+import {Counter} from "../src/Counter.sol";
+
+contract CounterTest is Test {
+    Counter public counter;
+
+    function setUp() public {
+        counter = new Counter();
+    }
+
+    function invariant_cond1() public view {
+        require(counter.cond() < 10, "condition 1 met");
+    }
+
+    function invariant_cond2() public view {
+        require(counter.cond() < 15, "condition 2 met");
+    }
+
+    function invariant_cond3() public view {
+        require(counter.cond() < 5, "condition 3 met");
+    }
+
+    function invariant_cond4() public view {
+        require(counter.cond() < 111111, "condition 4 met");
+    }
+
+    /// forge-config: default.invariant.fail-on-revert = true
+    function invariant_cond5() public view {
+        require(counter.cond() < 111111, "condition 5 met");
+    }
+}
+   "#,
+    );
+
+    // Check that running single `invariant_cond3` test continue to run until it breaks all other
+    // invariants.
+    cmd.args(["test", "--mt", "invariant_cond3"]).assert_failure().stdout_eq(str![[r#"
+[COMPILING_FILES] with [SOLC_VERSION]
+[SOLC_VERSION] [ELAPSED]
+Compiler run successful!
+
+Ran 1 test for test/CounterTest.t.sol:CounterTest
+[FAIL: condition 3 met]
+	[Sequence] (original: 5, shrunk: 5)
+...
+
+invariant_cond1: condition 1 met
+invariant_cond2: condition 2 met
+invariant_cond5: EvmError: Revert
+ invariant_cond3() (runs: 10, calls: 1000, reverts: [..])
+...
+
+"#]]);
+});

crates/forge/tests/fixtures/SimpleContractTestNonVerbose.json

@@ -5,6 +5,7 @@
       "test()": {
         "status": "Success",
         "reason": null,
+        "other_failures": [],
         "counterexample": null,
         "logs": [],
         "decoded_logs": [],

crates/forge/tests/fixtures/SimpleContractTestVerbose.json

@@ -5,6 +5,7 @@
       "test()": {
         "status": "Success",
         "reason": null,
+        "other_failures": [],
         "counterexample": null,
         "logs": [
           {