feat(benches): forge test --isolate (foundry-rs#11037)

yash-atreya · web-flow · commit 450f2c761f05 · 2025-07-23T19:19:52.000+05:30
diff --git a/.github/scripts/combine-benchmarks.sh b/.github/scripts/combine-benchmarks.sh
@@ -126,7 +126,7 @@ EOF
 FIRST_FILE=1
 SYSTEM_INFO=""
 
-for bench_file in "forge_test_bench.md" "forge_build_bench.md" "forge_coverage_bench.md"; do
+for bench_file in "forge_test_bench.md" "forge_isolate_test_bench.md" "forge_build_bench.md" "forge_coverage_bench.md"; do
     if [ -f "$OUTPUT_DIR/$bench_file" ]; then
         echo "Processing $bench_file..."
         
@@ -135,6 +135,9 @@ for bench_file in "forge_test_bench.md" "forge_build_bench.md" "forge_coverage_b
             "forge_test_bench.md")
                 SECTION_NAME="Forge Test"
                 ;;
+            "forge_isolate_test_bench.md")
+                SECTION_NAME="Forge Test (Isolated)"
+                ;;
             "forge_build_bench.md")
                 SECTION_NAME="Forge Build"
                 ;;
diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml
@@ -83,6 +83,20 @@ jobs:
             --benchmarks forge_test,forge_fuzz_test \
             --output-file forge_test_bench.md
 
+      - name: Run forge isolate test benchmarks
+        env:
+          FOUNDRY_DIR: ${{ github.workspace }}/.foundry
+        run: |
+          VERSIONS="${{ github.event.inputs.versions || 'stable,nightly' }}"
+          # Isolate tests default to Vectorized/solady but can be overridden
+          REPOS="${{ github.event.inputs.repos || env.VECTORIZED_SOLADY }}"
+
+          ./target/release/foundry-bench --output-dir ./benches --force-install \
+            --versions $VERSIONS \
+            --repos $REPOS \
+            --benchmarks forge_isolate_test \
+            --output-file forge_isolate_test_bench.md
+
       - name: Run forge build benchmarks
         env:
           FOUNDRY_DIR: ${{ github.workspace }}/.foundry
@@ -124,6 +138,7 @@ jobs:
           name: benchmark-results
           path: |
             benches/forge_test_bench.md
+            benches/forge_isolate_test_bench.md
             benches/forge_build_bench.md
             benches/forge_coverage_bench.md
             benches/LATEST.md
diff --git a/benches/LATEST.md b/benches/LATEST.md
@@ -1,6 +1,6 @@
 # 📊 Foundry Benchmark Results
 
-**Generated at**: 2025-07-17 14:25:08 UTC
+**Generated at**: 2025-07-23 09:01:16 UTC
 
 ## Forge Test
 
@@ -11,19 +11,33 @@
 ### Foundry Versions
 
 - **stable**: forge Version: 1.2.3-stable (a813a2c 2025-06-08)
-- **nightly**: forge Version: 1.3.0-nightly (0af4341 2025-07-17)
+- **nightly**: forge Version: 1.3.0-nightly (cb8f3bf 2025-07-23)
 
 | Repository | stable | nightly |
 |------------|----------|----------|
-| ithacaxyz-account | 5.12 s | 5.15 s |
-| solady | 2.99 s | 2.87 s |
+| ithacaxyz-account | 5.44 s | 4.91 s |
+| solady | 3.02 s | 2.76 s |
 
 ## Forge Fuzz Test
 
 | Repository | stable | nightly |
 |------------|----------|----------|
-| ithacaxyz-account | 5.45 s | 5.20 s |
-| solady | 3.08 s | 3.00 s |
+| ithacaxyz-account | 5.27 s | 4.90 s |
+| solady | 2.89 s | 2.82 s |
+
+## Forge Test (Isolated)
+
+### Repositories Tested
+
+1. [Vectorized/solady](https://github.com/Vectorized/solady)
+### Foundry Versions
+
+- **stable**: forge Version: 1.2.3-stable (a813a2c 2025-06-08)
+- **nightly**: forge Version: 1.3.0-nightly (cb8f3bf 2025-07-23)
+
+| Repository | stable | nightly |
+|------------|----------|----------|
+| solady | 3.21 s | 2.88 s |
 
 ## Forge Build
 
@@ -34,21 +48,21 @@
 ### Foundry Versions
 
 - **stable**: forge Version: 1.2.3-stable (a813a2c 2025-06-08)
-- **nightly**: forge Version: 1.3.0-nightly (0af4341 2025-07-17)
+- **nightly**: forge Version: 1.3.0-nightly (cb8f3bf 2025-07-23)
 
 ### No Cache
 
 | Repository | stable | nightly |
 |------------|----------|----------|
-| ithacaxyz-account | 2.58 s | 2.83 s |
-| solady | 3.88 s | 3.98 s |
+| ithacaxyz-account | 2.58 s | 2.79 s |
+| solady | 3.79 s | 3.89 s |
 
 ### With Cache
 
 | Repository | stable | nightly |
 |------------|----------|----------|
-| ithacaxyz-account | 0.210 s | 0.443 s |
-| solady | 0.093 s | 0.195 s |
+| ithacaxyz-account | 0.209 s | 0.458 s |
+| solady | 0.096 s | 0.226 s |
 
 ## Forge Coverage
 
@@ -58,11 +72,11 @@
 ### Foundry Versions
 
 - **stable**: forge Version: 1.2.3-stable (a813a2c 2025-06-08)
-- **nightly**: forge Version: 1.3.0-nightly (0af4341 2025-07-17)
+- **nightly**: forge Version: 1.3.0-nightly (cb8f3bf 2025-07-23)
 
 | Repository | stable | nightly |
 |------------|----------|----------|
-| ithacaxyz-account | 22.03 s | 21.98 s |
+| ithacaxyz-account | 21.88 s | 22.52 s |
 
 ## System Information
 
diff --git a/benches/src/lib.rs b/benches/src/lib.rs
@@ -355,6 +355,25 @@ impl BenchmarkProject {
         )
     }
 
+    /// Benchmark forge test with --isolate flag
+    pub fn bench_forge_isolate_test(
+        &self,
+        version: &str,
+        runs: u32,
+        verbose: bool,
+    ) -> Result<HyperfineResult> {
+        // Build before running tests
+        self.hyperfine(
+            "forge_isolate_test",
+            version,
+            "forge test --isolate",
+            runs,
+            Some("forge build"),
+            None,
+            verbose,
+        )
+    }
+
     /// Get the root path of the project
     pub fn root(&self) -> &Path {
         &self.root_path
@@ -374,6 +393,7 @@ impl BenchmarkProject {
             "forge_build_with_cache" => self.bench_forge_build_with_cache(version, runs, verbose),
             "forge_fuzz_test" => self.bench_forge_fuzz_test(version, runs, verbose),
             "forge_coverage" => self.bench_forge_coverage(version, runs, verbose),
+            "forge_isolate_test" => self.bench_forge_isolate_test(version, runs, verbose),
             _ => eyre::bail!("Unknown benchmark: {}", benchmark),
         }
     }
diff --git a/benches/src/main.rs b/benches/src/main.rs
@@ -10,12 +10,13 @@ use foundry_common::sh_println;
 use rayon::prelude::*;
 use std::{fs, path::PathBuf, process::Command, sync::Mutex};
 
-const ALL_BENCHMARKS: [&str; 5] = [
+const ALL_BENCHMARKS: [&str; 6] = [
     "forge_test",
     "forge_build_no_cache",
     "forge_build_with_cache",
     "forge_fuzz_test",
     "forge_coverage",
+    "forge_isolate_test",
 ];
 
 /// Foundry Benchmark Runner
diff --git a/benches/src/results.rs b/benches/src/results.rs
@@ -231,6 +231,7 @@ pub fn format_benchmark_name(name: &str) -> String {
         "forge_build_with_cache" => "Forge Build (With Cache)",
         "forge_fuzz_test" => "Forge Fuzz Test",
         "forge_coverage" => "Forge Coverage",
+        "forge_isolate_test" => "Forge Test (Isolated)",
         _ => name,
     }
     .to_string()

Original file line number	Diff line number	Diff line change
`@@ -231,6 +231,7 @@ pub fn format_benchmark_name(name: &str) -> String {`
`231`	`231`	`"forge_build_with_cache" => "Forge Build (With Cache)",`
`232`	`232`	`"forge_fuzz_test" => "Forge Fuzz Test",`
`233`	`233`	`"forge_coverage" => "Forge Coverage",`
	`234`	`+ "forge_isolate_test" => "Forge Test (Isolated)",`
`234`	`235`	`_ => name,`
`235`	`236`	`}`
`236`	`237`	`.to_string()`