Skip to content

Commit a616c7a

Browse files
yash-atreyayash-atreya
authored
fix(benches): forge build benches (foundry-rs#11036)
* run.sh + test and build bench * bench seq: test, fuzz and build * fix(`benches`): use prepare in forge build benches * coverage results * fix(`benches`): forge build prepare and cleanup * latest run * rm intermediate results * rm sequential run script * LINT_ON_BUILD=false * fix: FOUNDRY_LINT_LINT_ON_BUILD=false * latest build benches * fix
1 parent 9c3feff commit a616c7a

File tree

2 files changed

+75
-47
lines changed

2 files changed

+75
-47
lines changed

‎benches/LATEST.md‎

Lines changed: 50 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -1,86 +1,93 @@
11
# 📊 Foundry Benchmark Results
22

3-
**Generated at**: 2025-07-23 09:01:16 UTC
3+
**Generated at**: 2025-07-18 23:05:00 UTC
44

55
## Forge Test
66

77
### Repositories Tested
88

99
1. [ithacaxyz/account](https://github.com/ithacaxyz/account)
1010
2. [Vectorized/solady](https://github.com/Vectorized/solady)
11+
3. [sparkdotfi/spark-psm](https://github.com/sparkdotfi/spark-psm)
12+
4. [Uniswap/v4-core](https://github.com/Uniswap/v4-core)
13+
1114
### Foundry Versions
1215

13-
- **stable**: forge Version: 1.2.3-stable (a813a2c 2025-06-08)
14-
- **nightly**: forge Version: 1.3.0-nightly (cb8f3bf 2025-07-23)
16+
- **v1.2.3**: forge Version: 1.2.3-v1.2.3 (a813a2c 2025-06-08)
17+
- **nightly-05918765cb239024e9ca396825abb9f46257419a**: forge Version: 1.2.3-nightly (0591876 2025-07-15)
18+
- **nightly-0af43412f809c998d8b2fe69a1c9a789b7ebd085**: forge Version: 1.3.0-nightly (0af4341 2025-07-17)
1519

16-
| Repository | stable | nightly |
17-
|------------|----------|----------|
18-
| ithacaxyz-account | 5.44 s | 4.91 s |
19-
| solady | 3.02 s | 2.76 s |
20+
| Repository | v1.2.3 | nightly-05918765cb239024e9ca396825abb9f46257419a | nightly-0af43412f809c998d8b2fe69a1c9a789b7ebd085 |
21+
| -------------------- | ------- | ------------------------------------------------ | ------------------------------------------------ |
22+
| ithacaxyz-account | 6.74 s | 3.24 s | 3.52 s |
23+
| solady | 2.77 s | 2.76 s | 2.71 s |
24+
| sparkdotfi-spark-psm | 1m 3.7s | 1m 1.2s | 1m 5.3s |
25+
| Uniswap-v4-core | 8.04 s | 7.58 s | 8.44 s |
2026

2127
## Forge Fuzz Test
2228

23-
| Repository | stable | nightly |
24-
|------------|----------|----------|
25-
| ithacaxyz-account | 5.27 s | 4.90 s |
26-
| solady | 2.89 s | 2.82 s |
27-
28-
## Forge Test (Isolated)
29-
30-
### Repositories Tested
31-
32-
1. [Vectorized/solady](https://github.com/Vectorized/solady)
33-
### Foundry Versions
34-
35-
- **stable**: forge Version: 1.2.3-stable (a813a2c 2025-06-08)
36-
- **nightly**: forge Version: 1.3.0-nightly (cb8f3bf 2025-07-23)
37-
38-
| Repository | stable | nightly |
39-
|------------|----------|----------|
40-
| solady | 3.21 s | 2.88 s |
29+
| Repository | v1.2.3 | nightly-05918765cb239024e9ca396825abb9f46257419a | nightly-0af43412f809c998d8b2fe69a1c9a789b7ebd085 |
30+
| -------------------- | ------ | ------------------------------------------------ | ------------------------------------------------ |
31+
| ithacaxyz-account | 3.92 s | 3.37 s | 3.52 s |
32+
| solady | 2.96 s | 2.66 s | 2.82 s |
33+
| sparkdotfi-spark-psm | 3.68 s | 3.52 s | 3.63 s |
34+
| Uniswap-v4-core | 8.06 s | 7.82 s | 8.31 s |
4135

4236
## Forge Build
4337

4438
### Repositories Tested
4539

4640
1. [ithacaxyz/account](https://github.com/ithacaxyz/account)
4741
2. [Vectorized/solady](https://github.com/Vectorized/solady)
42+
3. [sparkdotfi/spark-psm](https://github.com/sparkdotfi/spark-psm)
43+
4. [Uniswap/v4-core](https://github.com/Uniswap/v4-core)
44+
4845
### Foundry Versions
4946

50-
- **stable**: forge Version: 1.2.3-stable (a813a2c 2025-06-08)
51-
- **nightly**: forge Version: 1.3.0-nightly (cb8f3bf 2025-07-23)
47+
- **v1.2.3**: forge Version: 1.2.3-v1.2.3 (a813a2c 2025-06-08)
48+
- **nightly-05918765cb239024e9ca396825abb9f46257419a**: forge Version: 1.2.3-nightly (0591876 2025-07-15)
49+
- **nightly-0af43412f809c998d8b2fe69a1c9a789b7ebd085**: forge Version: 1.3.0-nightly (0af4341 2025-07-17)
5250

5351
### No Cache
5452

55-
| Repository | stable | nightly |
56-
|------------|----------|----------|
57-
| ithacaxyz-account | 2.58 s | 2.79 s |
58-
| solady | 3.79 s | 3.89 s |
53+
| Repository | v1.2.3 | nightly-05918765cb239024e9ca396825abb9f46257419a | nightly-0af43412f809c998d8b2fe69a1c9a789b7ebd085 |
54+
| -------------------- | -------- | ------------------------------------------------ | ------------------------------------------------ |
55+
| ithacaxyz-account | 9.32 s | 9.32 s | 9.46 s |
56+
| solady | 15.01 s | 14.97 s | 14.81 s |
57+
| sparkdotfi-spark-psm | 13.42 s | 13.29 s | 13.28 s |
58+
| Uniswap-v4-core | 2m 11.6s | 2m 7.4s | 2m 6.3s |
5959

6060
### With Cache
6161

62-
| Repository | stable | nightly |
63-
|------------|----------|----------|
64-
| ithacaxyz-account | 0.209 s | 0.458 s |
65-
| solady | 0.096 s | 0.226 s |
62+
| Repository | v1.2.3 | nightly-05918765cb239024e9ca396825abb9f46257419a | nightly-0af43412f809c998d8b2fe69a1c9a789b7ebd085 |
63+
| -------------------- | ------- | ------------------------------------------------ | ------------------------------------------------ |
64+
| ithacaxyz-account | 0.203 s | 0.205 s | 0.201 s |
65+
| solady | 0.092 s | 0.090 s | 0.094 s |
66+
| sparkdotfi-spark-psm | 0.170 s | 0.174 s | 0.173 s |
67+
| Uniswap-v4-core | 0.135 s | 0.139 s | 0.135 s |
6668

6769
## Forge Coverage
6870

6971
### Repositories Tested
7072

7173
1. [ithacaxyz/account](https://github.com/ithacaxyz/account)
74+
2. [sparkdotfi/spark-psm](https://github.com/sparkdotfi/spark-psm)
75+
3. [Uniswap/v4-core](https://github.com/Uniswap/v4-core)
76+
7277
### Foundry Versions
7378

74-
- **stable**: forge Version: 1.2.3-stable (a813a2c 2025-06-08)
75-
- **nightly**: forge Version: 1.3.0-nightly (cb8f3bf 2025-07-23)
79+
- **v1.2.3**: forge Version: 1.2.3-v1.2.3 (a813a2c 2025-06-08)
80+
- **nightly-05918765cb239024e9ca396825abb9f46257419a**: forge Version: 1.2.3-nightly (0591876 2025-07-15)
81+
- **nightly-0af43412f809c998d8b2fe69a1c9a789b7ebd085**: forge Version: 1.3.0-nightly (0af4341 2025-07-17)
7682

77-
| Repository | stable | nightly |
78-
|------------|----------|----------|
79-
| ithacaxyz-account | 21.88 s | 22.52 s |
83+
| Repository | v1.2.3 | nightly-05918765cb239024e9ca396825abb9f46257419a | nightly-0af43412f809c998d8b2fe69a1c9a789b7ebd085 |
84+
| -------------------- | -------- | ------------------------------------------------ | ------------------------------------------------ |
85+
| ithacaxyz-account | 16.33 s | 17.31 s | 16.43 s |
86+
| sparkdotfi-spark-psm | 3m 52.9s | 4m 12.8s | 4m 15.0s |
87+
| Uniswap-v4-core | 1m 40.6s | 1m 42.7s | 1m 47.5s |
8088

8189
## System Information
8290

83-
84-
- **OS**: linux
91+
- **OS**: macos
8592
- **CPU**: 8
86-
- **Rustc**: rustc 1.88.0 (6b00bc388 2025-06-23)
93+
- **Rustc**: rustc 1.90.0-nightly (3014e79f9 2025-07-15)

‎benches/src/lib.rs‎

Lines changed: 25 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -195,12 +195,14 @@ impl BenchmarkProject {
195195
/// * `command` - The command to benchmark
196196
/// * `runs` - Number of runs to perform
197197
/// * `setup` - Optional setup command to run before the benchmark series (e.g., "forge build")
198+
/// * `prepare` - Optional prepare command to run before each timing run (e.g., "forge clean")
198199
/// * `conclude` - Optional conclude command to run after each timing run (e.g., cleanup)
199200
/// * `verbose` - Whether to show command output
200201
///
201202
/// # Hyperfine flags used:
202203
/// * `--runs` - Number of timing runs
203204
/// * `--setup` - Execute before the benchmark series (not before each run)
205+
/// * `--prepare` - Execute before each timing run
204206
/// * `--conclude` - Execute after each timing run
205207
/// * `--export-json` - Export results to JSON for parsing
206208
/// * `--shell=bash` - Use bash for shell command execution
@@ -213,6 +215,7 @@ impl BenchmarkProject {
213215
command: &str,
214216
runs: u32,
215217
setup: Option<&str>,
218+
prepare: Option<&str>,
216219
conclude: Option<&str>,
217220
verbose: bool,
218221
) -> Result<HyperfineResult> {
@@ -240,6 +243,11 @@ impl BenchmarkProject {
240243
hyperfine_cmd.arg("--setup").arg(setup_cmd);
241244
}
242245

246+
// Add optional prepare command
247+
if let Some(prepare_cmd) = prepare {
248+
hyperfine_cmd.arg("--prepare").arg(prepare_cmd);
249+
}
250+
243251
// Add optional conclude command
244252
if let Some(conclude_cmd) = conclude {
245253
hyperfine_cmd.arg("--conclude").arg(conclude_cmd);
@@ -282,6 +290,7 @@ impl BenchmarkProject {
282290
runs,
283291
Some("forge build"),
284292
None,
293+
None,
285294
verbose,
286295
)
287296
}
@@ -293,8 +302,16 @@ impl BenchmarkProject {
293302
runs: u32,
294303
verbose: bool,
295304
) -> Result<HyperfineResult> {
296-
// No setup needed, uses existing cache
297-
self.hyperfine("forge_build_with_cache", version, "forge build", runs, None, None, verbose)
305+
self.hyperfine(
306+
"forge_build_with_cache",
307+
version,
308+
"FOUNDRY_LINT_LINT_ON_BUILD=false forge build",
309+
runs,
310+
None,
311+
Some("forge build"),
312+
None,
313+
verbose,
314+
)
298315
}
299316

300317
/// Benchmark forge build without cache
@@ -304,14 +321,15 @@ impl BenchmarkProject {
304321
runs: u32,
305322
verbose: bool,
306323
) -> Result<HyperfineResult> {
307-
// Clean before the benchmark series
324+
// Clean before each timing run
308325
self.hyperfine(
309326
"forge_build_no_cache",
310327
version,
311-
"forge build",
328+
"FOUNDRY_LINT_LINT_ON_BUILD=false forge build",
312329
runs,
313330
Some("forge clean"),
314331
None,
332+
Some("forge clean"),
315333
verbose,
316334
)
317335
}
@@ -331,6 +349,7 @@ impl BenchmarkProject {
331349
runs,
332350
Some("forge build"),
333351
None,
352+
None,
334353
verbose,
335354
)
336355
}
@@ -351,6 +370,7 @@ impl BenchmarkProject {
351370
runs,
352371
None,
353372
None,
373+
None,
354374
verbose,
355375
)
356376
}
@@ -370,6 +390,7 @@ impl BenchmarkProject {
370390
runs,
371391
Some("forge build"),
372392
None,
393+
None,
373394
verbose,
374395
)
375396
}

0 commit comments

Comments
 (0)