Skip to content

Commit d3baee2

Browse files
xlcqiweiii
xlc
and
qiweiii
authored
feat: add benchmark with ci (#365)
* feat: Add benchmark suite for core JAM components Adds a `JAMBenchmarks` executable using `swift-benchmark` to test the performance of erasure coding, shuffling, and STF application. This change includes necessary helpers, public API adjustments, and VS Code launch configurations. * fix * disable jemalloc * improve review * benchmark ci * add tests * fix benchmark ci * fix benchmark ci * fix benchmark ci * fix benchmark ci * fix benchmark ci * fix benchmark ci --------- Co-authored-by: Qiwei Yang <[email protected]>
1 parent cfe98d5 commit d3baee2

File tree

23 files changed

+3236
-263
lines changed

23 files changed

+3236
-263
lines changed

‎.github/workflows/benchmark-pr.yml‎

Lines changed: 158 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,158 @@
1+
name: Benchmark PR
2+
3+
on:
4+
workflow_dispatch:
5+
pull_request:
6+
branches: [ master ]
7+
8+
concurrency:
9+
group: ${{ github.workflow }}-${{ github.ref }}
10+
cancel-in-progress: true
11+
12+
env:
13+
BENCHMARK_DISABLE_JEMALLOC: true
14+
15+
jobs:
16+
benchmark-delta:
17+
name: Comparison
18+
runs-on: [self-hosted, linux]
19+
continue-on-error: true
20+
timeout-minutes: 60
21+
22+
steps:
23+
- name: Checkout Code
24+
uses: actions/checkout@v4
25+
with:
26+
fetch-depth: 0
27+
submodules: recursive
28+
29+
- run: sudo apt-get update
30+
- uses: awalsh128/cache-apt-pkgs-action@latest
31+
with:
32+
packages: librocksdb-dev libzstd-dev libbz2-dev liblz4-dev
33+
34+
- uses: aws-actions/configure-aws-credentials@v4
35+
with:
36+
aws-region: us-east-2
37+
38+
- name: Cache SPM
39+
uses: runs-on/cache@v4
40+
with:
41+
path: '**/.build'
42+
key: ${{ runner.os }}-spm-release-${{ hashFiles('**/Package.resolved') }}
43+
restore-keys: |
44+
${{ runner.os }}-spm-release-
45+
env:
46+
RUNS_ON_S3_BUCKET_CACHE: laminar-gh-action-cache
47+
48+
- name: Cache Cargo
49+
uses: actions/cache@v4
50+
with:
51+
path: |
52+
~/.cargo/bin/
53+
~/.cargo/registry/index/
54+
~/.cargo/registry/cache/
55+
~/.cargo/git/db/
56+
target/
57+
key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
58+
59+
- name: Cache bandersnatch_vrfs static lib
60+
uses: actions/cache@v4
61+
with:
62+
path: .lib/libbandersnatch_vrfs.a
63+
key: ${{ runner.os }}-libs-libbandersnatch-${{ hashFiles('Utils/Sources/bandersnatch/**') }}
64+
restore-keys: |
65+
${{ runner.os }}-libs-libbandersnatch
66+
67+
- name: Cache bls static lib
68+
uses: actions/cache@v4
69+
with:
70+
path: .lib/libbls.a
71+
key: ${{ runner.os }}-libs-libbls-${{ hashFiles('Utils/Sources/bls/**') }}
72+
restore-keys: |
73+
${{ runner.os }}-libs-libbls
74+
75+
- name: Cache erasure-coding static lib
76+
uses: actions/cache@v4
77+
with:
78+
path: .lib/libec.a
79+
key: ${{ runner.os }}-libs-libec-${{ hashFiles('Utils/Sources/erasure-coding/**') }}
80+
restore-keys: |
81+
${{ runner.os }}-libs-libec
82+
83+
- name: Setup Swift
84+
uses: SwiftyLab/setup-swift@latest
85+
86+
- name: Setup Rust
87+
uses: dtolnay/rust-toolchain@nightly
88+
with:
89+
components: rustfmt
90+
91+
- name: Build dependencies
92+
run: make deps
93+
94+
- name: Check if benchmarks exist and setup environment
95+
run: |
96+
[ -d "JAMTests/Benchmarks" ] && echo "hasBenchmark=1" >> $GITHUB_ENV
97+
98+
- name: Switch to branch 'master'
99+
if: ${{ env.hasBenchmark == '1' }}
100+
run: |
101+
git checkout master
102+
103+
- name: Run benchmarks for branch 'master'
104+
if: ${{ env.hasBenchmark == '1' }}
105+
run: |
106+
cd JAMTests
107+
# Check if benchmarks exist on master and if the benchmark plugin is available
108+
if [ -d "Benchmarks" ] && swift package plugin --list 2>/dev/null | grep -q benchmark; then
109+
swift package --allow-writing-to-directory .benchmarkBaselines/ benchmark baseline update master --no-progress --quiet
110+
else
111+
echo "Benchmarks don't exist on master branch yet or benchmark plugin not available - skipping master baseline"
112+
fi
113+
114+
- name: Switch to PR branch
115+
if: ${{ env.hasBenchmark == '1' }}
116+
run: |
117+
git checkout ${{ github.head_ref }}
118+
119+
- name: Run benchmarks for PR branch
120+
if: ${{ env.hasBenchmark == '1' }}
121+
run: |
122+
cd JAMTests
123+
swift package --allow-writing-to-directory .benchmarkBaselines/ benchmark baseline update pull_request --no-progress --quiet
124+
125+
- name: Compare PR and master
126+
if: ${{ env.hasBenchmark == '1' }}
127+
id: benchmark
128+
run: |
129+
cd JAMTests
130+
echo "# 📊 Benchmark Results"
131+
echo "Generated on: $(date)"
132+
echo ""
133+
echo "exitStatus=1" >> $GITHUB_ENV
134+
135+
# Check if master baseline exists for comparison
136+
if [ -f ".benchmarkBaselines/BenchmarkTestVectors/master/results.json" ]; then
137+
echo "## 🔄 Comparison with Master Branch"
138+
echo ""
139+
swift package benchmark baseline check master pull_request --format markdown
140+
echo "exitStatus=0" >> $GITHUB_ENV
141+
else
142+
echo "## 🆕 First Benchmark Run"
143+
echo ""
144+
echo "⚠️ Master baseline not found. This appears to be the first benchmark run."
145+
echo "The following results will become the baseline for future comparisons:"
146+
echo ""
147+
148+
# Run benchmarks to generate results for first time
149+
echo "### Benchmark Results:"
150+
swift package benchmark --format markdown 2>&1 || true
151+
echo "exitStatus=0" >> $GITHUB_ENV
152+
fi
153+
continue-on-error: true
154+
155+
- name: Exit with correct status
156+
if: ${{ env.hasBenchmark == '1' }}
157+
run: |
158+
exit ${{ env.exitStatus }}

‎.github/workflows/ci.yml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,9 @@ concurrency:
1010
group: ${{ github.workflow }}-${{ github.ref }}
1111
cancel-in-progress: true
1212

13+
env:
14+
BENCHMARK_DISABLE_JEMALLOC: true
15+
1316
jobs:
1417
lint:
1518
name: Swift Lint

‎.github/workflows/coverage.yml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,9 @@ concurrency:
1010
group: ${{ github.workflow }}-${{ github.ref }}
1111
cancel-in-progress: true
1212

13+
env:
14+
BENCHMARK_DISABLE_JEMALLOC: true
15+
1316
jobs:
1417
coverage:
1518
name: Code Coverage

‎.github/workflows/polka-codes-review-pr.yml‎

Lines changed: 16 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,8 @@ concurrency:
1919
cancel-in-progress: true
2020

2121
jobs:
22-
handle:
22+
review_gemini:
23+
name: Review With Gemini 2.5 Pro
2324
timeout-minutes: 40
2425
runs-on: ubuntu-latest
2526
steps:
@@ -28,11 +29,12 @@ jobs:
2829
with:
2930
fetch-depth: 0
3031

31-
- uses: google-github-actions/auth@v2
32+
- name: Google Auth
33+
uses: google-github-actions/auth@v2
3234
with:
3335
credentials_json: ${{ secrets.GOOGLE_CREDENTIALS }}
3436

35-
- name: Review PR With Gemini 2.5 Pro
37+
- name: Run Polka Codes (Gemini)
3638
uses: polka-codes/action@master
3739
with:
3840
pr_number: ${{ github.event.inputs.pr_number || github.event.number }}
@@ -44,7 +46,17 @@ jobs:
4446
GOOGLE_VERTEX_LOCATION: us-central1
4547
GOOGLE_VERTEX_PROJECT: ${{ secrets.GOOGLE_VERTEX_PROJECT }}
4648

47-
- name: Review PR With GPT-5
49+
review_gpt5:
50+
name: Review With GPT-5
51+
timeout-minutes: 40
52+
runs-on: ubuntu-latest
53+
steps:
54+
- name: Checkout repository
55+
uses: actions/checkout@v4
56+
with:
57+
fetch-depth: 0
58+
59+
- name: Run Polka Codes (GPT-5)
4860
uses: polka-codes/action@master
4961
with:
5062
pr_number: ${{ github.event.inputs.pr_number || github.event.number }}

‎.vscode/launch.json‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,24 @@
7171
"name": "Release BokaFuzzer (Fuzzing)",
7272
"program": "${workspaceFolder:boka}/Fuzzing/.build/release/BokaFuzzer",
7373
"preLaunchTask": "swift: Build Release BokaFuzzer (Fuzzing)"
74+
},
75+
{
76+
"type": "swift",
77+
"request": "launch",
78+
"args": [],
79+
"cwd": "${workspaceFolder:boka}/JAMTests",
80+
"name": "Debug BenchmarkTestVectors (JAMTests)",
81+
"program": "${workspaceFolder:boka}/JAMTests/.build/debug/BenchmarkTestVectors",
82+
"preLaunchTask": "swift: Build Debug BenchmarkTestVectors (JAMTests)"
83+
},
84+
{
85+
"type": "swift",
86+
"request": "launch",
87+
"args": [],
88+
"cwd": "${workspaceFolder:boka}/JAMTests",
89+
"name": "Release BenchmarkTestVectors (JAMTests)",
90+
"program": "${workspaceFolder:boka}/JAMTests/.build/release/BenchmarkTestVectors",
91+
"preLaunchTask": "swift: Build Release BenchmarkTestVectors (JAMTests)"
7492
}
7593
]
7694
}

‎Blockchain/Sources/Blockchain/RuntimeProtocols/Accumulation.swift‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -569,7 +569,7 @@ extension Accumulation {
569569

570570
let rightQueueItems = accumulationQueue.array[index...]
571571
let leftQueueItems = accumulationQueue.array[0 ..< index]
572-
var allQueueItems = rightQueueItems.flatMap { $0 } + leftQueueItems.flatMap { $0 } + newQueueItems
572+
var allQueueItems = rightQueueItems.flatMap(\.self) + leftQueueItems.flatMap(\.self) + newQueueItems
573573

574574
editQueue(items: &allQueueItems, accumulatedPackages: Set(zeroPrereqReports.map(\.packageSpecification.workPackageHash)))
575575

@@ -697,7 +697,7 @@ extension Accumulation {
697697
for (service, _) in accumulateOutput.gasUsed {
698698
if accumulateStats[service] != nil { continue }
699699

700-
let digests = accumulated.compactMap(\.digests).flatMap { $0 }
700+
let digests = accumulated.compactMap(\.digests).flatMap(\.self)
701701
let num = digests.filter { $0.serviceIndex == service }.count
702702

703703
if num == 0 { continue }

‎JAMTests/Benchmarks/TestVectors/TestVectors.swift‎

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
import Benchmark
2+
import Codec
3+
import Foundation
4+
import JAMTests
5+
import Utils
6+
7+
let benchmarks: @Sendable () -> Void = {
8+
Benchmark.defaultConfiguration.timeUnits = BenchmarkTimeUnits.milliseconds
9+
10+
// W3F Erasure (full): encode + reconstruct
11+
struct ErasureCodingTestcase: Codable { let data: Data; let shards: [Data] }
12+
let erasureCases = (try? TestLoader.getTestcases(path: "erasure/full", extension: "bin")) ?? []
13+
if !erasureCases.isEmpty {
14+
let config = TestVariants.full.config
15+
let basicSize = config.value.erasureCodedPieceSize
16+
let recoveryCount = config.value.totalNumberOfValidators
17+
let originalCount = basicSize / 2
18+
Benchmark("w3f.erasure.full.encode+reconstruct") { _ in
19+
for testcase in erasureCases {
20+
if let t = try? JamDecoder.decode(ErasureCodingTestcase.self, from: testcase.data, withConfig: config),
21+
let shards = try? ErasureCoding.chunk(data: t.data, basicSize: basicSize, recoveryCount: recoveryCount)
22+
{
23+
let typed = shards.enumerated().map { ErasureCoding.Shard(data: $0.element, index: UInt32($0.offset)) }
24+
_ = try? ErasureCoding.reconstruct(
25+
shards: Array(typed.prefix(originalCount)),
26+
basicSize: basicSize,
27+
originalCount: originalCount,
28+
recoveryCount: recoveryCount
29+
)
30+
}
31+
}
32+
}
33+
}
34+
35+
// W3F Shuffle
36+
struct ShuffleTestCase: Codable { let input: Int; let entropy: String; let output: [Int] }
37+
if let data = try? TestLoader.getFile(path: "shuffle/shuffle_tests", extension: "json"),
38+
let tests = try? JSONDecoder().decode([ShuffleTestCase].self, from: data),
39+
!tests.isEmpty
40+
{
41+
Benchmark("w3f.shuffle") { _ in
42+
for test in tests {
43+
var input = Array(0 ..< test.input)
44+
if let entropy = Data32(fromHexString: test.entropy) {
45+
input.shuffle(randomness: entropy)
46+
blackHole(input)
47+
}
48+
}
49+
}
50+
}
51+
52+
// Traces
53+
let tracePaths = ["traces/fallback", "traces/safrole", "traces/storage", "traces/preimages"]
54+
for path in tracePaths {
55+
let traces = try! JamTestnet.loadTests(path: path, src: .w3f)
56+
Benchmark("w3f.traces.\(path.components(separatedBy: "/").last!)") { benchmark in
57+
for trace in traces {
58+
let testcase = try! JamTestnet.decodeTestcase(trace)
59+
benchmark.startMeasurement()
60+
let result = try? await JamTestnet.runSTF(testcase)
61+
switch result {
62+
case let .success(stateRef):
63+
let root = await stateRef.value.stateRoot
64+
blackHole(root)
65+
case .failure:
66+
blackHole(trace.description)
67+
case .none:
68+
blackHole(trace.description)
69+
}
70+
benchmark.stopMeasurement()
71+
}
72+
}
73+
}
74+
}

‎JAMTests/Package.swift‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ let package = Package(
2222
.package(path: "../Blockchain"),
2323
.package(path: "../PolkaVM"),
2424
.package(url: "https://github.com/apple/swift-testing.git", branch: "6.0.0"),
25+
.package(url: "https://github.com/ordo-one/package-benchmark.git", .upToNextMajor(from: "1.29.4")),
2526
],
2627
targets: [
2728
// Targets are the basic building blocks of a package, defining a module or a test suite.
@@ -56,6 +57,25 @@ let package = Package(
5657
.interoperabilityMode(.Cxx),
5758
]
5859
),
60+
.executableTarget(
61+
name: "BenchmarkTestVectors",
62+
dependencies: [
63+
.product(name: "Benchmark", package: "package-benchmark"),
64+
"JAMTests",
65+
"Utils",
66+
"Codec",
67+
],
68+
path: "Benchmarks/TestVectors",
69+
cxxSettings: [
70+
.unsafeFlags(["-Wno-incomplete-umbrella"]),
71+
],
72+
swiftSettings: [
73+
.interoperabilityMode(.Cxx),
74+
],
75+
plugins: [
76+
.plugin(name: "BenchmarkPlugin", package: "package-benchmark"),
77+
],
78+
),
5979
],
6080
swiftLanguageModes: [.version("6")]
6181
)

0 commit comments

Comments
 (0)