feat: add benchmark with ci

.github/workflows/benchmark-pr.yml

@@ -0,0 +1,158 @@
+name: Benchmark PR
+
+on:
+  workflow_dispatch:
+  pull_request:
+    branches: [ master ]
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+env:
+  BENCHMARK_DISABLE_JEMALLOC: true
+
+jobs:
+  benchmark-delta:
+    name: Comparison
+    runs-on: [self-hosted, linux]
+    continue-on-error: true
+    timeout-minutes: 60
+
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          submodules: recursive
+
+      - run: sudo apt-get update
+      - uses: awalsh128/cache-apt-pkgs-action@latest
+        with:
+          packages: librocksdb-dev libzstd-dev libbz2-dev liblz4-dev
+
+      - uses: aws-actions/configure-aws-credentials@v4
+        with:
+          aws-region: us-east-2
+
+      - name: Cache SPM
+        uses: runs-on/cache@v4
+        with:
+          path: '**/.build'
+          key: ${{ runner.os }}-spm-release-${{ hashFiles('**/Package.resolved') }}
+          restore-keys: |
+            ${{ runner.os }}-spm-release-
+        env:
+          RUNS_ON_S3_BUCKET_CACHE: laminar-gh-action-cache
+
+      - name: Cache Cargo
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cargo/bin/
+            ~/.cargo/registry/index/
+            ~/.cargo/registry/cache/
+            ~/.cargo/git/db/
+            target/
+          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
+
+      - name: Cache bandersnatch_vrfs static lib
+        uses: actions/cache@v4
+        with:
+          path: .lib/libbandersnatch_vrfs.a
+          key: ${{ runner.os }}-libs-libbandersnatch-${{ hashFiles('Utils/Sources/bandersnatch/**') }}
+          restore-keys: |
+              ${{ runner.os }}-libs-libbandersnatch
+
+      - name: Cache bls static lib
+        uses: actions/cache@v4
+        with:
+          path: .lib/libbls.a
+          key: ${{ runner.os }}-libs-libbls-${{ hashFiles('Utils/Sources/bls/**') }}
+          restore-keys: |
+              ${{ runner.os }}-libs-libbls
+
+      - name: Cache erasure-coding static lib
+        uses: actions/cache@v4
+        with:
+          path: .lib/libec.a
+          key: ${{ runner.os }}-libs-libec-${{ hashFiles('Utils/Sources/erasure-coding/**') }}
+          restore-keys: |
+              ${{ runner.os }}-libs-libec
+
+      - name: Setup Swift
+        uses: SwiftyLab/setup-swift@latest
+
+      - name: Setup Rust
+        uses: dtolnay/rust-toolchain@nightly
+        with:
+          components: rustfmt
+
+      - name: Build dependencies
+        run: make deps
+
+      - name: Check if benchmarks exist and setup environment
+        run: |
+          [ -d "JAMTests/Benchmarks" ] && echo "hasBenchmark=1" >> $GITHUB_ENV
+
+      - name: Switch to branch 'master'
+        if: ${{ env.hasBenchmark == '1' }}
+        run: |
+          git checkout master
+
+      - name: Run benchmarks for branch 'master'
+        if: ${{ env.hasBenchmark == '1' }}
+        run: |
+          cd JAMTests
+          # Check if benchmarks exist on master and if the benchmark plugin is available
+          if [ -d "Benchmarks" ] && swift package plugin --list 2>/dev/null | grep -q benchmark; then
+            swift package --allow-writing-to-directory .benchmarkBaselines/ benchmark baseline update master --no-progress --quiet
+          else
+            echo "Benchmarks don't exist on master branch yet or benchmark plugin not available - skipping master baseline"
+          fi
+
+      - name: Switch to PR branch
+        if: ${{ env.hasBenchmark == '1' }}
+        run: |
+          git checkout ${{ github.head_ref }}
+
+      - name: Run benchmarks for PR branch
+        if: ${{ env.hasBenchmark == '1' }}
+        run: |
+          cd JAMTests
+          swift package --allow-writing-to-directory .benchmarkBaselines/ benchmark baseline update pull_request --no-progress --quiet
+
+      - name: Compare PR and master
+        if: ${{ env.hasBenchmark == '1' }}
+        id: benchmark
+        run: |
+          cd JAMTests
+          echo "# 📊 Benchmark Results"
+          echo "Generated on: $(date)"
+          echo ""
+          echo "exitStatus=1" >> $GITHUB_ENV
+
+          # Check if master baseline exists for comparison
+          if [ -f ".benchmarkBaselines/BenchmarkTestVectors/master/results.json" ]; then
+            echo "## 🔄 Comparison with Master Branch"
+            echo ""
+            swift package benchmark baseline check master pull_request --format markdown
+            echo "exitStatus=0" >> $GITHUB_ENV
+          else
+            echo "## 🆕 First Benchmark Run"
+            echo ""
+            echo "⚠️ Master baseline not found. This appears to be the first benchmark run."
+            echo "The following results will become the baseline for future comparisons:"
+            echo ""
+
+            # Run benchmarks to generate results for first time
+            echo "### Benchmark Results:"
+            swift package benchmark --format markdown 2>&1 || true
+            echo "exitStatus=0" >> $GITHUB_ENV
+          fi
+        continue-on-error: true
+
+      - name: Exit with correct status
+        if: ${{ env.hasBenchmark == '1' }}
+        run: |
+          exit ${{ env.exitStatus }}

.github/workflows/ci.yml

@@ -10,6 +10,9 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 
+env:
+  BENCHMARK_DISABLE_JEMALLOC: true
+
 jobs:
   lint:
     name: Swift Lint

.github/workflows/coverage.yml

@@ -10,6 +10,9 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 
+env:
+  BENCHMARK_DISABLE_JEMALLOC: true
+
 jobs:
   coverage:
     name: Code Coverage

.github/workflows/polka-codes-review-pr.yml

@@ -19,7 +19,8 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  handle:
+  review_gemini:
+    name: Review With Gemini 2.5 Pro
     timeout-minutes: 40
     runs-on: ubuntu-latest
     steps:
@@ -28,11 +29,12 @@ jobs:
         with:
           fetch-depth: 0
 
-      - uses: google-github-actions/auth@v2
+      - name: Google Auth
+        uses: google-github-actions/auth@v2
         with:
           credentials_json: ${{ secrets.GOOGLE_CREDENTIALS }}
 
-      - name: Review PR With Gemini 2.5 Pro
+      - name: Run Polka Codes (Gemini)
         uses: polka-codes/action@master
         with:
           pr_number: ${{ github.event.inputs.pr_number || github.event.number }}
@@ -44,7 +46,17 @@ jobs:
           GOOGLE_VERTEX_LOCATION: us-central1
           GOOGLE_VERTEX_PROJECT: ${{ secrets.GOOGLE_VERTEX_PROJECT }}
 
-      - name: Review PR With GPT-5
+  review_gpt5:
+    name: Review With GPT-5
+    timeout-minutes: 40
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Run Polka Codes (GPT-5)
         uses: polka-codes/action@master
         with:
           pr_number: ${{ github.event.inputs.pr_number || github.event.number }}

.vscode/launch.json

@@ -71,6 +71,24 @@
 			"name": "Release BokaFuzzer (Fuzzing)",
 			"program": "${workspaceFolder:boka}/Fuzzing/.build/release/BokaFuzzer",
 			"preLaunchTask": "swift: Build Release BokaFuzzer (Fuzzing)"
+		},
+		{
+			"type": "swift",
+			"request": "launch",
+			"args": [],
+			"cwd": "${workspaceFolder:boka}/JAMTests",
+			"name": "Debug BenchmarkTestVectors (JAMTests)",
+			"program": "${workspaceFolder:boka}/JAMTests/.build/debug/BenchmarkTestVectors",
+			"preLaunchTask": "swift: Build Debug BenchmarkTestVectors (JAMTests)"
+		},
+		{
+			"type": "swift",
+			"request": "launch",
+			"args": [],
+			"cwd": "${workspaceFolder:boka}/JAMTests",
+			"name": "Release BenchmarkTestVectors (JAMTests)",
+			"program": "${workspaceFolder:boka}/JAMTests/.build/release/BenchmarkTestVectors",
+			"preLaunchTask": "swift: Build Release BenchmarkTestVectors (JAMTests)"
 		}
 	]
 }

Blockchain/Sources/Blockchain/RuntimeProtocols/Accumulation.swift

@@ -569,7 +569,7 @@ extension Accumulation {
 
         let rightQueueItems = accumulationQueue.array[index...]
         let leftQueueItems = accumulationQueue.array[0 ..< index]
-        var allQueueItems = rightQueueItems.flatMap { $0 } + leftQueueItems.flatMap { $0 } + newQueueItems
+        var allQueueItems = rightQueueItems.flatMap(\.self) + leftQueueItems.flatMap(\.self) + newQueueItems
 
         editQueue(items: &allQueueItems, accumulatedPackages: Set(zeroPrereqReports.map(\.packageSpecification.workPackageHash)))
 
@@ -697,7 +697,7 @@ extension Accumulation {
         for (service, _) in accumulateOutput.gasUsed {
             if accumulateStats[service] != nil { continue }
 
-            let digests = accumulated.compactMap(\.digests).flatMap { $0 }
+            let digests = accumulated.compactMap(\.digests).flatMap(\.self)
             let num = digests.filter { $0.serviceIndex == service }.count
 
             if num == 0 { continue }

JAMTests/Benchmarks/TestVectors/TestVectors.swift

@@ -0,0 +1,74 @@
+import Benchmark
+import Codec
+import Foundation
+import JAMTests
+import Utils
+
+let benchmarks: @Sendable () -> Void = {
+    Benchmark.defaultConfiguration.timeUnits = BenchmarkTimeUnits.milliseconds
+
+    // W3F Erasure (full): encode + reconstruct
+    struct ErasureCodingTestcase: Codable { let data: Data; let shards: [Data] }
+    let erasureCases = (try? TestLoader.getTestcases(path: "erasure/full", extension: "bin")) ?? []
+    if !erasureCases.isEmpty {
+        let config = TestVariants.full.config
+        let basicSize = config.value.erasureCodedPieceSize
+        let recoveryCount = config.value.totalNumberOfValidators
+        let originalCount = basicSize / 2
+        Benchmark("w3f.erasure.full.encode+reconstruct") { _ in
+            for testcase in erasureCases {
+                if let t = try? JamDecoder.decode(ErasureCodingTestcase.self, from: testcase.data, withConfig: config),
+                   let shards = try? ErasureCoding.chunk(data: t.data, basicSize: basicSize, recoveryCount: recoveryCount)
+                {
+                    let typed = shards.enumerated().map { ErasureCoding.Shard(data: $0.element, index: UInt32($0.offset)) }
+                    _ = try? ErasureCoding.reconstruct(
+                        shards: Array(typed.prefix(originalCount)),
+                        basicSize: basicSize,
+                        originalCount: originalCount,
+                        recoveryCount: recoveryCount
+                    )
+                }
+            }
+        }
+    }
+
+    // W3F Shuffle
+    struct ShuffleTestCase: Codable { let input: Int; let entropy: String; let output: [Int] }
+    if let data = try? TestLoader.getFile(path: "shuffle/shuffle_tests", extension: "json"),
+       let tests = try? JSONDecoder().decode([ShuffleTestCase].self, from: data),
+       !tests.isEmpty
+    {
+        Benchmark("w3f.shuffle") { _ in
+            for test in tests {
+                var input = Array(0 ..< test.input)
+                if let entropy = Data32(fromHexString: test.entropy) {
+                    input.shuffle(randomness: entropy)
+                    blackHole(input)
+                }
+            }
+        }
+    }
+
+    // Traces
+    let tracePaths = ["traces/fallback", "traces/safrole", "traces/storage", "traces/preimages"]
+    for path in tracePaths {
+        let traces = try! JamTestnet.loadTests(path: path, src: .w3f)
+        Benchmark("w3f.traces.\(path.components(separatedBy: "/").last!)") { benchmark in
+            for trace in traces {
+                let testcase = try! JamTestnet.decodeTestcase(trace)
+                benchmark.startMeasurement()
+                let result = try? await JamTestnet.runSTF(testcase)
+                switch result {
+                case let .success(stateRef):
+                    let root = await stateRef.value.stateRoot
+                    blackHole(root)
+                case .failure:
+                    blackHole(trace.description)
+                case .none:
+                    blackHole(trace.description)
+                }
+                benchmark.stopMeasurement()
+            }
+        }
+    }
+}

JAMTests/Package.swift

@@ -22,6 +22,7 @@ let package = Package(
         .package(path: "../Blockchain"),
         .package(path: "../PolkaVM"),
         .package(url: "https://github.com/apple/swift-testing.git", branch: "6.0.0"),
+        .package(url: "https://github.com/ordo-one/package-benchmark.git", .upToNextMajor(from: "1.29.4")),
     ],
     targets: [
         // Targets are the basic building blocks of a package, defining a module or a test suite.
@@ -56,6 +57,25 @@ let package = Package(
                 .interoperabilityMode(.Cxx),
             ]
         ),
+        .executableTarget(
+            name: "BenchmarkTestVectors",
+            dependencies: [
+                .product(name: "Benchmark", package: "package-benchmark"),
+                "JAMTests",
+                "Utils",
+                "Codec",
+            ],
+            path: "Benchmarks/TestVectors",
+            cxxSettings: [
+                .unsafeFlags(["-Wno-incomplete-umbrella"]),
+            ],
+            swiftSettings: [
+                .interoperabilityMode(.Cxx),
+            ],
+            plugins: [
+                .plugin(name: "BenchmarkPlugin", package: "package-benchmark"),
+            ],
+        ),
     ],
     swiftLanguageModes: [.version("6")]
 )