Benchmark Nightly #15
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Benchmark Nightly | |
| on: | |
| schedule: | |
| # Run at 2 AM UTC every day (adjust timezone as needed) | |
| - cron: '0 2 * * *' | |
| workflow_dispatch: | |
| env: | |
| BENCHMARK_DISABLE_JEMALLOC: true | |
| jobs: | |
| benchmark-nightly: | |
| name: Nightly Benchmarks | |
| runs-on: [self-hosted, linux] | |
| timeout-minutes: 120 | |
| steps: | |
| - name: Checkout Code | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| submodules: recursive | |
| - run: sudo apt-get update | |
| - uses: awalsh128/cache-apt-pkgs-action@latest | |
| with: | |
| packages: librocksdb-dev libzstd-dev libbz2-dev liblz4-dev | |
| - uses: aws-actions/configure-aws-credentials@v4 | |
| with: | |
| aws-region: us-east-2 | |
| - name: Setup Swift | |
| uses: SwiftyLab/setup-swift@latest | |
| - name: Get Swift Version | |
| id: swift-version | |
| run: | | |
| SWIFT_VERSION=$(swift --version 2>&1 | head -n1 | sed -E 's/.*Swift version ([0-9.]+).*/\1/') | |
| echo "Detected Swift version: $SWIFT_VERSION" | |
| echo "version=$SWIFT_VERSION" >> $GITHUB_OUTPUT | |
| - name: Cache SPM | |
| uses: runs-on/cache@v4 | |
| with: | |
| path: '**/.build' | |
| key: ${{ runner.os }}-spm-release-swift-${{ steps.swift-version.outputs.version }}-${{ hashFiles('**/Package.resolved') }} | |
| restore-keys: | | |
| ${{ runner.os }}-spm-release-swift-${{ steps.swift-version.outputs.version }}- | |
| env: | |
| RUNS_ON_S3_BUCKET_CACHE: laminar-gh-action-cache | |
| - name: Cache Cargo | |
| uses: actions/cache@v4 | |
| with: | |
| path: | | |
| ~/.cargo/bin/ | |
| ~/.cargo/registry/index/ | |
| ~/.cargo/registry/cache/ | |
| ~/.cargo/git/db/ | |
| target/ | |
| key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} | |
| - name: Cache bandersnatch_vrfs static lib | |
| uses: actions/cache@v4 | |
| with: | |
| path: .lib/libbandersnatch_vrfs.a | |
| key: ${{ runner.os }}-libs-libbandersnatch-${{ hashFiles('Utils/Sources/bandersnatch/**') }} | |
| restore-keys: | | |
| ${{ runner.os }}-libs-libbandersnatch | |
| - name: Cache bls static lib | |
| uses: actions/cache@v4 | |
| with: | |
| path: .lib/libbls.a | |
| key: ${{ runner.os }}-libs-libbls-${{ hashFiles('Utils/Sources/bls/**') }} | |
| restore-keys: | | |
| ${{ runner.os }}-libs-libbls | |
| - name: Cache erasure-coding static lib | |
| uses: actions/cache@v4 | |
| with: | |
| path: .lib/libec.a | |
| key: ${{ runner.os }}-libs-libec-${{ hashFiles('Utils/Sources/erasure-coding/**') }} | |
| restore-keys: | | |
| ${{ runner.os }}-libs-libec | |
| - name: Cache ed25519-zebra static lib | |
| uses: actions/cache@v4 | |
| with: | |
| path: .lib/libed25519_zebra_ffi.a | |
| key: ${{ runner.os }}-libs-libed25519-${{ hashFiles('Utils/Sources/ed25519-zebra/**') }} | |
| restore-keys: | | |
| ${{ runner.os }}-libs-libed25519 | |
| - name: Setup Rust | |
| uses: dtolnay/rust-toolchain@nightly | |
| with: | |
| components: rustfmt | |
| - name: Build dependencies | |
| run: make deps | |
| - name: Check if benchmarks exist | |
| id: check-benchmarks | |
| run: | | |
| if [ -d "JAMTests/Benchmarks" ]; then | |
| echo "hasBenchmark=1" >> $GITHUB_ENV | |
| else | |
| echo "hasBenchmark=0" >> $GITHUB_ENV | |
| fi | |
| - name: Run all benchmarks | |
| if: ${{ env.hasBenchmark == '1' }} | |
| run: | | |
| cd JAMTests | |
| echo "# 🌙 Nightly Benchmark Results" >> $GITHUB_STEP_SUMMARY | |
| echo "Generated on: $(date)" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "Commit: ${{ github.sha }}" >> $GITHUB_STEP_SUMMARY | |
| echo "Branch: ${{ github.ref_name }}" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| # Backup existing nightly baseline if it exists | |
| if [ -f ".benchmarkBaselines/Benchmarks/nightly/results.json" ]; then | |
| mkdir -p .benchmarkBaselines/Benchmarks/nightly-previous | |
| cp .benchmarkBaselines/Benchmarks/nightly/results.json .benchmarkBaselines/Benchmarks/nightly-previous/results.json | |
| fi | |
| # Run all benchmarks and create nightly-new baseline | |
| swift package --allow-writing-to-directory .benchmarkBaselines/ benchmark baseline update nightly-new --no-progress | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "## Benchmark Results Summary" >> $GITHUB_STEP_SUMMARY | |
| if [ -f ".benchmarkBaselines/Benchmarks/nightly-previous/results.json" ]; then | |
| swift package benchmark baseline compare nightly-new nightly-previous --format markdown >> $GITHUB_STEP_SUMMARY 2>&1 || true | |
| else | |
| echo "No previous nightly baseline to compare against." >> $GITHUB_STEP_SUMMARY | |
| fi | |
| - name: Upload benchmark results | |
| if: ${{ env.hasBenchmark == '1' }} | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: nightly-benchmark-results-${{ github.run_number }} | |
| path: | | |
| JAMTests/.benchmarkBaselines/Benchmarks/nightly-new/results.json | |
| JAMTests/.benchmarkBaselines/thresholds.json | |
| retention-days: 90 | |
| - name: Check for performance regressions | |
| if: ${{ env.hasBenchmark == '1' }} | |
| run: | | |
| cd JAMTests | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "## 🔍 Regression Analysis (vs. Previous Nightly)" >> $GITHUB_STEP_SUMMARY | |
| # Compare nightly-new against nightly-previous if available | |
| if [ -f ".benchmarkBaselines/Benchmarks/nightly-previous/results.json" ]; then | |
| if [ -f ".benchmarkBaselines/thresholds.json" ]; then | |
| swift package benchmark baseline check nightly-new nightly-previous --thresholds .benchmarkBaselines/thresholds.json --format markdown >> $GITHUB_STEP_SUMMARY 2>&1 || echo "⚠️ Regression check completed with warnings" >> $GITHUB_STEP_SUMMARY | |
| else | |
| echo "⚠️ No thresholds.json found - skipping detailed regression check" >> $GITHUB_STEP_SUMMARY | |
| swift package benchmark baseline check nightly-new nightly-previous --format markdown >> $GITHUB_STEP_SUMMARY 2>&1 || true | |
| fi | |
| else | |
| echo "No previous nightly baseline available for regression check." >> $GITHUB_STEP_SUMMARY | |
| fi | |
| - name: Generate benchmark report | |
| if: ${{ env.hasBenchmark == '1' }} | |
| run: | | |
| cd JAMTests | |
| # Create a detailed benchmark report | |
| cat > benchmark-report.md << 'EOF' | |
| # 🌙 Nightly Benchmark Report | |
| **Date**: $(date -u +"%Y-%m-%d %H:%M:%S UTC") | |
| **Commit**: ${{ github.sha }} | |
| **Branch**: ${{ github.ref_name }} | |
| ## Benchmark Coverage | |
| This report includes performance metrics for: | |
| - Runtime & State Transition Logic | |
| - Blockchain Operations Pipeline | |
| - VM Execution with Realistic Workloads | |
| - Merkle Trie & State Backend Operations | |
| - Validator Node Operations | |
| - RocksDB Persistent Storage | |
| ## Results | |
| Full results are available in the workflow artifacts. | |
| EOF | |
| cat benchmark-report.md >> $GITHUB_STEP_SUMMARY | |
| - name: Update nightly baseline | |
| if: ${{ env.hasBenchmark == '1' }} | |
| run: | | |
| cd JAMTests | |
| # Replace nightly with nightly-new after all checks pass | |
| if [ -d ".benchmarkBaselines/Benchmarks/nightly-new" ]; then | |
| rm -rf .benchmarkBaselines/Benchmarks/nightly | |
| mv .benchmarkBaselines/Benchmarks/nightly-new .benchmarkBaselines/Benchmarks/nightly | |
| echo "✅ Updated nightly baseline with new results" | |
| fi | |
| - name: Commit updated baseline | |
| if: ${{ env.hasBenchmark == '1' }} | |
| run: | | |
| cd JAMTests | |
| # Configure git | |
| git config user.name "github-actions[bot]" | |
| git config user.email "github-actions[bot]@users.noreply.github.com" | |
| # Commit and push updated baseline | |
| if [ -d ".benchmarkBaselines/Benchmarks/nightly" ]; then | |
| git add .benchmarkBaselines/Benchmarks/nightly/results.json | |
| if git diff --staged --quiet; then | |
| echo "No changes to nightly baseline" | |
| else | |
| git commit -m "chore: update nightly benchmark results [skip ci]" | |
| git push | |
| echo "✅ Committed and pushed updated nightly baseline" | |
| fi | |
| fi |