Materialize unplugged from global cache #163
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Benchmarks | |
| on: | |
| pull_request: | |
| branches: [main] | |
| paths-ignore: | |
| - 'documentation/**' | |
| - 'README.md' | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| env: | |
| CARGO_TERM_COLOR: always | |
| jobs: | |
| benchmark: | |
| name: Run benchmarks | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout the repository | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ github.event.pull_request.head.sha }} | |
| fetch-depth: 0 | |
| - name: Find merge-base and prepare worktrees | |
| id: setup | |
| run: | | |
| git fetch origin ${{ github.event.pull_request.base.ref }} | |
| MERGE_BASE=$(git merge-base HEAD origin/${{ github.event.pull_request.base.ref }}) | |
| echo "merge_base=$MERGE_BASE" >> $GITHUB_OUTPUT | |
| # Create worktrees for base and head | |
| git worktree add ../base $MERGE_BASE | |
| git worktree add ../head HEAD | |
| - name: Setup Rust toolchain | |
| uses: dtolnay/rust-toolchain@stable | |
| - name: Rust Cache | |
| uses: Swatinem/rust-cache@98c8021b550208e191a6a3145459bfc9fb29c4c0 # v2.8.0 | |
| with: | |
| shared-key: benchmark | |
| workspaces: | | |
| ../base -> target | |
| ../head -> target | |
| - name: Install hyperfine | |
| run: | | |
| wget https://github.com/sharkdp/hyperfine/releases/download/v1.18.0/hyperfine_1.18.0_amd64.deb | |
| sudo dpkg -i hyperfine_1.18.0_amd64.deb | |
| - name: Build Yarn (base) | |
| working-directory: ../base | |
| run: cargo build --release -p zpm --bin yarn-bin | |
| - name: Build Yarn (head) | |
| working-directory: ../head | |
| run: cargo build --release -p zpm --bin yarn-bin | |
| - name: Start mock proxy | |
| working-directory: ../head | |
| run: | | |
| ./target/release/yarn-bin debug mock-proxy -p 4873 https://registry.npmjs.org & | |
| echo "YARN_NPM_REGISTRY_SERVER=http://localhost:4873" >> $GITHUB_ENV | |
| echo "YARN_UNSAFE_HTTP_WHITELIST=localhost" >> $GITHUB_ENV | |
| - name: Run benchmark (base) | |
| working-directory: ../base | |
| run: ./target/release/yarn-bin debug bench gatsby install-full-cold | |
| - name: Run benchmark (head) | |
| working-directory: ../head | |
| run: ./target/release/yarn-bin debug bench gatsby install-full-cold | |
| - name: Generate benchmark comment | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| const fs = require('fs'); | |
| // Read both benchmark results | |
| const baseResults = JSON.parse(fs.readFileSync('../base/bench-gatsby-install-full-cold.json', 'utf8')); | |
| const headResults = JSON.parse(fs.readFileSync('../head/bench-gatsby-install-full-cold.json', 'utf8')); | |
| // Rename commands to indicate before/after | |
| baseResults.results.forEach(r => { | |
| r.command = `[base] ${r.command.split('/').pop()}`; | |
| }); | |
| headResults.results.forEach(r => { | |
| r.command = `[head] ${r.command.split('/').pop()}`; | |
| }); | |
| // Merge results | |
| const mergedResults = { | |
| results: [...baseResults.results, ...headResults.results] | |
| }; | |
| // Save merged results | |
| fs.writeFileSync('bench-merged.json', JSON.stringify(mergedResults, null, 2)); | |
| // Calculate comparison metrics | |
| const baseMean = baseResults.results[0].mean; | |
| const headMean = headResults.results[0].mean; | |
| const baseMedian = baseResults.results[0].median; | |
| const headMedian = headResults.results[0].median; | |
| const meanDiff = headMean - baseMean; | |
| const meanPct = ((meanDiff / baseMean) * 100).toFixed(2); | |
| const medianDiff = headMedian - baseMedian; | |
| const medianPct = ((medianDiff / baseMedian) * 100).toFixed(2); | |
| const formatTime = (t) => `${t.toFixed(3)}s`; | |
| const formatDiff = (pct) => { | |
| const num = parseFloat(pct); | |
| if (num > 0) return `+${pct}% ⚠️`; | |
| if (num < 0) return `${pct}% ✅`; | |
| return `${pct}%`; | |
| }; | |
| // Build the comment | |
| const comment = `## ⏱️ Benchmark Results | |
| | Metric | Base | Head | Difference | | |
| |--------|------|------|------------| | |
| | **Mean** | ${formatTime(baseMean)} | ${formatTime(headMean)} | ${formatDiff(meanPct)} | | |
| | **Median** | ${formatTime(baseMedian)} | ${formatTime(headMedian)} | ${formatDiff(medianPct)} | | |
| | **Min** | ${formatTime(baseResults.results[0].min)} | ${formatTime(headResults.results[0].min)} | | | |
| | **Max** | ${formatTime(baseResults.results[0].max)} | ${formatTime(headResults.results[0].max)} | | | |
| | **Std Dev** | ${formatTime(baseResults.results[0].stddev)} | ${formatTime(headResults.results[0].stddev)} | | | |
| <details> | |
| <summary>📊 Raw benchmark data</summary> | |
| **Base times:** ${baseResults.results[0].times.map(t => formatTime(t)).join(', ')} | |
| **Head times:** ${headResults.results[0].times.map(t => formatTime(t)).join(', ')} | |
| </details> | |
| --- | |
| *Benchmark: \`gatsby install-full-cold\`* | |
| `.split('\n').map(l => l.trim()).join('\n'); | |
| // Save comment body and PR number for the perfcheck-comment workflow | |
| fs.mkdirSync('perfcheck-comment', { recursive: true }); | |
| fs.writeFileSync('perfcheck-comment/comment-body.md', comment); | |
| fs.writeFileSync('perfcheck-comment/pr-number.txt', String(context.issue.number)); | |
| - name: Upload comment body artifact | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: perfcheck-comment | |
| path: perfcheck-comment/ | |
| - name: Upload merged benchmark results | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: benchmark-results | |
| path: bench-merged.json |