CI benchmarks #7
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Benchmark | |
| on: | |
| issue_comment: | |
| types: [created] | |
| pull_request: | |
| types: [labeled] | |
| jobs: | |
| benchmark: | |
| name: Benchmark | |
| if: | | |
| (github.event_name == 'issue_comment' && github.event.issue.pull_request && contains(github.event.comment.body, '/benchmark')) || | |
| (github.event_name == 'pull_request' && github.event.label.name == 'benchmark') | |
| runs-on: macos-26 | |
| permissions: | |
| contents: read | |
| pull-requests: write | |
| steps: | |
| - name: Get PR number | |
| id: pr-number | |
| run: | | |
| if [ "${{ github.event_name }}" = "issue_comment" ]; then | |
| echo "number=${{ github.event.issue.number }}" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "number=${{ github.event.pull_request.number }}" >> "$GITHUB_OUTPUT" | |
| fi | |
| - name: Get PR details | |
| id: pr | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| run: | | |
| PR_DATA=$(gh api repos/${{ github.repository }}/pulls/${{ steps.pr-number.outputs.number }}) | |
| HEAD_SHA=$(echo "$PR_DATA" | jq -r '.head.sha') | |
| HEAD_REF=$(echo "$PR_DATA" | jq -r '.head.ref') | |
| BASE_REF=$(echo "$PR_DATA" | jq -r '.base.ref') | |
| echo "head_sha=$HEAD_SHA" >> "$GITHUB_OUTPUT" | |
| echo "head_ref=$HEAD_REF" >> "$GITHUB_OUTPUT" | |
| echo "base_ref=$BASE_REF" >> "$GITHUB_OUTPUT" | |
| - name: Add reaction to comment | |
| if: github.event_name == 'issue_comment' | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| run: | | |
| gh api repos/${{ github.repository }}/issues/comments/${{ github.event.comment.id }}/reactions \ | |
| -f content='+1' | |
| - uses: actions/checkout@master | |
| with: | |
| ref: ${{ steps.pr.outputs.head_sha }} | |
| fetch-depth: 0 | |
| - name: Select Xcode version | |
| run: sudo xcode-select -s /Applications/Xcode_26.2.0.app | |
| - name: Install hyperfine | |
| run: brew install hyperfine | |
| - name: Get merge-base | |
| id: commits | |
| run: | | |
| git fetch origin ${{ steps.pr.outputs.base_ref }} | |
| MERGE_BASE=$(git merge-base HEAD origin/${{ steps.pr.outputs.base_ref }}) | |
| echo "merge_base=$MERGE_BASE" >> "$GITHUB_OUTPUT" | |
| echo "Head SHA: ${{ steps.pr.outputs.head_sha }}" | |
| echo "Merge-base SHA: $MERGE_BASE" | |
| - name: Build PR | |
| run: swift build -c release --product periphery --enable-index-store | |
| - name: Benchmark HEAD | |
| run: | | |
| hyperfine --show-output --warmup 3 --export-json head-benchmark.json './.build/release/periphery scan --quiet --skip-build --strict --baseline baselines/macOS-release.json --index-store-path .build/release/index/store' | |
| - name: Build merge-base | |
| run: | | |
| git checkout ${{ steps.commits.outputs.merge_base }} | |
| swift build -c release --product periphery --enable-index-store | |
| - name: Benchmark merge-base | |
| run: | | |
| hyperfine --warmup 3 --export-json base-benchmark.json './.build/release/periphery scan --quiet --skip-build --strict --baseline baselines/macOS-release.json --index-store-path .build/release/index/store' | |
| - name: Post results | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| run: | | |
| HEAD_MEAN=$(jq '.results[0].mean' head-benchmark.json) | |
| BASE_MEAN=$(jq '.results[0].mean' base-benchmark.json) | |
| HEAD_STDDEV=$(jq '.results[0].stddev' head-benchmark.json) | |
| BASE_STDDEV=$(jq '.results[0].stddev' base-benchmark.json) | |
| CHANGE=$(echo "scale=2; (($HEAD_MEAN - $BASE_MEAN) / $BASE_MEAN) * 100" | bc) | |
| COMMENT="## Benchmark Results | |
| | Commit | Mean (s) | Std Dev (s) | | |
| |--------|----------|-------------| | |
| | Merge-base (\`$(echo ${{ steps.commits.outputs.merge_base }} | cut -c1-7)\`) | $(printf "%.3f" $BASE_MEAN) | ±$(printf "%.3f" $BASE_STDDEV) | | |
| | HEAD (\`$(echo ${{ steps.pr.outputs.head_sha }} | cut -c1-7)\`) | $(printf "%.3f" $HEAD_MEAN) | ±$(printf "%.3f" $HEAD_STDDEV) | | |
| **Change: $(printf "%+.1f%%" $CHANGE)**" | |
| gh pr comment ${{ steps.pr-number.outputs.number }} --body "$COMMENT" | |
| - name: Remove benchmark label | |
| if: github.event_name == 'pull_request' | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| run: | | |
| gh pr edit ${{ steps.pr-number.outputs.number }} --remove-label benchmark | |
| - name: Upload benchmark results | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: benchmark-results | |
| path: "*-benchmark.json" |