Benchmark Typeberry PR #29
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # PR Benchmark Workflow | |
| # | |
| # This workflow benchmarks PRs from github.com/fluffylabs/typeberry by: | |
| # 1. Downloading a docker image artifact from a typeberry PR's workflow run | |
| # 2. Running picofuzz tests against that docker image | |
| # 3. Comparing results with baseline statistics from https://typeberry.fluffylabs.dev/ | |
| # 4. Posting a comment to the PR with the comparison results | |
| # | |
| # Usage: | |
| # Trigger manually with workflow_dispatch: | |
| # - pr_number: The PR number from fluffylabs/typeberry (e.g., "704") | |
| # | |
| # Or trigger via repository_dispatch with type "benchmark-pr": | |
| # client_payload: | |
| # pr_number: "704" | |
| # | |
| # Required Secrets: | |
| # - TYPEBERRY_PAT: GitHub Personal Access Token with permissions to: | |
| # * Read artifacts from fluffylabs/typeberry (repo:public_repo or repo:read) | |
| # * Post comments to PRs in fluffylabs/typeberry (repo:public_repo or issues:write) | |
| # Note: GITHUB_TOKEN is used as fallback but may not have sufficient permissions | |
| # | |
| name: Benchmark Typeberry PR | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| pr_number: | |
| description: 'PR number from fluffylabs/typeberry' | |
| required: true | |
| type: string | |
| repository_dispatch: | |
| types: [benchmark-pr] | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.inputs.pr_number || github.event.client_payload.pr_number }} | |
| cancel-in-progress: true | |
| jobs: | |
| setup-docker-image: | |
| name: Setup Docker Image | |
| runs-on: self-hosted | |
| outputs: | |
| pr_number: ${{ steps.set-vars.outputs.pr_number }} | |
| workflow_run_id: ${{ steps.find-workflow-run.outputs.workflow_run_id }} | |
| is_perf_runner: ${{ steps.detect-perf.outputs.is_perf }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Detect if running on perf hardware | |
| id: detect-perf | |
| uses: ./.github/actions/detect-perf | |
| - name: Set variables | |
| id: set-vars | |
| run: | | |
| if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then | |
| echo "pr_number=${{ github.event.inputs.pr_number }}" >> $GITHUB_OUTPUT | |
| else | |
| echo "pr_number=${{ github.event.client_payload.pr_number }}" >> $GITHUB_OUTPUT | |
| fi | |
| - name: Find latest build-docker workflow run | |
| id: find-workflow-run | |
| env: | |
| GH_TOKEN: ${{ secrets.TYPEBERRY_PAT }} | |
| PR_NUMBER: ${{ steps.set-vars.outputs.pr_number }} | |
| run: | | |
| # Get PR details to find the head branch | |
| PR_DATA=$(curl -s \ | |
| -H "Authorization: token $GH_TOKEN" \ | |
| -H "Accept: application/vnd.github.v3+json" \ | |
| "https://api.github.com/repos/fluffylabs/typeberry/pulls/$PR_NUMBER") | |
| HEAD_BRANCH=$(echo "$PR_DATA" | jq -r '.head.ref') | |
| echo "PR #$PR_NUMBER branch: $HEAD_BRANCH" | |
| # Get the workflow ID for build-docker | |
| WORKFLOW_ID=$(curl -s \ | |
| -H "Authorization: token $GH_TOKEN" \ | |
| -H "Accept: application/vnd.github.v3+json" \ | |
| "https://api.github.com/repos/fluffylabs/typeberry/actions/workflows" \ | |
| | jq -r '.workflows[] | select(.name == "Build - Docker" or (.path | endswith("build-docker.yml"))) | .id' \ | |
| | head -n 1) | |
| if [ -z "$WORKFLOW_ID" ]; then | |
| echo "Error: Could not find build-docker workflow" | |
| exit 1 | |
| fi | |
| echo "build-docker workflow ID: $WORKFLOW_ID" | |
| # Find the latest successful workflow run for this branch with artifacts | |
| WORKFLOW_RUN_ID=$(curl -s \ | |
| -H "Authorization: token $GH_TOKEN" \ | |
| -H "Accept: application/vnd.github.v3+json" \ | |
| "https://api.github.com/repos/fluffylabs/typeberry/actions/workflows/$WORKFLOW_ID/runs?branch=$HEAD_BRANCH&status=success&per_page=10" \ | |
| | jq -r '.workflow_runs[] | select(.artifacts_url != null) | .id' \ | |
| | head -n 1) | |
| if [ -z "$WORKFLOW_RUN_ID" ]; then | |
| echo "Error: Could not find successful build-docker workflow run for branch $HEAD_BRANCH" | |
| echo "Please ensure the build-docker workflow has completed successfully for this PR" | |
| exit 1 | |
| fi | |
| echo "Found workflow run ID: $WORKFLOW_RUN_ID" | |
| echo "workflow_run_id=$WORKFLOW_RUN_ID" >> $GITHUB_OUTPUT | |
| - name: Download docker image artifact | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: typeberry-docker-image | |
| path: ./docker-image | |
| github-token: ${{ secrets.TYPEBERRY_PAT || secrets.GITHUB_TOKEN }} | |
| repository: fluffylabs/typeberry | |
| run-id: ${{ steps.find-workflow-run.outputs.workflow_run_id }} | |
| - name: Load docker image | |
| run: | | |
| gunzip ./docker-image/typeberry-image.tar.gz | |
| docker load -i ./docker-image/typeberry-image.tar | |
| # Tag the image as latest so existing tests work | |
| IMAGE_ID=$(docker images --format "{{.ID}}" typeberry | head -n 1) | |
| if [ -z "$IMAGE_ID" ]; then | |
| echo "Error: No typeberry image found after loading" | |
| exit 1 | |
| fi | |
| docker tag $IMAGE_ID ghcr.io/fluffylabs/typeberry:latest | |
| docker images | grep typeberry | |
| benchmark: | |
| name: Run Benchmark Tests | |
| needs: setup-docker-image | |
| runs-on: self-hosted | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| submodules: 'recursive' | |
| - name: Detect if running on perf hardware | |
| id: detect-perf | |
| uses: ./.github/actions/detect-perf | |
| - name: Setup Node.js | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: '20' | |
| cache: 'npm' | |
| - name: Run benchmark tests and generate report | |
| run: npm start -w picofuzz-benchmark | |
| - name: Upload benchmark report | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: benchmark-report | |
| path: ./benchmark-report.md | |
| if: always() | |
| - name: Upload CSV results | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: picofuzz-results | |
| path: ./picofuzz-result/*.csv | |
| retention-days: 90 | |
| # Only upload results from perf hardware for aggregation | |
| if: always() && steps.detect-perf.outputs.is_perf == 'true' | |
| report: | |
| name: Post Results to PR | |
| needs: [benchmark, setup-docker-image] | |
| runs-on: ubuntu-latest | |
| if: always() | |
| steps: | |
| - name: Generate GitHub App token | |
| id: generate-token | |
| uses: actions/create-github-app-token@v2 | |
| with: | |
| app-id: ${{ secrets.APP_ID }} | |
| private-key: ${{ secrets.APP_PRIVATE_KEY }} | |
| owner: fluffylabs | |
| repositories: typeberry | |
| - name: Download benchmark report | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: benchmark-report | |
| - name: Post or update comment on PR | |
| uses: actions/github-script@v7 | |
| with: | |
| github-token: ${{ steps.generate-token.outputs.token }} | |
| script: | | |
| const fs = require('fs'); | |
| const report = fs.readFileSync('benchmark-report.md', 'utf8'); | |
| // Add a marker to identify our comments | |
| const marker = '<!-- typeberry-benchmark-report -->'; | |
| const body = marker + '\n' + report; | |
| const owner = 'fluffylabs'; | |
| const repo = 'typeberry'; | |
| const issue_number = ${{ needs.setup-docker-image.outputs.pr_number }}; | |
| // Find existing comment | |
| const { data: comments } = await github.rest.issues.listComments({ | |
| owner, | |
| repo, | |
| issue_number, | |
| }); | |
| const existingComment = comments.find(comment => | |
| comment.body && comment.body.includes(marker) | |
| ); | |
| if (existingComment) { | |
| // Update existing comment | |
| await github.rest.issues.updateComment({ | |
| owner, | |
| repo, | |
| comment_id: existingComment.id, | |
| body | |
| }); | |
| console.log(`Updated existing comment: ${existingComment.html_url}`); | |
| } else { | |
| // Create new comment | |
| const { data: newComment } = await github.rest.issues.createComment({ | |
| owner, | |
| repo, | |
| issue_number, | |
| body | |
| }); | |
| console.log(`Created new comment: ${newComment.html_url}`); | |
| } |