Fix retrieving docker build workflow id

.github/workflows/pr-benchmark.yml

@@ -20,7 +20,7 @@
 #     * Post comments to PRs in fluffylabs/typeberry (repo:public_repo or issues:write)
 #     Note: GITHUB_TOKEN is used as fallback but may not have sufficient permissions
 #
-name: PR Benchmark
+name: Benchmark Typeberry PR
 
 on:
   workflow_dispatch:
@@ -56,7 +56,7 @@ jobs:
       - name: Find latest build-docker workflow run
         id: find-workflow-run
         env:
-          GH_TOKEN: ${{ secrets.TYPEBERRY_PAT || secrets.GITHUB_TOKEN }}
+          GH_TOKEN: ${{ secrets.TYPEBERRY_PAT }}
           PR_NUMBER: ${{ steps.set-vars.outputs.pr_number }}
         run: |
           # Get PR details to find the head branch
@@ -73,7 +73,7 @@ jobs:
             -H "Authorization: token $GH_TOKEN" \
             -H "Accept: application/vnd.github.v3+json" \
             "https://api.github.com/repos/fluffylabs/typeberry/actions/workflows" \
-            | jq -r '.workflows[] | select(.name == "build-docker" or .path | endswith("build-docker.yml")) | .id' \
+            | jq -r '.workflows[] | select(.name == "Build - Docker" or (.path | endswith("build-docker.yml"))) | .id' \
             | head -n 1)
 
           if [ -z "$WORKFLOW_ID" ]; then
@@ -104,15 +104,15 @@ jobs:
         uses: actions/download-artifact@v4
         with:
           name: typeberry-docker-image
-          path: /tmp/docker-image
+          path: ./docker-image
           github-token: ${{ secrets.TYPEBERRY_PAT || secrets.GITHUB_TOKEN }}
           repository: fluffylabs/typeberry
           run-id: ${{ steps.find-workflow-run.outputs.workflow_run_id }}
 
       - name: Load docker image
         run: |
-          gunzip /tmp/docker-image/typeberry-image.tar.gz
-          docker load -i /tmp/docker-image/typeberry-image.tar
+          gunzip ./docker-image/typeberry-image.tar.gz
+          docker load -i ./docker-image/typeberry-image.tar
           # Tag the image as latest so existing tests work
           IMAGE_ID=$(docker images --format "{{.ID}}" --filter "reference=ghcr.io/fluffylabs/typeberry:*" | head -n 1)
           if [ -z "$IMAGE_ID" ]; then
@@ -161,22 +161,63 @@ jobs:
     runs-on: ubuntu-latest
     if: always()
     steps:
+      - name: Generate GitHub App token
+        id: generate-token
+        uses: actions/create-github-app-token@v1
+        with:
+          app-id: ${{ secrets.APP_ID }}
+          private-key: ${{ secrets.APP_PRIVATE_KEY }}
+          owner: fluffylabs
+          repositories: typeberry
+
       - name: Download benchmark report
         uses: actions/download-artifact@v4
         with:
           name: benchmark-report
 
-      - name: Post comment to PR
+      - name: Post or update comment on PR
         uses: actions/github-script@v7
         with:
-          github-token: ${{ secrets.TYPEBERRY_PAT || secrets.GITHUB_TOKEN }}
+          github-token: ${{ steps.generate-token.outputs.token }}
           script: |
             const fs = require('fs');
             const report = fs.readFileSync('benchmark-report.md', 'utf8');
 
-            await github.rest.issues.createComment({
-              owner: 'fluffylabs',
-              repo: 'typeberry',
-              issue_number: ${{ needs.setup-docker-image.outputs.pr_number }},
-              body: report
+            // Add a marker to identify our comments
+            const marker = '<!-- typeberry-benchmark-report -->';
+            const body = marker + '\n' + report;
+
+            const owner = 'fluffylabs';
+            const repo = 'typeberry';
+            const issue_number = ${{ needs.setup-docker-image.outputs.pr_number }};
+
+            // Find existing comment
+            const { data: comments } = await github.rest.issues.listComments({
+              owner,
+              repo,
+              issue_number,
             });
+
+            const existingComment = comments.find(comment =>
+              comment.body && comment.body.includes(marker)
+            );
+
+            if (existingComment) {
+              // Update existing comment
+              await github.rest.issues.updateComment({
+                owner,
+                repo,
+                comment_id: existingComment.id,
+                body
+              });
+              console.log(`Updated existing comment: ${existingComment.html_url}`);
+            } else {
+              // Create new comment
+              const { data: newComment } = await github.rest.issues.createComment({
+                owner,
+                repo,
+                issue_number,
+                body
+              });
+              console.log(`Created new comment: ${newComment.html_url}`);
+            }

picofuzz-benchmark/README.md

@@ -9,7 +9,6 @@ This directory contains scripts for running picofuzz benchmarks on latest typebe
 Prepares the environment for benchmark tests:
 - Installs npm dependencies
 - Builds the picofuzz Docker image
-- Prepares test data
 - Creates result directories
 
 **Usage:**

picofuzz-benchmark/index.ts

@@ -8,7 +8,7 @@ const WORKSPACE_ROOT = path.resolve(
 );
 process.chdir(WORKSPACE_ROOT);
 
-const THRESHOLD = 0.05; // 5% difference is considered no-change.
+const THRESHOLD = 0.1; // 10% difference is considered no-change.
 
 const BASELINE_URL = "https://typeberry.fluffylabs.dev";
 const RESULT_DIR = `${WORKSPACE_ROOT}/picofuzz-result`;
@@ -25,18 +25,18 @@ const TEST_FILES = [
 type TestName = (typeof TESTS)[number];
 
 interface TestStats {
-  timestamp: string;
-  peer: string;
-  total_duration_ms: number;
-  imports: number;
+  name: string;
   avg_import_ms: number;
   min_import_ms: number;
-  max_import_ms: number;
-  p50_import_ms: number;
   p90_import_ms: number;
   p99_import_ms: number;
 }
 
+// Convert nanoseconds to milliseconds
+function nsToMs(ns: number): number {
+  return ns / 1_000_000;
+}
+
 async function downloadBaseline(test: TestName): Promise<TestStats | null> {
   const url = `${BASELINE_URL}/${test}.csv`;
   console.log(`Fetching baseline for ${test} from ${url}...`);
@@ -61,16 +61,11 @@ async function downloadBaseline(test: TestName): Promise<TestStats | null> {
     const values = lastLine.split(",");
 
     return {
-      timestamp: values[0],
-      peer: values[1],
-      total_duration_ms: Number.parseFloat(values[2]),
-      imports: Number.parseInt(values[3], 10),
-      avg_import_ms: Number.parseFloat(values[4]),
-      min_import_ms: Number.parseFloat(values[5]),
-      max_import_ms: Number.parseFloat(values[6]),
-      p50_import_ms: Number.parseFloat(values[7]),
-      p90_import_ms: Number.parseFloat(values[8]),
-      p99_import_ms: Number.parseFloat(values[9]),
+      name: values[0],
+      avg_import_ms: nsToMs(Number.parseFloat(values[4])),
+      min_import_ms: nsToMs(Number.parseFloat(values[6])),
+      p90_import_ms: nsToMs(Number.parseFloat(values[17])),
+      p99_import_ms: nsToMs(Number.parseFloat(values[19])),
     };
   } catch (error) {
     console.error(`Error fetching baseline for ${test}:`, error);
@@ -96,16 +91,11 @@ function parseCSV(filePath: string): TestStats | null {
   const values = lastLine.split(",");
 
   return {
-    timestamp: values[0],
-    peer: values[1],
-    total_duration_ms: Number.parseFloat(values[2]),
-    imports: Number.parseInt(values[3], 10),
-    avg_import_ms: Number.parseFloat(values[4]),
-    min_import_ms: Number.parseFloat(values[5]),
-    max_import_ms: Number.parseFloat(values[6]),
-    p50_import_ms: Number.parseFloat(values[7]),
-    p90_import_ms: Number.parseFloat(values[8]),
-    p99_import_ms: Number.parseFloat(values[9]),
+    name: values[0],
+    avg_import_ms: nsToMs(Number.parseFloat(values[4])),
+    min_import_ms: nsToMs(Number.parseFloat(values[6])),
+    p90_import_ms: nsToMs(Number.parseFloat(values[17])),
+    p99_import_ms: nsToMs(Number.parseFloat(values[19])),
   };
 }
 
@@ -188,17 +178,15 @@ async function generateReport(
     report += "|--------|----------|---------|------------|\n";
 
     if (baseline) {
-      report += `| Avg Import Time | ${baseline.avg_import_ms.toFixed(2)}ms | ${current.avg_import_ms.toFixed(2)}ms | ${formatDiff(baseline.avg_import_ms, current.avg_import_ms)} |\n`;
-      report += `| P50 Import Time | ${baseline.p50_import_ms.toFixed(2)}ms | ${current.p50_import_ms.toFixed(2)}ms | ${formatDiff(baseline.p50_import_ms, current.p50_import_ms)} |\n`;
-      report += `| P90 Import Time | ${baseline.p90_import_ms.toFixed(2)}ms | ${current.p90_import_ms.toFixed(2)}ms | ${formatDiff(baseline.p90_import_ms, current.p90_import_ms)} |\n`;
-      report += `| P99 Import Time | ${baseline.p99_import_ms.toFixed(2)}ms | ${current.p99_import_ms.toFixed(2)}ms | ${formatDiff(baseline.p99_import_ms, current.p99_import_ms)} |\n`;
-      report += `| Total Duration | ${baseline.total_duration_ms.toFixed(2)}ms | ${current.total_duration_ms.toFixed(2)}ms | ${formatDiff(baseline.total_duration_ms, current.total_duration_ms)} |\n`;
+      report += `| min | ${baseline.min_import_ms.toFixed(2)}ms | ${current.min_import_ms.toFixed(2)}ms | ${formatDiff(baseline.min_import_ms, current.min_import_ms)} |\n`;
+      report += `| mean | ${baseline.avg_import_ms.toFixed(2)}ms | ${current.avg_import_ms.toFixed(2)}ms | ${formatDiff(baseline.avg_import_ms, current.avg_import_ms)} |\n`;
+      report += `| p90 | ${baseline.p90_import_ms.toFixed(2)}ms | ${current.p90_import_ms.toFixed(2)}ms | ${formatDiff(baseline.p90_import_ms, current.p90_import_ms)} |\n`;
+      report += `| p99 | ${baseline.p99_import_ms.toFixed(2)}ms | ${current.p99_import_ms.toFixed(2)}ms | ${formatDiff(baseline.p99_import_ms, current.p99_import_ms)} |\n`;
     } else {
-      report += `| Avg Import Time | N/A | ${current.avg_import_ms.toFixed(2)}ms | Baseline not available |\n`;
-      report += `| P50 Import Time | N/A | ${current.p50_import_ms.toFixed(2)}ms | Baseline not available |\n`;
-      report += `| P90 Import Time | N/A | ${current.p90_import_ms.toFixed(2)}ms | Baseline not available |\n`;
-      report += `| P99 Import Time | N/A | ${current.p99_import_ms.toFixed(2)}ms | Baseline not available |\n`;
-      report += `| Total Duration | N/A | ${current.total_duration_ms.toFixed(2)}ms | Baseline not available |\n`;
+      report += `| min | N/A | ${current.min_import_ms.toFixed(2)}ms | Baseline not available |\n`;
+      report += `| mean | N/A | ${current.avg_import_ms.toFixed(2)}ms | Baseline not available |\n`;
+      report += `| p90 | N/A | ${current.p90_import_ms.toFixed(2)}ms | Baseline not available |\n`;
+      report += `| p99 | N/A | ${current.p99_import_ms.toFixed(2)}ms | Baseline not available |\n`;
     }
 
     report += "\n";

picofuzz-benchmark/setup.sh

@@ -1,8 +1,13 @@
 #!/usr/bin/env bash
-set -euo pipefail
+set -eu
 
-echo "==> Installing dependencies..."
-npm ci
+# Jump to workspace root.
+cd "$(dirname "${BASH_SOURCE[0]}")"
+cd ..
+
+# for some reason it's not available sometimes
+echo "Make sure tsx is installed"
+npm i tsx
 
 echo "==> Building picofuzz..."
 npm run build-docker -w @fluffylabs/picofuzz