From b0568fc2e04d2881cff1d9eb9d47fdde51244911 Mon Sep 17 00:00:00 2001 From: Shigoto-dev19 Date: Tue, 14 Oct 2025 11:13:17 +0300 Subject: [PATCH 1/7] Add performance regression tests to CI --- .../actions/upload-perf-baseline/action.yml | 35 +++++++++++++++++++ .github/workflows/checks.yml | 12 +++++++ run-ci-tests.sh | 6 ++++ 3 files changed, 53 insertions(+) create mode 100644 .github/actions/upload-perf-baseline/action.yml diff --git a/.github/actions/upload-perf-baseline/action.yml b/.github/actions/upload-perf-baseline/action.yml new file mode 100644 index 0000000000..59fab5ca26 --- /dev/null +++ b/.github/actions/upload-perf-baseline/action.yml @@ -0,0 +1,35 @@ +name: Upload Perf Baseline (optional) +description: Upload perf-regression.json only when PERF_MODE is --dump. +inputs: + perf_mode: + description: "--dump or --check" + required: true + path: + description: "Path to perf baseline JSON" + required: false + default: tests/perf-regression/perf-regression.json + name: + description: "Artifact name" + required: false + default: perf-regression-json + retention-days: + description: "Artifact retention (days)" + required: false + default: "30" + +runs: + using: "composite" + steps: + - name: Skip unless in dump mode + if: inputs.perf_mode != '--dump' + shell: bash + run: echo "Not in --dump mode; skipping upload." + + - name: Upload baseline + if: inputs.perf_mode == '--dump' + uses: actions/upload-artifact@v4 + with: + name: ${{ inputs.name }} + path: ${{ inputs.path }} + if-no-files-found: error + retention-days: ${{ inputs.retention-days }} diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 546dc719c0..eaa4bfbe03 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -152,6 +152,7 @@ jobs: 'Verification Key Regression Check 2', 'CommonJS test', 'Cache Regression', + 'Performance Regression', ] steps: - name: Checkout repository with submodules @@ -176,7 +177,18 @@ jobs: - name: Execute tests env: TEST_TYPE: ${{ matrix.test_type }} + PERF_MODE: ${{ contains(github.event.pull_request.labels.*.name, 'dump-performance') && '--dump' || '--check' }} run: sh run-ci-tests.sh + - name: Upload performance regressions baseline + if: > + matrix.test_type == 'Performance Regression' && + contains(github.event.pull_request.labels.*.name, 'dump-performance') + uses: ./.github/actions/upload-perf-baseline + with: + perf_mode: ${{ contains(github.event.pull_request.labels.*.name, 'dump-performance') && '--dump' || '--check' }} + path: tests/perf-regression/perf-regression.json + name: perf-regression-json + retention-days: 30 - name: Add to job summary if: always() run: | diff --git a/run-ci-tests.sh b/run-ci-tests.sh index 213514dc0c..dbc1ee71d3 100755 --- a/run-ci-tests.sh +++ b/run-ci-tests.sh @@ -53,6 +53,12 @@ case $TEST_TYPE in echo "Cache Regression" ./scripts/tests/check-cache-regressions.sh ;; + +"Performance Regression") + echo "Running Performance Regression Check" + PERF_MODE="${PERF_MODE:---check}" + ./tests/perf-regression/perf-regression.sh "$PERF_MODE" + ;; *) echo "ERROR: Invalid environment variable, not clear what tests to run! $CI_NODE_INDEX" exit 1 From e512d764dac78c9db348ef9b8bcd390acf394a94 Mon Sep 17 00:00:00 2001 From: Shigoto-dev19 Date: Tue, 14 Oct 2025 19:12:10 +0300 Subject: [PATCH 2/7] Fix performance regression check in CI by downloading the baseline --- .github/workflows/checks.yml | 76 ++++++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index eaa4bfbe03..c68cc40739 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -33,6 +33,7 @@ concurrency: permissions: contents: write + actions: read jobs: Prepare: @@ -174,6 +175,81 @@ jobs: credentials_json: '${{ secrets.GCP_O1JS_CI_BUCKET_SERVICE_ACCOUNT_KEY }}' - name: Prepare for tests run: touch profiling.md + - name: Find a prior run that actually has perf baseline + id: find-baseline + if: > + matrix.test_type == 'Performance Regression' && !contains(github.event.pull_request.labels.*.name, 'dump-performance') + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + TARGET_REPO: ${{ inputs.target_repo || github.repository }} + WORKFLOW_NAME: ${{ github.workflow }} # display name, e.g. "Checks" + BRANCH: ${{ github.head_ref || github.ref_name }} + ARTIFACT_NAME: perf-regression-json + run: | + set -euo pipefail + + echo "Looking for completed prior runs of '$WORKFLOW_NAME' on '$BRANCH' in $TARGET_REPO with artifact '$ARTIFACT_NAME'" + + # List recent COMPLETED runs (don't require 'success' to survive matrix failures) + RUN_IDS=$(gh run list \ + --repo "$TARGET_REPO" \ + --workflow "$WORKFLOW_NAME" \ + --json databaseId,headBranch,status,conclusion \ + --limit 40 \ + --jq "[.[] | select(.status==\"completed\" and .headBranch==\"$BRANCH\") | .databaseId] | .[]" \ + || true) + + if [[ -z "${RUN_IDS:-}" ]]; then + echo "No completed runs found on this branch in $TARGET_REPO." + echo "Seed a baseline once (add 'dump-performance' label) or relax branch to 'main' later." + exit 1 + fi + + # Pick the first run that actually has the artifact and it's not expired + FOUND="" + for RID in $RUN_IDS; do + HAS=$(gh api repos/"$TARGET_REPO"/actions/runs/"$RID"/artifacts \ + --jq ".artifacts | map(select(.name==\"$ARTIFACT_NAME\" and .expired==false)) | length") + echo "Run $RID has $HAS matching artifact(s)." + if [[ "$HAS" -gt 0 ]]; then + FOUND="$RID" + break + fi + done + + if [[ -z "${FOUND:-}" ]]; then + echo "No prior completed run on '$BRANCH' has artifact '$ARTIFACT_NAME'." + echo "Seed a baseline once (add 'dump-performance' label) so check mode can download it." + exit 1 + fi + + echo "run_id=$FOUND" >> "$GITHUB_OUTPUT" + echo "Found suitable run id: $FOUND" + - name: Download baseline artifact from that run + if: > + matrix.test_type == 'Performance Regression' && !contains(github.event.pull_request.labels.*.name, 'dump-performance') + uses: actions/download-artifact@v4 + with: + name: perf-regression-json + run-id: ${{ steps.find-baseline.outputs.run_id }} + path: tests/perf-regression + github-token: ${{ secrets.GITHUB_TOKEN }} + - name: Normalize baseline path for the script + if: > + matrix.test_type == 'Performance Regression' &&!contains(github.event.pull_request.labels.*.name, 'dump-performance') + run: | + set -euo pipefail + FILE="$(find tests/perf-regression -name 'perf-regression.json' -print -quit || true)" + if [[ -z "${FILE:-}" ]]; then + echo "ERROR: perf-regression.json not found inside downloaded artifact." + echo "Listing:" + ls -R tests/perf-regression || true + exit 1 + fi + if [[ "$FILE" != "tests/perf-regression/perf-regression.json" ]]; then + mv -f "$FILE" tests/perf-regression/perf-regression.json + fi + echo "Baseline ready at tests/perf-regression/perf-regression.json" - name: Execute tests env: TEST_TYPE: ${{ matrix.test_type }} From 55e2072bd3256cac14acf65fa641fa43b7f8dcd8 Mon Sep 17 00:00:00 2001 From: Shigoto-dev19 Date: Wed, 15 Oct 2025 13:23:06 +0300 Subject: [PATCH 3/7] Prettify logs for zkprograms performance regression check --- src/lib/testing/perf-regression.ts | 75 +++++++++++++++++++++++++----- 1 file changed, 64 insertions(+), 11 deletions(-) diff --git a/src/lib/testing/perf-regression.ts b/src/lib/testing/perf-regression.ts index cd754ef085..86bf415eef 100644 --- a/src/lib/testing/perf-regression.ts +++ b/src/lib/testing/perf-regression.ts @@ -26,7 +26,7 @@ import minimist from 'minimist'; import path from 'path'; import { ConstraintSystemSummary } from '../provable/core/provable-context.js'; -export { PerfRegressionEntry, Performance }; +export { PerfRegressionEntry, Performance, logPerf }; type MethodsInfo = Record< string, @@ -358,10 +358,12 @@ function checkAgainstBaseline(params: { // tolerances const compileTol = 1.05; // 5% - const compileTiny = 1.08; // for near-zero baselines + const compileTiny = 1.08; // for near-zero baselines (< 5e-5s) const timeTolDefault = 1.1; // 10% for prove/verify const timeTolSmall = 1.25; // 25% for very small times (<0.2s) + const labelPretty = label[0].toUpperCase() + label.slice(1); + if (label === 'compile') { const expected = baseline.compileTime; if (expected == null) { @@ -369,15 +371,21 @@ function checkAgainstBaseline(params: { `No baseline compileTime for "${programName}". Run --dump (compile) to set it.` ); } + const tol = expected < 5e-5 ? compileTiny : compileTol; const allowedPct = (tol - 1) * 100; + const regressionPct = expected === 0 ? 0 : ((actualTime - expected) / expected) * 100; + const failed = actualTime > expected * tol; - if (actualTime > expected * tol) { - const regressionPct = ((actualTime - expected) / expected) * 100; + // colorized perf log + logPerf(programName, label, expected, actualTime, regressionPct, allowedPct, failed); + + if (failed) { throw new Error( `Compile regression for ${programName}\n` + - ` Actual: ${actualTime.toFixed(6)}s\n` + - ` Regression: +${regressionPct.toFixed(2)}% (allowed +${allowedPct.toFixed(0)}%)` + ` Actual: ${actualTime.toFixed(6)}s\n` + + ` Baseline: ${expected.toFixed(6)}s\n` + + ` Regression: +${Number.isFinite(regressionPct) ? regressionPct.toFixed(2) : '∞'}% (allowed +${allowedPct.toFixed(0)}%)` ); } return; @@ -390,6 +398,7 @@ function checkAgainstBaseline(params: { `No baseline method entry for ${programName}.${methodName}. Run --dump (${label}) to add it.` ); } + if (baseMethod.digest !== digest) { throw new Error( `Digest mismatch for ${programName}.${methodName}\n` + @@ -399,21 +408,65 @@ function checkAgainstBaseline(params: { } const expected = label === 'prove' ? baseMethod.proveTime : baseMethod.verifyTime; - const labelPretty = label.charAt(0).toUpperCase(); if (expected == null) { throw new Error( `No baseline ${label}Time for ${programName}.${methodName}. Run --dump (${label}) to set it.` ); } + const tol = expected < 0.2 ? timeTolSmall : timeTolDefault; const allowedPct = (tol - 1) * 100; + const regressionPct = expected === 0 ? 0 : ((actualTime - expected) / expected) * 100; + const failed = actualTime > expected * tol; + + logPerf( + `${programName}.${methodName}`, + label, + expected, + actualTime, + regressionPct, + allowedPct, + failed + ); - if (actualTime > expected * tol) { - const regressionPct = ((actualTime - expected) / expected) * 100; + if (failed) { throw new Error( `${labelPretty} regression for ${programName}.${methodName}\n` + - ` Actual: ${actualTime.toFixed(3)}s\n` + - ` Regression: +${regressionPct.toFixed(2)}% (allowed +${allowedPct.toFixed(0)}%)` + ` Actual: ${actualTime.toFixed(3)}s\n` + + ` Baseline: ${expected.toFixed(3)}s\n` + + ` Regression: +${Number.isFinite(regressionPct) ? regressionPct.toFixed(2) : '∞'}% (allowed +${allowedPct.toFixed(0)}%)` ); } } + +function logPerf( + scope: string, + label: string, + expected: number, + actual: number, + regressionPct: number, + allowedPct: number, + failed: boolean +) { + const COLORS = { + reset: '\x1b[0m', + red: '\x1b[31m', + green: '\x1b[32m', + yellow: '\x1b[33m', + cyan: '\x1b[36m', + }; + + let color: string; + if (failed) color = COLORS.red; + else if (regressionPct > 0) color = COLORS.yellow; + else color = COLORS.green; + + console.log( + `${COLORS.cyan}[Perf][${scope}]${COLORS.reset} ${label}: ` + + `baseline=${expected.toFixed(6)}s, actual=${actual.toFixed(6)}s, ` + + `${color}regression=${regressionPct >= 0 ? '+' : ''}${ + Number.isFinite(regressionPct) ? regressionPct.toFixed(2) : '∞' + }%${COLORS.reset} ` + + `(allowed +${allowedPct.toFixed(0)}%)` + ); +} From 08b2fb12aba6550144f1a32c7a73f3ea61971b9b Mon Sep 17 00:00:00 2001 From: Shigoto-dev19 Date: Wed, 15 Oct 2025 13:23:53 +0300 Subject: [PATCH 4/7] Prettify logs and adjust tolerances for zkApp and CS performance regression checks --- tests/perf-regression/perf-regression.ts | 35 ++++++++++++++++++++---- 1 file changed, 29 insertions(+), 6 deletions(-) diff --git a/tests/perf-regression/perf-regression.ts b/tests/perf-regression/perf-regression.ts index 171f050ac7..3f445657f9 100644 --- a/tests/perf-regression/perf-regression.ts +++ b/tests/perf-regression/perf-regression.ts @@ -20,7 +20,7 @@ import { TokenContract, createDex } from '../../src/examples/zkapps/dex/dex.js'; import { HelloWorld } from '../../src/examples/zkapps/hello-world/hello-world.js'; import { Membership_ } from '../../src/examples/zkapps/voting/membership.js'; import { Voting_ } from '../../src/examples/zkapps/voting/voting.js'; -import { PerfRegressionEntry } from '../../src/lib/testing/perf-regression.js'; +import { PerfRegressionEntry, logPerf } from '../../src/lib/testing/perf-regression.js'; import { tic, toc } from '../../src/lib/util/tic-toc.js'; import { BasicCS, @@ -118,14 +118,37 @@ async function checkPerf(contracts: MinimumConstraintSystem[]) { continue; } - const tolerance = expectedCompile < 5e-5 ? 1.08 : 1.05; - const allowedPct = (tolerance - 1) * 100; + // Tiered tolerances: + // < 0.00001s → 45% + // 0.00001s ≤ t < 0.0001s → 30% + // ≥ 0.0001s → 20% + let allowedPct: number; + if (expectedCompile < 1e-5) { + allowedPct = 45; + } else if (expectedCompile < 1e-4) { + allowedPct = 30; + } else { + allowedPct = 20; + } + const tolerance = 1 + allowedPct / 100; + + const regressionPct = + expectedCompile === 0 + ? compileTime === 0 + ? 0 + : Infinity + : ((compileTime - expectedCompile) / expectedCompile) * 100; + + // colorized log using imported utility + const failed = compileTime > expectedCompile * tolerance; + logPerf(c.name, 'compile', expectedCompile, compileTime, regressionPct, allowedPct, failed); - if (compileTime > expectedCompile * tolerance) { - const regressionPct = ((compileTime - expectedCompile) / expectedCompile) * 100; + // handle failure + if (failed) { errorStack += `\n\nCompile regression for ${c.name} Actual: ${compileTime.toFixed(6)}s - Regression: +${regressionPct.toFixed(2)}% (allowed +${allowedPct.toFixed(0)}%)`; + Baseline: ${expectedCompile.toFixed(6)}s + Regression: +${Number.isFinite(regressionPct) ? regressionPct.toFixed(2) : '∞'}% (allowed +${allowedPct.toFixed(0)}%)`; } } From 4702ace6b0e4b834f33bf130cbdd9c0884947cf0 Mon Sep 17 00:00:00 2001 From: Shigoto-dev19 Date: Wed, 15 Oct 2025 14:06:36 +0300 Subject: [PATCH 5/7] Refactor performance regression CI tests into unified composite action --- .github/actions/perf-regression/action.yml | 199 ++++++++++++++++++ .../actions/upload-perf-baseline/action.yml | 35 --- .github/workflows/checks.yml | 100 ++------- 3 files changed, 214 insertions(+), 120 deletions(-) create mode 100644 .github/actions/perf-regression/action.yml delete mode 100644 .github/actions/upload-perf-baseline/action.yml diff --git a/.github/actions/perf-regression/action.yml b/.github/actions/perf-regression/action.yml new file mode 100644 index 0000000000..23e0f7b95e --- /dev/null +++ b/.github/actions/perf-regression/action.yml @@ -0,0 +1,199 @@ +name: Perf Regression (dump/check) +description: Dump or check perf baseline; auto mode via label; find/download prior baseline; run tests; upload on dump. + +inputs: + mode: + description: "'dump' | 'check' | 'auto' (auto uses live PR labels)" + required: false + default: auto + label_name: + description: "Label that triggers dump when present (auto mode)" + required: false + default: dump-performance + + artifact_name: + description: "Artifact name used to store the baseline" + required: false + default: perf-regression-json + baseline_path: + description: "Path for the baseline JSON in the workspace" + required: false + default: tests/perf-regression/perf-regression.json + + # NOTE: defaults here must be literals; leave empty and resolve at runtime + workflow_name: + description: "Workflow display name to search runs for" + required: false + default: "" + branch: + description: "Branch to search prior runs on" + required: false + default: "" + target_repo: + description: "owner/repo to search in" + required: false + default: "" + + run_command: + description: "Shell command to run perf tests (reads PERF_MODE env)" + required: false + default: sh run-ci-tests.sh + +runs: + using: "composite" + steps: + - name: Echo inputs + shell: bash + run: | + echo "mode=${{ inputs.mode }}" + echo "label_name=${{ inputs.label_name }}" + echo "artifact_name=${{ inputs.artifact_name }}" + echo "baseline_path=${{ inputs.baseline_path }}" + echo "workflow_name=${{ inputs.workflow_name }}" + echo "branch=${{ inputs.branch }}" + echo "target_repo=${{ inputs.target_repo }}" + echo "run_command=${{ inputs.run_command }}" + + # Resolve workflow/repo/branch defaults using runtime contexts (allowed in steps) + - name: Resolve defaults + shell: bash + env: + IN_WORKFLOW: ${{ inputs.workflow_name }} + IN_BRANCH: ${{ inputs.branch }} + IN_REPO: ${{ inputs.target_repo }} + WF_CTX: ${{ github.workflow }} + HEAD_REF: ${{ github.head_ref }} + REF_NAME: ${{ github.ref_name }} + REPO_CTX: ${{ github.repository }} + run: | + set -euo pipefail + WF_NAME="${IN_WORKFLOW:-}" + [[ -z "$WF_NAME" ]] && WF_NAME="${WF_CTX}" + + BRANCH="${IN_BRANCH:-}" + if [[ -z "$BRANCH" ]]; then + if [[ -n "${HEAD_REF}" ]]; then BRANCH="${HEAD_REF}"; else BRANCH="${REF_NAME}"; fi + fi + + REPO="${IN_REPO:-}" + [[ -z "$REPO" ]] && REPO="${REPO_CTX}" + + echo "WF_NAME=$WF_NAME" >> "$GITHUB_ENV" + echo "BRANCH=$BRANCH" >> "$GITHUB_ENV" + echo "TARGET_REPO=$REPO" >> "$GITHUB_ENV" + echo "Resolved: WF_NAME=$WF_NAME BRANCH=$BRANCH TARGET_REPO=$REPO" + + - name: Resolve PERF_MODE + id: resolve-mode + shell: bash + env: + MODE: ${{ inputs.mode }} + LABEL_NAME: ${{ inputs.label_name }} + GH_TOKEN: ${{ github.token }} + EVENT_NAME: ${{ github.event_name }} + OWNER_REPO: ${{ github.repository }} + PR_NUMBER: ${{ github.event.pull_request.number }} + run: | + set -euo pipefail + MODE="${MODE:-auto}" + if [[ "$MODE" == "dump" ]]; then + PERF_MODE="--dump" + elif [[ "$MODE" == "check" ]]; then + PERF_MODE="--check" + else + if [[ "$EVENT_NAME" == "pull_request" ]]; then + LABELS=$(gh api "repos/$OWNER_REPO/issues/$PR_NUMBER/labels" --jq '.[].name' | tr '\n' ' ') + if echo "$LABELS" | grep -q -w "$LABEL_NAME"; then + PERF_MODE="--dump" + else + PERF_MODE="--check" + fi + else + PERF_MODE="--check" + fi + fi + echo "PERF_MODE=$PERF_MODE" | tee -a "$GITHUB_ENV" + echo "perf_mode=$PERF_MODE" >> "$GITHUB_OUTPUT" + + # Only in check mode: find previous run with artifact and download it + - name: Find prior run with baseline + id: find-baseline + if: ${{ steps.resolve-mode.outputs.perf_mode == '--check' }} + shell: bash + env: + GH_TOKEN: ${{ github.token }} + TARGET_REPO: ${{ env.TARGET_REPO }} + WORKFLOW_NAME: ${{ env.WF_NAME }} + BRANCH: ${{ env.BRANCH }} + ARTIFACT_NAME: ${{ inputs.artifact_name }} + run: | + set -euo pipefail + echo "Searching $TARGET_REPO / '$WORKFLOW_NAME' / branch '$BRANCH' for artifact '$ARTIFACT_NAME'" + RUN_IDS=$(gh run list \ + --repo "$TARGET_REPO" \ + --workflow "$WORKFLOW_NAME" \ + --json databaseId,headBranch,status,conclusion \ + --limit 50 \ + --jq "[.[] | select(.status==\"completed\" and .headBranch==\"$BRANCH\") | .databaseId] | .[]" \ + || true) + + if [[ -z "${RUN_IDS:-}" ]]; then + echo "No completed runs found on branch '$BRANCH' in $TARGET_REPO." + exit 1 + fi + + FOUND="" + for RID in $RUN_IDS; do + HAS=$(gh api "repos/$TARGET_REPO/actions/runs/$RID/artifacts" \ + --jq ".artifacts | map(select(.name==\"$ARTIFACT_NAME\" and .expired==false)) | length") + echo "Run $RID has $HAS matching artifact(s)" + if [[ "$HAS" -gt 0 ]]; then FOUND="$RID"; break; fi + done + + if [[ -z "${FOUND:-}" ]]; then + echo "No prior completed run on '$BRANCH' has artifact '$ARTIFACT_NAME'. Seed a baseline once." + exit 1 + fi + + echo "run_id=$FOUND" >> "$GITHUB_OUTPUT" + + - name: Download baseline + if: ${{ steps.resolve-mode.outputs.perf_mode == '--check' }} + uses: actions/download-artifact@v4 + with: + name: ${{ inputs.artifact_name }} + run-id: ${{ steps.find-baseline.outputs.run_id }} + path: tests/perf-regression + github-token: ${{ github.token }} + + - name: Normalize baseline path + if: ${{ steps.resolve-mode.outputs.perf_mode == '--check' }} + shell: bash + env: + DEST: ${{ inputs.baseline_path }} + run: | + set -euo pipefail + FILE="$(find tests/perf-regression -name 'perf-regression.json' -print -quit || true)" + if [[ -z "${FILE:-}" ]]; then + echo "ERROR: perf-regression.json not found inside downloaded artifact." + ls -R tests/perf-regression || true + exit 1 + fi + mkdir -p "$(dirname "$DEST")" + if [[ "$FILE" != "$DEST" ]]; then mv -f "$FILE" "$DEST"; fi + echo "Baseline ready at $DEST" + + - name: Run perf tests + shell: bash + env: + PERF_MODE: ${{ env.PERF_MODE }} + run: ${{ inputs.run_command }} + + - name: Upload baseline (dump mode only) + if: ${{ steps.resolve-mode.outputs.perf_mode == '--dump' }} + uses: actions/upload-artifact@v4 + with: + name: ${{ inputs.artifact_name }} + path: ${{ inputs.baseline_path }} + if-no-files-found: error + retention-days: 30 diff --git a/.github/actions/upload-perf-baseline/action.yml b/.github/actions/upload-perf-baseline/action.yml deleted file mode 100644 index 59fab5ca26..0000000000 --- a/.github/actions/upload-perf-baseline/action.yml +++ /dev/null @@ -1,35 +0,0 @@ -name: Upload Perf Baseline (optional) -description: Upload perf-regression.json only when PERF_MODE is --dump. -inputs: - perf_mode: - description: "--dump or --check" - required: true - path: - description: "Path to perf baseline JSON" - required: false - default: tests/perf-regression/perf-regression.json - name: - description: "Artifact name" - required: false - default: perf-regression-json - retention-days: - description: "Artifact retention (days)" - required: false - default: "30" - -runs: - using: "composite" - steps: - - name: Skip unless in dump mode - if: inputs.perf_mode != '--dump' - shell: bash - run: echo "Not in --dump mode; skipping upload." - - - name: Upload baseline - if: inputs.perf_mode == '--dump' - uses: actions/upload-artifact@v4 - with: - name: ${{ inputs.name }} - path: ${{ inputs.path }} - if-no-files-found: error - retention-days: ${{ inputs.retention-days }} diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index c68cc40739..d37e313767 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -175,96 +175,26 @@ jobs: credentials_json: '${{ secrets.GCP_O1JS_CI_BUCKET_SERVICE_ACCOUNT_KEY }}' - name: Prepare for tests run: touch profiling.md - - name: Find a prior run that actually has perf baseline - id: find-baseline - if: > - matrix.test_type == 'Performance Regression' && !contains(github.event.pull_request.labels.*.name, 'dump-performance') - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - TARGET_REPO: ${{ inputs.target_repo || github.repository }} - WORKFLOW_NAME: ${{ github.workflow }} # display name, e.g. "Checks" - BRANCH: ${{ github.head_ref || github.ref_name }} - ARTIFACT_NAME: perf-regression-json - run: | - set -euo pipefail - - echo "Looking for completed prior runs of '$WORKFLOW_NAME' on '$BRANCH' in $TARGET_REPO with artifact '$ARTIFACT_NAME'" - - # List recent COMPLETED runs (don't require 'success' to survive matrix failures) - RUN_IDS=$(gh run list \ - --repo "$TARGET_REPO" \ - --workflow "$WORKFLOW_NAME" \ - --json databaseId,headBranch,status,conclusion \ - --limit 40 \ - --jq "[.[] | select(.status==\"completed\" and .headBranch==\"$BRANCH\") | .databaseId] | .[]" \ - || true) - - if [[ -z "${RUN_IDS:-}" ]]; then - echo "No completed runs found on this branch in $TARGET_REPO." - echo "Seed a baseline once (add 'dump-performance' label) or relax branch to 'main' later." - exit 1 - fi - - # Pick the first run that actually has the artifact and it's not expired - FOUND="" - for RID in $RUN_IDS; do - HAS=$(gh api repos/"$TARGET_REPO"/actions/runs/"$RID"/artifacts \ - --jq ".artifacts | map(select(.name==\"$ARTIFACT_NAME\" and .expired==false)) | length") - echo "Run $RID has $HAS matching artifact(s)." - if [[ "$HAS" -gt 0 ]]; then - FOUND="$RID" - break - fi - done - - if [[ -z "${FOUND:-}" ]]; then - echo "No prior completed run on '$BRANCH' has artifact '$ARTIFACT_NAME'." - echo "Seed a baseline once (add 'dump-performance' label) so check mode can download it." - exit 1 - fi - - echo "run_id=$FOUND" >> "$GITHUB_OUTPUT" - echo "Found suitable run id: $FOUND" - - name: Download baseline artifact from that run - if: > - matrix.test_type == 'Performance Regression' && !contains(github.event.pull_request.labels.*.name, 'dump-performance') - uses: actions/download-artifact@v4 - with: - name: perf-regression-json - run-id: ${{ steps.find-baseline.outputs.run_id }} - path: tests/perf-regression - github-token: ${{ secrets.GITHUB_TOKEN }} - - name: Normalize baseline path for the script - if: > - matrix.test_type == 'Performance Regression' &&!contains(github.event.pull_request.labels.*.name, 'dump-performance') - run: | - set -euo pipefail - FILE="$(find tests/perf-regression -name 'perf-regression.json' -print -quit || true)" - if [[ -z "${FILE:-}" ]]; then - echo "ERROR: perf-regression.json not found inside downloaded artifact." - echo "Listing:" - ls -R tests/perf-regression || true - exit 1 - fi - if [[ "$FILE" != "tests/perf-regression/perf-regression.json" ]]; then - mv -f "$FILE" tests/perf-regression/perf-regression.json - fi - echo "Baseline ready at tests/perf-regression/perf-regression.json" - name: Execute tests + if: matrix.test_type != 'Performance Regression' env: TEST_TYPE: ${{ matrix.test_type }} - PERF_MODE: ${{ contains(github.event.pull_request.labels.*.name, 'dump-performance') && '--dump' || '--check' }} run: sh run-ci-tests.sh - - name: Upload performance regressions baseline - if: > - matrix.test_type == 'Performance Regression' && - contains(github.event.pull_request.labels.*.name, 'dump-performance') - uses: ./.github/actions/upload-perf-baseline + + - name: Perf regression (dump/check) + if: matrix.test_type == 'Performance Regression' + uses: ./.github/actions/perf-regression + env: + TEST_TYPE: Performance Regression with: - perf_mode: ${{ contains(github.event.pull_request.labels.*.name, 'dump-performance') && '--dump' || '--check' }} - path: tests/perf-regression/perf-regression.json - name: perf-regression-json - retention-days: 30 + mode: auto + label_name: dump-performance + artifact_name: perf-regression-json + baseline_path: tests/perf-regression/perf-regression.json + workflow_name: ${{ github.workflow }} + branch: ${{ github.head_ref || github.ref_name }} + target_repo: ${{ inputs.target_repo || github.repository }} + run_command: sh run-ci-tests.sh - name: Add to job summary if: always() run: | From 8ba94f24c517ba24741794019476b80ae300baf6 Mon Sep 17 00:00:00 2001 From: Shigoto-dev19 Date: Wed, 15 Oct 2025 16:35:46 +0300 Subject: [PATCH 6/7] Refine naming in performance regression CI action --- .github/actions/perf-regression/action.yml | 85 ++++++++++------------ .github/workflows/checks.yml | 3 +- 2 files changed, 38 insertions(+), 50 deletions(-) diff --git a/.github/actions/perf-regression/action.yml b/.github/actions/perf-regression/action.yml index 23e0f7b95e..8329311cbc 100644 --- a/.github/actions/perf-regression/action.yml +++ b/.github/actions/perf-regression/action.yml @@ -1,28 +1,27 @@ -name: Perf Regression (dump/check) -description: Dump or check perf baseline; auto mode via label; find/download prior baseline; run tests; upload on dump. +name: Performance Regression (Dump or Check) +description: Runs performance regression tests in dump or check mode. Auto mode determines the mode based on the PR label. inputs: mode: - description: "'dump' | 'check' | 'auto' (auto uses live PR labels)" + description: "'dump' | 'check' | 'auto' (auto uses the PR label)" required: false default: auto label_name: - description: "Label that triggers dump when present (auto mode)" + description: "Label that enables dump mode when present (auto mode)" required: false default: dump-performance artifact_name: - description: "Artifact name used to store the baseline" + description: "Artifact name used to store the performance baseline JSON" required: false default: perf-regression-json baseline_path: - description: "Path for the baseline JSON in the workspace" + description: "Path where the performance baseline JSON is stored" required: false default: tests/perf-regression/perf-regression.json - # NOTE: defaults here must be literals; leave empty and resolve at runtime workflow_name: - description: "Workflow display name to search runs for" + description: "Workflow name to search for prior runs" required: false default: "" branch: @@ -35,27 +34,14 @@ inputs: default: "" run_command: - description: "Shell command to run perf tests (reads PERF_MODE env)" + description: "Shell command to run performance tests (reads PERF_MODE env)" required: false default: sh run-ci-tests.sh runs: using: "composite" steps: - - name: Echo inputs - shell: bash - run: | - echo "mode=${{ inputs.mode }}" - echo "label_name=${{ inputs.label_name }}" - echo "artifact_name=${{ inputs.artifact_name }}" - echo "baseline_path=${{ inputs.baseline_path }}" - echo "workflow_name=${{ inputs.workflow_name }}" - echo "branch=${{ inputs.branch }}" - echo "target_repo=${{ inputs.target_repo }}" - echo "run_command=${{ inputs.run_command }}" - - # Resolve workflow/repo/branch defaults using runtime contexts (allowed in steps) - - name: Resolve defaults + - name: Resolve workflow and repository context shell: bash env: IN_WORKFLOW: ${{ inputs.workflow_name }} @@ -67,23 +53,19 @@ runs: REPO_CTX: ${{ github.repository }} run: | set -euo pipefail - WF_NAME="${IN_WORKFLOW:-}" - [[ -z "$WF_NAME" ]] && WF_NAME="${WF_CTX}" - - BRANCH="${IN_BRANCH:-}" - if [[ -z "$BRANCH" ]]; then - if [[ -n "${HEAD_REF}" ]]; then BRANCH="${HEAD_REF}"; else BRANCH="${REF_NAME}"; fi - fi - - REPO="${IN_REPO:-}" - [[ -z "$REPO" ]] && REPO="${REPO_CTX}" + WF_NAME="${IN_WORKFLOW:-$WF_CTX}" + BRANCH="${IN_BRANCH:-${HEAD_REF:-$REF_NAME}}" + REPO="${IN_REPO:-$REPO_CTX}" echo "WF_NAME=$WF_NAME" >> "$GITHUB_ENV" echo "BRANCH=$BRANCH" >> "$GITHUB_ENV" echo "TARGET_REPO=$REPO" >> "$GITHUB_ENV" - echo "Resolved: WF_NAME=$WF_NAME BRANCH=$BRANCH TARGET_REPO=$REPO" + echo "Resolved context:" + echo " Workflow: $WF_NAME" + echo " Branch: $BRANCH" + echo " Repository: $REPO" - - name: Resolve PERF_MODE + - name: Determine dump or check mode id: resolve-mode shell: bash env: @@ -98,25 +80,31 @@ runs: MODE="${MODE:-auto}" if [[ "$MODE" == "dump" ]]; then PERF_MODE="--dump" + REASON="explicit input" elif [[ "$MODE" == "check" ]]; then PERF_MODE="--check" + REASON="explicit input" else if [[ "$EVENT_NAME" == "pull_request" ]]; then LABELS=$(gh api "repos/$OWNER_REPO/issues/$PR_NUMBER/labels" --jq '.[].name' | tr '\n' ' ') if echo "$LABELS" | grep -q -w "$LABEL_NAME"; then PERF_MODE="--dump" + REASON="label $LABEL_NAME present" else PERF_MODE="--check" + REASON="label $LABEL_NAME absent" fi else PERF_MODE="--check" + REASON="non-PR event" fi fi + echo "PERF_MODE=$PERF_MODE" | tee -a "$GITHUB_ENV" echo "perf_mode=$PERF_MODE" >> "$GITHUB_OUTPUT" + echo "Mode: $PERF_MODE ($REASON)" - # Only in check mode: find previous run with artifact and download it - - name: Find prior run with baseline + - name: Find prior run with performance baseline artifact id: find-baseline if: ${{ steps.resolve-mode.outputs.perf_mode == '--check' }} shell: bash @@ -128,7 +116,7 @@ runs: ARTIFACT_NAME: ${{ inputs.artifact_name }} run: | set -euo pipefail - echo "Searching $TARGET_REPO / '$WORKFLOW_NAME' / branch '$BRANCH' for artifact '$ARTIFACT_NAME'" + echo "Searching $TARGET_REPO (workflow: $WORKFLOW_NAME, branch: $BRANCH) for artifact $ARTIFACT_NAME..." RUN_IDS=$(gh run list \ --repo "$TARGET_REPO" \ --workflow "$WORKFLOW_NAME" \ @@ -138,7 +126,7 @@ runs: || true) if [[ -z "${RUN_IDS:-}" ]]; then - echo "No completed runs found on branch '$BRANCH' in $TARGET_REPO." + echo "No completed runs found on branch $BRANCH." exit 1 fi @@ -146,18 +134,19 @@ runs: for RID in $RUN_IDS; do HAS=$(gh api "repos/$TARGET_REPO/actions/runs/$RID/artifacts" \ --jq ".artifacts | map(select(.name==\"$ARTIFACT_NAME\" and .expired==false)) | length") - echo "Run $RID has $HAS matching artifact(s)" + echo "Run $RID → $HAS artifact(s)" if [[ "$HAS" -gt 0 ]]; then FOUND="$RID"; break; fi done if [[ -z "${FOUND:-}" ]]; then - echo "No prior completed run on '$BRANCH' has artifact '$ARTIFACT_NAME'. Seed a baseline once." + echo "No suitable run with artifact $ARTIFACT_NAME found." exit 1 fi echo "run_id=$FOUND" >> "$GITHUB_OUTPUT" + echo "Found performance baseline in run: $FOUND" - - name: Download baseline + - name: Download performance baseline artifact if: ${{ steps.resolve-mode.outputs.perf_mode == '--check' }} uses: actions/download-artifact@v4 with: @@ -166,7 +155,7 @@ runs: path: tests/perf-regression github-token: ${{ github.token }} - - name: Normalize baseline path + - name: Normalize performance baseline path if: ${{ steps.resolve-mode.outputs.perf_mode == '--check' }} shell: bash env: @@ -175,21 +164,21 @@ runs: set -euo pipefail FILE="$(find tests/perf-regression -name 'perf-regression.json' -print -quit || true)" if [[ -z "${FILE:-}" ]]; then - echo "ERROR: perf-regression.json not found inside downloaded artifact." + echo "ERROR: perf-regression.json not found." ls -R tests/perf-regression || true exit 1 fi mkdir -p "$(dirname "$DEST")" - if [[ "$FILE" != "$DEST" ]]; then mv -f "$FILE" "$DEST"; fi - echo "Baseline ready at $DEST" + [[ "$FILE" != "$DEST" ]] && mv -f "$FILE" "$DEST" + echo "Performance Baseline ready at: $DEST" - - name: Run perf tests + - name: Run performance regression tests shell: bash env: PERF_MODE: ${{ env.PERF_MODE }} run: ${{ inputs.run_command }} - - name: Upload baseline (dump mode only) + - name: Upload new performance baseline (dump mode) if: ${{ steps.resolve-mode.outputs.perf_mode == '--dump' }} uses: actions/upload-artifact@v4 with: diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index d37e313767..3a448ee194 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -180,8 +180,7 @@ jobs: env: TEST_TYPE: ${{ matrix.test_type }} run: sh run-ci-tests.sh - - - name: Perf regression (dump/check) + - name: Performance Regression (dump or check) if: matrix.test_type == 'Performance Regression' uses: ./.github/actions/perf-regression env: From d15de7fde5fbb67d8812b9278ff3c9240962f909 Mon Sep 17 00:00:00 2001 From: Shigoto-dev19 Date: Wed, 15 Oct 2025 17:07:57 +0300 Subject: [PATCH 7/7] Make performance regression CI checks use main as baseline source --- .github/actions/perf-regression/action.yml | 89 ++++++++++++++-------- .github/workflows/checks.yml | 2 +- 2 files changed, 57 insertions(+), 34 deletions(-) diff --git a/.github/actions/perf-regression/action.yml b/.github/actions/perf-regression/action.yml index 8329311cbc..1930665cc3 100644 --- a/.github/actions/perf-regression/action.yml +++ b/.github/actions/perf-regression/action.yml @@ -1,9 +1,9 @@ name: Performance Regression (Dump or Check) -description: Runs performance regression tests in dump or check mode. Auto mode determines the mode based on the PR label. +description: Runs performance regression tests in dump or check mode. Auto mode decides via PR label on PRs, and via merged PR labels on push to main. inputs: mode: - description: "'dump' | 'check' | 'auto' (auto uses the PR label)" + description: "'dump' | 'check' | 'auto' (auto uses labels)" required: false default: auto label_name: @@ -21,15 +21,15 @@ inputs: default: tests/perf-regression/perf-regression.json workflow_name: - description: "Workflow name to search for prior runs" + description: "Workflow name to search when locating prior runs (defaults to current)" required: false default: "" branch: - description: "Branch to search prior runs on" + description: "Branch to search prior runs on (defaults to main)" required: false - default: "" + default: "main" target_repo: - description: "owner/repo to search in" + description: "owner/repo to search in (defaults to current)" required: false default: "" @@ -48,22 +48,20 @@ runs: IN_BRANCH: ${{ inputs.branch }} IN_REPO: ${{ inputs.target_repo }} WF_CTX: ${{ github.workflow }} - HEAD_REF: ${{ github.head_ref }} - REF_NAME: ${{ github.ref_name }} REPO_CTX: ${{ github.repository }} run: | set -euo pipefail WF_NAME="${IN_WORKFLOW:-$WF_CTX}" - BRANCH="${IN_BRANCH:-${HEAD_REF:-$REF_NAME}}" + BRANCH="${IN_BRANCH}" # default is "main" unless caller overrides REPO="${IN_REPO:-$REPO_CTX}" echo "WF_NAME=$WF_NAME" >> "$GITHUB_ENV" echo "BRANCH=$BRANCH" >> "$GITHUB_ENV" echo "TARGET_REPO=$REPO" >> "$GITHUB_ENV" - echo "Resolved context:" - echo " Workflow: $WF_NAME" - echo " Branch: $BRANCH" - echo " Repository: $REPO" + + echo "Using workflow: $WF_NAME" + echo "Searching performance baselines on branch: $BRANCH" + echo "Target repository: $REPO" - name: Determine dump or check mode id: resolve-mode @@ -75,28 +73,52 @@ runs: EVENT_NAME: ${{ github.event_name }} OWNER_REPO: ${{ github.repository }} PR_NUMBER: ${{ github.event.pull_request.number }} + GIT_REF: ${{ github.ref }} + GIT_SHA: ${{ github.sha }} run: | set -euo pipefail + + decide_from_pr_labels () { + local repo="$1" pr="$2" label="$3" + local labels + labels=$(gh api "repos/$repo/issues/$pr/labels" --jq '.[].name' | tr '\n' ' ') + if echo "$labels" | grep -q -w "$label"; then + echo "--dump" + else + echo "--check" + fi + } + + decide_from_merged_pr_labels () { + local repo="$1" sha="$2" label="$3" + local prs should_dump=0 + prs=$(gh api "repos/$repo/commits/$sha/pulls" --jq '.[].number' || true) + for n in $prs; do + local ls + ls=$(gh api "repos/$repo/issues/$n/labels" --jq '.[].name' | tr '\n' ' ') + if echo "$ls" | grep -q -w "$label"; then + should_dump=1; break + fi + done + if [[ $should_dump -eq 1 ]]; then echo "--dump"; else echo "--check"; fi + } + MODE="${MODE:-auto}" + REASON="" + if [[ "$MODE" == "dump" ]]; then - PERF_MODE="--dump" - REASON="explicit input" + PERF_MODE="--dump"; REASON="explicit input" elif [[ "$MODE" == "check" ]]; then - PERF_MODE="--check" - REASON="explicit input" + PERF_MODE="--check"; REASON="explicit input" else if [[ "$EVENT_NAME" == "pull_request" ]]; then - LABELS=$(gh api "repos/$OWNER_REPO/issues/$PR_NUMBER/labels" --jq '.[].name' | tr '\n' ' ') - if echo "$LABELS" | grep -q -w "$LABEL_NAME"; then - PERF_MODE="--dump" - REASON="label $LABEL_NAME present" - else - PERF_MODE="--check" - REASON="label $LABEL_NAME absent" - fi + PERF_MODE="$(decide_from_pr_labels "$OWNER_REPO" "$PR_NUMBER" "$LABEL_NAME")" + REASON="PR label check" + elif [[ "$EVENT_NAME" == "push" && "$GIT_REF" == "refs/heads/main" ]]; then + PERF_MODE="$(decide_from_merged_pr_labels "$OWNER_REPO" "$GIT_SHA" "$LABEL_NAME")" + REASON="merged PR label check" else - PERF_MODE="--check" - REASON="non-PR event" + PERF_MODE="--check"; REASON="default for non-PR, non-main push" fi fi @@ -104,7 +126,7 @@ runs: echo "perf_mode=$PERF_MODE" >> "$GITHUB_OUTPUT" echo "Mode: $PERF_MODE ($REASON)" - - name: Find prior run with performance baseline artifact + - name: Find prior run with performance baseline artifact (from target branch) id: find-baseline if: ${{ steps.resolve-mode.outputs.perf_mode == '--check' }} shell: bash @@ -116,7 +138,8 @@ runs: ARTIFACT_NAME: ${{ inputs.artifact_name }} run: | set -euo pipefail - echo "Searching $TARGET_REPO (workflow: $WORKFLOW_NAME, branch: $BRANCH) for artifact $ARTIFACT_NAME..." + echo "Searching $TARGET_REPO (workflow: $WORKFLOW_NAME, branch: $BRANCH) for artifact $ARTIFACT_NAME" + RUN_IDS=$(gh run list \ --repo "$TARGET_REPO" \ --workflow "$WORKFLOW_NAME" \ @@ -134,17 +157,17 @@ runs: for RID in $RUN_IDS; do HAS=$(gh api "repos/$TARGET_REPO/actions/runs/$RID/artifacts" \ --jq ".artifacts | map(select(.name==\"$ARTIFACT_NAME\" and .expired==false)) | length") - echo "Run $RID → $HAS artifact(s)" + echo "Run $RID has $HAS matching artifact(s)" if [[ "$HAS" -gt 0 ]]; then FOUND="$RID"; break; fi done if [[ -z "${FOUND:-}" ]]; then - echo "No suitable run with artifact $ARTIFACT_NAME found." + echo "No suitable run with artifact $ARTIFACT_NAME found on $BRANCH." exit 1 fi echo "run_id=$FOUND" >> "$GITHUB_OUTPUT" - echo "Found performance baseline in run: $FOUND" + echo "Using performance baseline from run: $FOUND" - name: Download performance baseline artifact if: ${{ steps.resolve-mode.outputs.perf_mode == '--check' }} @@ -164,7 +187,7 @@ runs: set -euo pipefail FILE="$(find tests/perf-regression -name 'perf-regression.json' -print -quit || true)" if [[ -z "${FILE:-}" ]]; then - echo "ERROR: perf-regression.json not found." + echo "ERROR: perf-regression.json not found in downloaded artifact." ls -R tests/perf-regression || true exit 1 fi diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 3a448ee194..bb5f0daa21 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -191,7 +191,7 @@ jobs: artifact_name: perf-regression-json baseline_path: tests/perf-regression/perf-regression.json workflow_name: ${{ github.workflow }} - branch: ${{ github.head_ref || github.ref_name }} + branch: main target_repo: ${{ inputs.target_repo || github.repository }} run_command: sh run-ci-tests.sh - name: Add to job summary