Skip to content

Perf (and AB-perf) consolidation #8000

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
maxtropets wants to merge 3 commits into
base: main
Choose a base branch
from
+700 −555
Draft

Perf (and AB-perf) consolidation #8000

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
9131402
WIP
maxtropets Jul 1, 2026
1d8e84f
Merge branch 'main' into f/ab-bench-plots
maxtropets Jul 1, 2026
a27625f
Merge branch 'main' into f/ab-bench-plots
achamayou Jul 3, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
125 changes: 46 additions & 79 deletions .github/workflows/bencher-ab.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,74 +45,22 @@ jobs:
- name: Build and run benchmarks
run: |
git config --global --add safe.directory /__w/CCF/CCF
mkdir build
cd build
cmake -GNinja -DWORKER_THREADS=2 ..
ninja
# Microbenchmarks
./tests.sh -VV -L benchmark
# End to end performance tests
./tests.sh -VV -L perf -C perf
# Convert microbenchmark output to bencher json
source env/bin/activate
PYTHONPATH=../tests python convert_pico_to_bencher.py
./scripts/bench-ab.sh run --results-dir build/bench-ab

- name: Upload PR results
- name: Upload benchmark logs
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
with:
name: benchmark-pr-${{ github.run_id }}
path: build/bencher.json
name: benchmark-pr-logs-${{ github.run_id }}
path: build/bench-ab/logs/*.log
if-no-files-found: ignore
retention-days: 7
if: success() || failure()

benchmark_main:
name: Benchmark Main
runs-on:
[
self-hosted,
1ES.Pool=gha-vmss-d16av6-ci,
"JobId=bab_benchmark_main-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}",
]
if: *check_trigger_conditions
container:
image: mcr.microsoft.com/azurelinux/base/core:3.0
options: --user root
steps:
- name: Setup container dependencies
run: |
gpg --import /etc/pki/rpm-gpg/MICROSOFT-RPM-GPG-KEY
tdnf -y update && tdnf -y install ca-certificates git

- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
with:
ref: main
fetch-depth: 0

- name: Install dependencies
run: ./scripts/setup-ci.sh

- name: Confirm platform
run: python3 tests/infra/platform_detection.py virtual

- name: Build and run benchmarks
run: |
git config --global --add safe.directory /__w/CCF/CCF
mkdir build
cd build
cmake -GNinja -DWORKER_THREADS=2 ..
ninja
# Microbenchmarks
./tests.sh -VV -L benchmark
# End to end performance tests
./tests.sh -VV -L perf -C perf
# Convert microbenchmark output to bencher json
source env/bin/activate
PYTHONPATH=../tests python convert_pico_to_bencher.py

- name: Upload main results
- name: Upload PR results
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
with:
name: benchmark-main-${{ github.run_id }}
path: build/bencher.json
name: benchmark-pr-${{ github.run_id }}
path: build/bench-ab/bencher-pr-*.json
retention-days: 7

compare:
Expand All @@ -126,35 +74,54 @@ jobs:
container:
image: mcr.microsoft.com/azurelinux/base/core:3.0
options: --user root
needs: [benchmark_pr, benchmark_main]
needs: [benchmark_pr]
if: *check_trigger_conditions
steps:
- name: Setup container dependencies
run: |
gpg --import /etc/pki/rpm-gpg/MICROSOFT-RPM-GPG-KEY
tdnf -y update && tdnf -y install ca-certificates git
tdnf -y update && tdnf -y install ca-certificates git gh

- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0

- name: Download artifacts
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
with:
pattern: benchmark-*-${{ github.run_id }}
merge-multiple: false
pattern: benchmark-pr-${{ github.run_id }}
path: benchmark-pr-${{ github.run_id }}
merge-multiple: true

- name: Generate comparison
- name: Restore main perf results
run: |
echo "# Benchmark Comparison: main vs PR" > report.md
echo "" >> report.md
echo "**PR Commit:** \`${{ github.event.pull_request.head.sha }}\`" >> report.md
echo "**Base Commit:** \`${{ github.event.pull_request.base.sha }}\`" >> report.md
echo "**Run ID:** [${{ github.run_id }}](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})" >> report.md
echo "" >> report.md
echo '```' >> report.md
python3 scripts/compare_bencher_ab.py \
benchmark-main-${{ github.run_id }}/bencher.json \
benchmark-pr-${{ github.run_id }}/bencher.json \
--label1 "main" --label2 "PR" >> report.md
echo '```' >> report.md
set -euo pipefail
mkdir -p perf
rm -f perf/*.json
git config --global --add safe.directory "$GITHUB_WORKSPACE"
run_ids=$(gh api "repos/${{ github.repository }}/actions/artifacts?name=perf-bench-virtual-main&per_page=100" \
--jq '[.artifacts[] | select(.expired == false)] | sort_by(.created_at) | reverse | .[0:10] | .[].workflow_run.id')
restored=false
for run_id in $run_ids; do
download_dir="prev_artifact/$run_id"
mkdir -p "$download_dir"
if gh run download "$run_id" --name "perf-bench-virtual-main" --dir "$download_dir"; then
if [[ -n "$(find "$download_dir" -name '*.json' -print -quit)" ]]; then
restored=true
fi
fi
done
if [[ "$restored" != "true" ]]; then
echo "No main perf artifacts restored"
exit 1
fi
find prev_artifact -name '*.json' -exec cp {} perf/ \;
env:
GH_TOKEN: ${{ github.token }}

cat report.md >> $GITHUB_STEP_SUMMARY
- name: Generate comparison
run: |
./scripts/bench-ab.sh report \
--main-perf-dir perf \
--results-dir benchmark-pr-${{ github.run_id }} \
--label PR \
--output report.md \
--summary
209 changes: 209 additions & 0 deletions scripts/bench-ab.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,209 @@
#!/bin/bash
# Copyright (c) Microsoft Corporation. All rights reserved.
# Licensed under the Apache 2.0 License.

set -euo pipefail

SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
ROOT_DIR=$( dirname "$SCRIPT_DIR" )

usage() {
cat <<'EOF'
Usage: scripts/bench-ab.sh [run|report|local] [options]

No command means local: restore main perf, run benchmarks, and write a report.

Options:
--build-dir DIR Build directory. Default: build-bench-ab
--results-dir DIR Benchmark JSON directory. Default: build/bench-ab
--main-perf-dir DIR Main-branch perf JSON directory. Default: build/bench-ab/main
--output FILE Markdown report path. Default: RESULTS_DIR/report.md
--iterations N Benchmark repetitions. Default: 3
--label LABEL Report label. Default: run
--summary Append report to GITHUB_STEP_SUMMARY
EOF
}

abs_path() {
if [[ "$1" == /* ]]; then
echo "$1"
else
echo "$ROOT_DIR/$1"
fi
}

cmd=local
case "${1:-}" in
-h|--help)
usage
exit 0
;;
run|report|local)
cmd=$1
shift
;;
esac

build_dir=build-bench-ab
results_dir=build/bench-ab
main_perf_dir=build/bench-ab/main
output_file=
iterations=3
label=run
summary=false
repository=${GITHUB_REPOSITORY:-microsoft/CCF}
main_artifact=perf-bench-virtual-main

while [[ $# -gt 0 ]]; do
opt=$1
case "$opt" in
--build-dir) build_dir=${2:?$opt requires a value}; shift 2 ;;
--results-dir) results_dir=${2:?$opt requires a value}; shift 2 ;;
--main-perf-dir) main_perf_dir=${2:?$opt requires a value}; shift 2 ;;
--output) output_file=${2:?$opt requires a value}; shift 2 ;;
--iterations) iterations=${2:?$opt requires a value}; shift 2 ;;
--label) label=${2:?$opt requires a value}; shift 2 ;;
--summary) summary=true; shift ;;
*) echo "Unknown argument: $opt" >&2; usage >&2; exit 1 ;;
esac
done

if ! [[ "$iterations" =~ ^[1-9][0-9]*$ ]]; then
echo "--iterations must be a positive integer" >&2
exit 1
fi

output_file=${output_file:-"$results_dir/report.md"}

build_dir=$(abs_path "$build_dir")
results_dir=$(abs_path "$results_dir")
main_perf_dir=$(abs_path "$main_perf_dir")
output_file=$(abs_path "$output_file")

restore_main_perf() {
mkdir -p "$main_perf_dir"
tmp_dir=$(mktemp -d)
restored=false

echo "Restoring latest main perf results..."
if command -v gh >/dev/null 2>&1; then
run_ids=$(gh api "repos/$repository/actions/artifacts?name=$main_artifact&per_page=100" \
--jq '[.artifacts[] | select(.expired == false)] | sort_by(.created_at) | reverse | .[0:10] | .[].workflow_run.id') || run_ids=

for run_id in $run_ids; do
download_dir="$tmp_dir/$run_id"
mkdir -p "$download_dir"
if gh run download "$run_id" --repo "$repository" --name "$main_artifact" --dir "$download_dir" >/dev/null 2>&1; then
[[ -n "$(find "$download_dir" -name '*.json' -print -quit)" ]] && restored=true
fi
done
elif command -v curl >/dev/null 2>&1 && command -v unzip >/dev/null 2>&1; then
artifacts_json="$tmp_dir/artifacts.json"
if curl -fsSL \
-H "Accept: application/vnd.github+json" \
"https://api.github.com/repos/$repository/actions/artifacts?name=$main_artifact&per_page=100" \
-o "$artifacts_json"; then
python3 - "$artifacts_json" > "$tmp_dir/urls" <<'PY'
import json
import sys

with open(sys.argv[1], encoding="utf-8") as f:
data = json.load(f)

artifacts = [
artifact
for artifact in data.get("artifacts", [])
if not artifact.get("expired")
]
for artifact in sorted(
artifacts, key=lambda artifact: artifact.get("created_at", ""), reverse=True
)[:10]:
print(artifact["archive_download_url"])
PY

index=0
while IFS= read -r url; do
index=$((index + 1))
archive="$tmp_dir/$index.zip"
download_dir="$tmp_dir/$index"
mkdir -p "$download_dir"
if curl -fsSL -L "$url" -o "$archive" &&
unzip -q "$archive" -d "$download_dir"; then
[[ -n "$(find "$download_dir" -name '*.json' -print -quit)" ]] && restored=true
fi
done < "$tmp_dir/urls"
fi
fi

if [[ "$restored" == "true" ]]; then
rm -f "$main_perf_dir"/*.json
find "$tmp_dir" -mindepth 2 -name '*.json' -exec cp {} "$main_perf_dir"/ \;
echo "Main perf results saved in $main_perf_dir"
else
echo "Could not restore main perf results; using $main_perf_dir" >&2
fi
rm -rf "$tmp_dir"
}

run_benchmarks() {
mkdir -p "$build_dir" "$results_dir"
rm -f "$results_dir"/bencher-pr-*.json
log_dir="$results_dir/logs"
mkdir -p "$log_dir"
rm -f "$log_dir"/*.log

build_log="$log_dir/build.log"
echo "Building..."
{
cmake -S "$ROOT_DIR" -B "$build_dir" -GNinja -DWORKER_THREADS=2 &&
cmake --build "$build_dir"
} > "$build_log" 2>&1 || {
echo "Build failed. See $build_log" >&2
exit 1
}

pushd "$build_dir" >/dev/null
for ((i = 1; i <= iterations; i++)); do
echo "Running $i/$iterations..."
run_log="$log_dir/iteration-${i}.log"
rm -f bencher.json
{
./tests.sh -VV -L benchmark &&
./tests.sh -VV -L perf -C perf &&
PYTHONPATH="$ROOT_DIR/tests" env/bin/python convert_pico_to_bencher.py
} > "$run_log" 2>&1 || {
echo "Iteration $i failed. See $run_log" >&2
exit 1
}
result="$results_dir/bencher-pr-${i}.json"
mv bencher.json "$result"
done
popd >/dev/null

echo "Benchmark results saved in $results_dir"
echo "Logs saved in $log_dir"
}

write_report() {
mkdir -p "$( dirname "$output_file" )"
python3 "$ROOT_DIR/scripts/compare_bencher_ab.py" \
"$main_perf_dir" \
"$results_dir" \
--label2 "$label" > "$output_file"

if [[ "$summary" == "true" ]]; then
if [[ -z "${GITHUB_STEP_SUMMARY:-}" ]]; then
echo "GITHUB_STEP_SUMMARY is not set" >&2
exit 1
fi
cat "$output_file" >> "$GITHUB_STEP_SUMMARY"
fi

echo "Report saved in $output_file"
}

case "$cmd" in
run) run_benchmarks ;;
report) write_report ;;
local) restore_main_perf; run_benchmarks; write_report; echo "Done." ;;
esac
Loading