Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
114 changes: 112 additions & 2 deletions .github/workflows/dns-benchmark-compare.yml
Original file line number Diff line number Diff line change
Expand Up @@ -105,6 +105,16 @@ jobs:
BENCH_TIME="${{ github.event.inputs.bench_time }}"
fi

{
echo "BASE_REF=$BASE_REF"
echo "HEAD_REF=$HEAD_REF"
echo "BASE_STRATEGY=$BASE_STRATEGY"
echo "SUITE_PROFILE=$SUITE_PROFILE"
echo "SUITE_LIST=$SUITE_LIST"
echo "BENCH_COUNT=$BENCH_COUNT"
echo "BENCH_TIME=$BENCH_TIME"
} > bench-artifacts/meta.env

set +e
DNS_BENCH_PROFILE="$SUITE_PROFILE" \
DNS_BENCH_SUITES="$SUITE_LIST" \
Expand Down Expand Up @@ -148,14 +158,114 @@ jobs:
path: bench-artifacts
if-no-files-found: warn

- name: Build PR Summary
if: always() && github.event_name == 'pull_request'
shell: bash
run: |
set -euo pipefail

PR_SUMMARY="bench-artifacts/pr-summary.md"
META_FILE="bench-artifacts/meta.env"
if [[ ! -f "$META_FILE" ]]; then
exit 0
fi

# shellcheck disable=SC1090
source "$META_FILE"

total=0
passed=0
failed=0
suite_lines=""
suite_delta_lines=""

extract_time_geomean_line() {
local file="$1"
[[ -f "$file" ]] || return 1
awk '
$1=="geomean" {print; exit}
' "$file"
}

suite_delta_summary() {
local suite="$1"
local benchstat_file="bench-artifacts/$suite/benchstat_common.txt"
local geomean_line old_v new_v delta_v
geomean_line="$(extract_time_geomean_line "$benchstat_file" || true)"
if [[ -z "$geomean_line" ]]; then
echo "- \`$suite\`: n/a (no common benchmark delta)"
return 0
fi

old_v="$(awk '{print $2}' <<<"$geomean_line")"
new_v="$(awk '{print $3}' <<<"$geomean_line")"
delta_v="$(grep -Eo '([+-][0-9]+(\.[0-9]+)?%|[+-]?Inf%)' <<<"$geomean_line" | head -n1 || true)"
[[ -z "$old_v" ]] && old_v="n/a"
[[ -z "$new_v" ]] && new_v="n/a"
[[ -z "$delta_v" ]] && delta_v="n/a"

echo "- \`$suite\`: \`$old_v -> $new_v\` (Δ \`$delta_v\`)"
}

while IFS= read -r status_file; do
suite="$(basename "$(dirname "$status_file")")"
status="$(cat "$status_file" 2>/dev/null || echo 1)"
total=$((total + 1))
if [[ "$status" == "0" ]]; then
passed=$((passed + 1))
suite_lines+="- ✅ \`$suite\`: pass"$'\n'
else
failed=$((failed + 1))
suite_lines+="- ❌ \`$suite\`: failed"$'\n'
fi
suite_delta_lines+="$(suite_delta_summary "$suite")"$'\n'
done < <(find bench-artifacts -mindepth 2 -maxdepth 2 -name status.txt | sort)

overall="pass"
if [[ "$failed" -gt 0 || "$total" -eq 0 ]]; then
overall="not pass"
fi

{
echo "## DNS Benchmark Compare Summary"
echo
echo "- Before (base): \`$BASE_REF\`"
echo "- After (head): \`$HEAD_REF\`"
echo "- Base strategy: \`$BASE_STRATEGY\`"
if [[ -n "$SUITE_LIST" ]]; then
echo "- Suite selection: \`$SUITE_LIST\`"
else
echo "- Suite profile: \`$SUITE_PROFILE\`"
fi
echo "- Overall: **$overall** ($passed/$total passed)"
echo "- Workflow run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
echo "- Full report artifact: \`dns-benchmark-compare-${{ github.run_id }}\`"
echo
echo "### Suite Status"
echo
if [[ "$total" -eq 0 ]]; then
echo "- No suite status generated (benchmark setup failed early)."
else
printf "%s" "$suite_lines"
fi
echo
echo "### Suite Delta (time/op geomean)"
echo
if [[ "$total" -eq 0 ]]; then
echo "- No suite delta generated."
else
printf "%s" "$suite_delta_lines"
fi
} > "$PR_SUMMARY"

- name: Update PR Comment
if: always() && github.event_name == 'pull_request' && hashFiles('bench-artifacts/report.md') != ''
if: always() && github.event_name == 'pull_request' && hashFiles('bench-artifacts/pr-summary.md') != ''
uses: actions/github-script@v7
with:
script: |
const fs = require("fs");
const marker = "<!-- dae-dns-benchmark-compare -->";
const report = fs.readFileSync("bench-artifacts/report.md", "utf8");
const report = fs.readFileSync("bench-artifacts/pr-summary.md", "utf8");
const body = `${marker}\n${report}`;
const { owner, repo } = context.repo;
const issue_number = context.issue.number;
Expand Down
13 changes: 9 additions & 4 deletions .github/workflows/policy-smart-sci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ permissions:
jobs:
sci-1:
name: SCI-1 Strategy Correctness
if: github.event_name == 'workflow_dispatch' || startsWith(github.head_ref, 'feat/policy-smart') || startsWith(github.head_ref, 'policy-smart')
runs-on: ubuntu-22.04
env:
GOWORK: off
Expand Down Expand Up @@ -48,6 +49,7 @@ jobs:

sci-2:
name: SCI-2 TCP Retry Semantics
if: github.event_name == 'workflow_dispatch' || startsWith(github.head_ref, 'feat/policy-smart') || startsWith(github.head_ref, 'policy-smart')
runs-on: ubuntu-22.04
env:
GOWORK: off
Expand Down Expand Up @@ -89,6 +91,7 @@ jobs:

sci-3:
name: SCI-3 Concurrency Race
if: github.event_name == 'workflow_dispatch' || startsWith(github.head_ref, 'feat/policy-smart') || startsWith(github.head_ref, 'policy-smart')
runs-on: ubuntu-22.04
env:
GOWORK: off
Expand Down Expand Up @@ -134,8 +137,8 @@ jobs:
fi

report:
name: SCI PR Comment
if: always() && github.event_name == 'pull_request'
name: Update PR Comment
if: always() && github.event_name == 'pull_request' && (startsWith(github.head_ref, 'feat/policy-smart') || startsWith(github.head_ref, 'policy-smart'))
needs: [sci-1, sci-2, sci-3]
runs-on: ubuntu-22.04
permissions:
Expand Down Expand Up @@ -173,13 +176,15 @@ jobs:
{ name: "SCI-3 Concurrency Race", result: process.env.SCI3_RESULT || "unknown" },
];

const passCount = jobs.filter((j) => j.result === "success").length;
const allSuccess = jobs.every((j) => j.result === "success");
const summary = allSuccess ? "pass" : "not pass";
const body = [
marker,
"## Policy Smart SCI",
"## Policy Smart SCI Summary",
"",
`- Summary: **${summary}**`,
`- Scope: \`${context.payload.pull_request.head.ref}\` -> \`${context.payload.pull_request.base.ref}\``,
`- Overall: **${summary}** (${passCount}/${jobs.length} passed)`,
`- Workflow run: ${process.env.RUN_URL}`,
"",
"### Job Status",
Expand Down
Loading