Skip to content

Commit 07c02f2

Browse files
SeungheonOhzeme-wana
SeungheonOh
and
zeme-wana
authored
Autorun budget change comparison (#7248)
* Add useful scripts * workflow for automatically generating budget comparison * budget change * wip * wip * wip * wip * wip * wip * wip * wip * Rename Workflow and add it to the Slack Message Broker * restore golden --------- Co-authored-by: zeme-wana <[email protected]>
1 parent edc9a11 commit 07c02f2

File tree

4 files changed

+286
-0
lines changed

4 files changed

+286
-0
lines changed

‎.github/workflows/compare-golden-budgets.yaml‎

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
name: "🏆 Compare Golden Budgets"
2+
3+
on:
4+
pull_request:
5+
paths:
6+
- '**.eval.golden'
7+
8+
jobs:
9+
run-script:
10+
runs-on: ubuntu-latest
11+
12+
permissions:
13+
pull-requests: write
14+
15+
steps:
16+
- name: Checkout
17+
uses: actions/checkout@main
18+
with:
19+
fetch-depth: 5
20+
21+
- name: Make script executable
22+
run: chmod +x scripts/eval-percent-diff.sh
23+
24+
- name: Run the bash script and capture output
25+
run: ./scripts/eval-percent-diff.sh -t ${{ github.event.pull_request.base.sha }} 2>&1 | tee output.log
26+
27+
- name: Comment script output on PR
28+
uses: actions/github-script@v7
29+
with:
30+
github-token: ${{ secrets.GITHUB_TOKEN }}
31+
script: |
32+
const fs = require('fs');
33+
34+
const botCommentIdentifier = "<!-- GITHUB_ACTION_GOLDEN_FILE_BOT -->";
35+
36+
const scriptOutput = fs.readFileSync('output.log', 'utf8');
37+
38+
const headSha = context.payload.pull_request.head.sha;
39+
const baseSha = context.payload.pull_request.base.sha;
40+
const commentBody = `
41+
${botCommentIdentifier}
42+
## Execution Budget Golden Diff
43+
44+
${baseSha.substring(0, 7)} (master) vs ${headSha.substring(0, 7)}
45+
46+
<details>
47+
<summary>output</summary>
48+
49+
${scriptOutput}
50+
51+
</const>
52+
53+
This comment will get updated when changes are made.
54+
`;
55+
56+
details { data: comments } = await github.rest.issues.listComments({
57+
owner: context.repo.owner,
58+
repo: context.repo.repo,
59+
issue_number: context.issue.number,
60+
});
61+
62+
const botComment = comments.find(comment => comment.body.includes(botCommentIdentifier));
63+
64+
if (botComment) {
65+
console.log('Found existing comment. Updating...');
66+
await github.rest.issues.updateComment({
67+
owner: context.repo.owner,
68+
repo: context.repo.repo,
69+
comment_id: botComment.id,
70+
body: commentBody
71+
});
72+
console.log('Comment updated successfully.');
73+
} else {
74+
console.log('No existing comment found. Creating a new one...');
75+
await github.rest.issues.createComment({
76+
owner: context.repo.owner,
77+
repo: context.repo.repo,
78+
issue_number: context.issue.number,
79+
body: commentBody
80+
});
81+
console.log('Comment posted successfully.');
82+
}

‎.github/workflows/slack-message-broker.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ on:
2121
- "🌘 Nightly Testsuite"
2222
- "📝 Papers & Specs"
2323
- "📊 Code Coverage Report"
24+
- "🏆 Compare Golden Budgets"
2425
- "pages-build-deployment"
2526

2627
jobs:

‎scripts/eval-percent-diff.sh‎

Lines changed: 132 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,132 @@
1+
#!/usr/bin/env bash
2+
# diff‑budget.sh – show percent deltas for *.eval.golden metrics
3+
# Usage: diff‑budget.sh [-t] [<base-commit>]
4+
# -t output as Markdown tables instead of colored text
5+
6+
set -euo pipefail
7+
8+
usage() {
9+
cat <<EOF
10+
Usage: $0 [-t] [<base-commit>]
11+
12+
-t output as Markdown tables
13+
<base-commit> if given, diff that commit against your working tree;
14+
otherwise diff working tree vs HEAD.
15+
EOF
16+
exit 1
17+
}
18+
19+
# --- flag parsing ------------------------------------------------------------
20+
TFLAG=0
21+
while getopts ":t" opt; do
22+
case $opt in
23+
t) TFLAG=1 ;;
24+
*) usage ;;
25+
esac
26+
done
27+
shift $((OPTIND - 1))
28+
29+
# --- positional argument (base commit) --------------------------------------
30+
if [[ $# -gt 1 ]]; then
31+
usage
32+
elif [[ $# -eq 1 ]]; then
33+
BASE="$1"
34+
GIT_DIFF_CMD=(git diff "$BASE" -- '*.eval.golden')
35+
else
36+
GIT_DIFF_CMD=(git diff -- '*.eval.golden')
37+
fi
38+
39+
# --- run diff and post‑process ----------------------------------------------
40+
if [[ $TFLAG -eq 1 ]]; then
41+
# Markdown table output
42+
"${GIT_DIFF_CMD[@]}" | gawk '
43+
function strip_us(num, t) { gsub(/_/, "", num); return num+0 }
44+
function pct(old, new) {
45+
if (old==0) return "N/A";
46+
return sprintf("%+.2f%%", 100*(new-old)/old)
47+
}
48+
BEGIN {
49+
known["CPU"]=1; known["Memory"]=1;
50+
known["Term Size"]=1; known["Flat Size"]=1;
51+
file=""; seen=0;
52+
}
53+
# new file header: reset seen flag, store filename
54+
/^--- a\/.*\.eval\.golden/ {
55+
file = substr($2,3)
56+
seen = 0
57+
next
58+
}
59+
# capture old values
60+
/^-[A-Za-z ]+:[[:space:]]+[0-9_]+/ {
61+
match($0,/^-([A-Za-z ]+):[[:space:]]+([0-9_]+)/,m)
62+
label=m[1]; if (!(label in known)) next
63+
old[label]=strip_us(m[2]); old_raw[label]=m[2]
64+
next
65+
}
66+
# capture new values and print row (with header if first)
67+
/^\+[A-Za-z ]+:[[:space:]]+[0-9_]+/ {
68+
match($0,/^\+([A-Za-z ]+):[[:space:]]+([0-9_]+)/,m)
69+
label=m[1]; if (!(label in known)) next
70+
newval=strip_us(m[2]); raw_new=m[2]
71+
if (label in old) {
72+
delta=pct(old[label],newval)
73+
if (!seen) {
74+
print ""
75+
print "### " file
76+
print ""
77+
print "| Metric | Old | New | Δ% |"
78+
print "|------------|---------|---------|---------|"
79+
seen=1
80+
}
81+
printf("| %-10s | `%7s` | `%7s` | %7s |\n",
82+
label, old_raw[label], raw_new, delta)
83+
delete old[label]
84+
}
85+
}
86+
'
87+
else
88+
# Colored text output
89+
"${GIT_DIFF_CMD[@]}" | gawk -v ESC="$(printf '\033')" '
90+
function strip_us(num, t) { gsub(/_/, "", num); return num+0 }
91+
function pct(old, new) {
92+
if (old==0) return "N/A";
93+
return sprintf("%+.2f%%", 100*(new-old)/old)
94+
}
95+
BEGIN {
96+
known["CPU"]=1; known["Memory"]=1;
97+
known["Term Size"]=1; known["Flat Size"]=1;
98+
red = ESC "[31m"; green = ESC "[32m"; reset = ESC "[0m";
99+
file=""; seen=0;
100+
}
101+
# new file header: reset seen flag, store filename
102+
/^--- a\/.*\.eval\.golden/ {
103+
file = substr($2,3)
104+
seen = 0
105+
next
106+
}
107+
# capture old values
108+
/^-[A-Za-z ]+:[[:space:]]+[0-9_]+/ {
109+
match($0,/^-([A-Za-z ]+):[[:space:]]+([0-9_]+)/,m)
110+
label=m[1]; if (!(label in known)) next
111+
old[label]=strip_us(m[2]); old_raw[label]=m[2]
112+
next
113+
}
114+
# capture new values and print line (with filename if first)
115+
/^\+[A-Za-z ]+:[[:space:]]+[0-9_]+/ {
116+
match($0,/^\+([A-Za-z ]+):[[:space:]]+([0-9_]+)/,m)
117+
label=m[1]; if (!(label in known)) next
118+
newval=strip_us(m[2]); raw_new=m[2]
119+
if (label in old) {
120+
delta=pct(old[label],newval)
121+
color=(newval<old[label]? green: red)
122+
if (!seen) {
123+
print "\n" file
124+
seen=1
125+
}
126+
printf(" %-11s %10s -> %s%10s (%s)%s\n",
127+
label ":", old_raw[label], color, raw_new, delta, reset)
128+
delete old[label]
129+
}
130+
}
131+
'
132+
fi

‎scripts/regen-goldens.sh‎

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
#!/usr/bin/env bash
2+
set -euo pipefail
3+
4+
# List of sub‑projects and their tests
5+
projects=(
6+
"plutus-conformance"
7+
"plutus-tx-plugin"
8+
"plutus-ledger-api"
9+
"plutus-benchmark"
10+
"cardano-constitution"
11+
"plutus-tx"
12+
"plutus-core"
13+
)
14+
15+
tests_plutus_conformance=(
16+
"cabal run haskell-conformance -- --accept"
17+
"cabal run haskell-steppable-conformance -- --accept"
18+
"cabal run agda-conformance -- --accept"
19+
)
20+
tests_plutus_tx_plugin=(
21+
"cabal run plutus-tx-plugin-tests -- --accept"
22+
"cabal run size -- --accept"
23+
)
24+
tests_plutus_ledger_api=(
25+
"cabal run plutus-ledger-api-test -- --accept"
26+
"cabal run plutus-ledger-api-plugin-test -- --accept"
27+
)
28+
tests_plutus_benchmark=(
29+
"cabal run plutus-benchmark-nofib-tests -- --accept"
30+
"cabal run plutus-benchmark-lists-tests -- --accept"
31+
"cabal run ed25519-costs-test -- --accept"
32+
"cabal run bls12-381-costs-test -- --accept"
33+
"cabal run plutus-benchmark-script-contexts-tests -- --accept"
34+
"cabal run plutus-benchmark-marlowe-tests -- --accept"
35+
"cabal run bitwise-test -- --accept"
36+
"cabal run coop-test -- --accept"
37+
"cabal run linear-vesting-test -- --accept"
38+
"cabal run cardano-loans-test -- --accept"
39+
)
40+
tests_cardano_constitution=(
41+
"cabal run cardano-constitution-test -- --accept"
42+
)
43+
tests_plutus_tx=(
44+
"cabal run plutus-tx-test -- --accept"
45+
)
46+
tests_plutus_core=(
47+
"cabal run plutus-core-test -- --accept"
48+
"cabal run untyped-plutus-core-test -- --accept"
49+
"cabal run plutus-ir-test -- --accept"
50+
)
51+
52+
# Run all tests
53+
for project in "${projects[@]}"; do
54+
echo "=== Entering $project"
55+
cd "$project"
56+
57+
# Construct the name of the tests array (replace - with _)
58+
varname="tests_${project//-/_}[@]"
59+
for cmd in "${!varname}"; do
60+
echo "-> $cmd"
61+
if ! $cmd; then
62+
echo "FAILURE in '$project': command failed -> $cmd" >&2
63+
exit 1
64+
fi
65+
clear
66+
done
67+
68+
cd ..
69+
done
70+
71+
echo "All tests passed!"

0 commit comments

Comments
 (0)