Skip to content

Commit 3db77f5

Browse files
ileitchileitch
committed
CI benchmarks
1 parent e683568 commit 3db77f5

File tree

2 files changed

+226
-0
lines changed

2 files changed

+226
-0
lines changed

‎.github/benchmark-projects.json‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
{
2+
"projects": [
3+
{
4+
"name": "Alamofire",
5+
"url": "https://github.com/Alamofire/Alamofire.git",
6+
"ref": "5.10.0"
7+
},
8+
{
9+
"name": "stripe-ios",
10+
"url": "https://github.com/stripe/stripe-ios.git",
11+
"ref": "25.3.1"
12+
}
13+
]
14+
}

‎.github/workflows/benchmark.yml‎

Lines changed: 212 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,212 @@
1+
name: Benchmark
2+
on:
3+
issue_comment:
4+
types: [created]
5+
pull_request:
6+
types: [labeled]
7+
8+
jobs:
9+
setup:
10+
name: Setup
11+
if: |
12+
(github.event_name == 'issue_comment' && github.event.issue.pull_request && contains(github.event.comment.body, '/benchmark')) ||
13+
(github.event_name == 'pull_request' && github.event.label.name == 'benchmark')
14+
runs-on: ubuntu-latest
15+
permissions:
16+
contents: read
17+
pull-requests: write
18+
outputs:
19+
matrix: ${{ steps.matrix.outputs.matrix }}
20+
pr_number: ${{ steps.pr-number.outputs.number }}
21+
head_sha: ${{ steps.pr.outputs.head_sha }}
22+
base_ref: ${{ steps.pr.outputs.base_ref }}
23+
merge_base: ${{ steps.commits.outputs.merge_base }}
24+
steps:
25+
- name: Get PR number
26+
id: pr-number
27+
run: |
28+
if [ "${{ github.event_name }}" = "issue_comment" ]; then
29+
echo "number=${{ github.event.issue.number }}" >> "$GITHUB_OUTPUT"
30+
else
31+
echo "number=${{ github.event.pull_request.number }}" >> "$GITHUB_OUTPUT"
32+
fi
33+
34+
- name: Get PR details
35+
id: pr
36+
env:
37+
GH_TOKEN: ${{ github.token }}
38+
run: |
39+
PR_DATA=$(gh api repos/${{ github.repository }}/pulls/${{ steps.pr-number.outputs.number }})
40+
HEAD_SHA=$(echo "$PR_DATA" | jq -r '.head.sha')
41+
HEAD_REF=$(echo "$PR_DATA" | jq -r '.head.ref')
42+
BASE_REF=$(echo "$PR_DATA" | jq -r '.base.ref')
43+
echo "head_sha=$HEAD_SHA" >> "$GITHUB_OUTPUT"
44+
echo "head_ref=$HEAD_REF" >> "$GITHUB_OUTPUT"
45+
echo "base_ref=$BASE_REF" >> "$GITHUB_OUTPUT"
46+
47+
- name: Add reaction to comment
48+
if: github.event_name == 'issue_comment'
49+
env:
50+
GH_TOKEN: ${{ github.token }}
51+
run: |
52+
gh api repos/${{ github.repository }}/issues/comments/${{ github.event.comment.id }}/reactions \
53+
-f content='+1'
54+
55+
- uses: actions/checkout@master
56+
with:
57+
ref: ${{ steps.pr.outputs.head_sha }}
58+
fetch-depth: 0
59+
60+
- name: Get merge-base
61+
id: commits
62+
run: |
63+
git fetch origin ${{ steps.pr.outputs.base_ref }}
64+
MERGE_BASE=$(git merge-base HEAD origin/${{ steps.pr.outputs.base_ref }})
65+
echo "merge_base=$MERGE_BASE" >> "$GITHUB_OUTPUT"
66+
67+
- name: Generate matrix
68+
id: matrix
69+
run: |
70+
# Prepend hardcoded periphery (self) project to the list from config
71+
PERIPHERY='{"name": "periphery", "self": true}'
72+
MATRIX=$(jq -c --argjson periphery "$PERIPHERY" '{project: ([$periphery] + .projects)}' .github/benchmark-projects.json)
73+
echo "matrix=$MATRIX" >> "$GITHUB_OUTPUT"
74+
75+
benchmark:
76+
name: Benchmark (${{ matrix.project.name }})
77+
needs: setup
78+
runs-on: macos-26
79+
permissions:
80+
contents: read
81+
strategy:
82+
fail-fast: false
83+
matrix: ${{ fromJson(needs.setup.outputs.matrix) }}
84+
steps:
85+
- uses: actions/checkout@master
86+
with:
87+
ref: ${{ needs.setup.outputs.head_sha }}
88+
fetch-depth: 0
89+
90+
- name: Select Xcode version
91+
run: sudo xcode-select -s /Applications/Xcode_26.2.0.app
92+
93+
- name: Install hyperfine
94+
run: brew install hyperfine
95+
96+
- name: Build Periphery (HEAD)
97+
run: swift build -c release --product periphery --enable-index-store
98+
99+
- name: Clone target project
100+
if: ${{ !matrix.project.self }}
101+
run: |
102+
git clone --depth 1 --branch ${{ matrix.project.ref }} ${{ matrix.project.url }} /tmp/target-project
103+
104+
- name: Benchmark HEAD (self)
105+
if: ${{ matrix.project.self }}
106+
run: |
107+
hyperfine --show-output --warmup 3 --export-json head-benchmark.json \
108+
'./.build/release/periphery scan --quiet --skip-build --index-store-path .build/release/index/store'
109+
110+
- name: Benchmark HEAD (external)
111+
if: ${{ !matrix.project.self }}
112+
working-directory: /tmp/target-project
113+
run: |
114+
hyperfine --show-output --warmup 3 --export-json ${{ github.workspace }}/head-benchmark.json \
115+
'${{ github.workspace }}/.build/release/periphery scan --quiet'
116+
117+
- name: Build Periphery (merge-base)
118+
run: |
119+
git checkout ${{ needs.setup.outputs.merge_base }}
120+
swift build -c release --product periphery --enable-index-store
121+
122+
- name: Benchmark merge-base (self)
123+
if: ${{ matrix.project.self }}
124+
run: |
125+
hyperfine --show-output --warmup 3 --export-json base-benchmark.json \
126+
'./.build/release/periphery scan --quiet --skip-build --index-store-path .build/release/index/store'
127+
128+
- name: Benchmark merge-base (external)
129+
if: ${{ !matrix.project.self }}
130+
working-directory: /tmp/target-project
131+
run: |
132+
hyperfine --show-output --warmup 3 --export-json ${{ github.workspace }}/base-benchmark.json \
133+
'${{ github.workspace }}/.build/release/periphery scan --quiet'
134+
135+
- name: Generate result summary
136+
run: |
137+
HEAD_MEAN=$(jq '.results[0].mean' head-benchmark.json)
138+
BASE_MEAN=$(jq '.results[0].mean' base-benchmark.json)
139+
HEAD_STDDEV=$(jq '.results[0].stddev' head-benchmark.json)
140+
BASE_STDDEV=$(jq '.results[0].stddev' base-benchmark.json)
141+
CHANGE=$(echo "scale=2; (($HEAD_MEAN - $BASE_MEAN) / $BASE_MEAN) * 100" | bc)
142+
143+
jq -n \
144+
--arg name "${{ matrix.project.name }}" \
145+
--argjson head_mean "$HEAD_MEAN" \
146+
--argjson base_mean "$BASE_MEAN" \
147+
--argjson head_stddev "$HEAD_STDDEV" \
148+
--argjson base_stddev "$BASE_STDDEV" \
149+
--argjson change "$CHANGE" \
150+
'{name: $name, head_mean: $head_mean, base_mean: $base_mean, head_stddev: $head_stddev, base_stddev: $base_stddev, change: $change}' \
151+
> result-${{ matrix.project.name }}.json
152+
153+
- name: Upload benchmark results
154+
uses: actions/upload-artifact@v4
155+
with:
156+
name: benchmark-${{ matrix.project.name }}
157+
path: |
158+
head-benchmark.json
159+
base-benchmark.json
160+
result-${{ matrix.project.name }}.json
161+
162+
summary:
163+
name: Post Results
164+
needs: [setup, benchmark]
165+
runs-on: ubuntu-latest
166+
permissions:
167+
pull-requests: write
168+
steps:
169+
- name: Download all artifacts
170+
uses: actions/download-artifact@v4
171+
with:
172+
pattern: benchmark-*
173+
merge-multiple: true
174+
175+
- name: Generate comment
176+
id: comment
177+
run: |
178+
echo "## Benchmark Results" > comment.md
179+
echo "" >> comment.md
180+
echo "Comparing \`${{ needs.setup.outputs.merge_base }}\` (merge-base) → \`${{ needs.setup.outputs.head_sha }}\` (HEAD)" >> comment.md
181+
echo "" >> comment.md
182+
echo "| Project | Merge-base (s) | HEAD (s) | Change |" >> comment.md
183+
echo "|---------|----------------|----------|--------|" >> comment.md
184+
185+
for f in result-*.json; do
186+
NAME=$(jq -r '.name' "$f")
187+
BASE_MEAN=$(jq -r '.base_mean' "$f")
188+
HEAD_MEAN=$(jq -r '.head_mean' "$f")
189+
BASE_STDDEV=$(jq -r '.base_stddev' "$f")
190+
HEAD_STDDEV=$(jq -r '.head_stddev' "$f")
191+
CHANGE=$(jq -r '.change' "$f")
192+
193+
printf "| %s | %.3f ±%.3f | %.3f ±%.3f | %+.1f%% |\n" \
194+
"$NAME" "$BASE_MEAN" "$BASE_STDDEV" "$HEAD_MEAN" "$HEAD_STDDEV" "$CHANGE" >> comment.md
195+
done
196+
197+
- name: Post comment
198+
env:
199+
GH_TOKEN: ${{ github.token }}
200+
run: |
201+
gh pr comment ${{ needs.setup.outputs.pr_number }} \
202+
--repo ${{ github.repository }} \
203+
--body-file comment.md
204+
205+
- name: Remove benchmark label
206+
if: github.event_name == 'pull_request'
207+
env:
208+
GH_TOKEN: ${{ github.token }}
209+
run: |
210+
gh pr edit ${{ needs.setup.outputs.pr_number }} \
211+
--repo ${{ github.repository }} \
212+
--remove-label benchmark

0 commit comments

Comments
 (0)