Skip to content

Commit a277ed3

Browse files
ileitchileitch
committed
CI benchmarks
1 parent e683568 commit a277ed3

File tree

2 files changed

+263
-0
lines changed

2 files changed

+263
-0
lines changed

‎.github/benchmark-projects.json‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
{
2+
"projects": [
3+
{
4+
"name": "Alamofire",
5+
"url": "https://github.com/Alamofire/Alamofire.git",
6+
"ref": "5.10.0"
7+
},
8+
{
9+
"name": "stripe-ios",
10+
"url": "https://github.com/stripe/stripe-ios.git",
11+
"ref": "25.3.1"
12+
}
13+
]
14+
}

‎.github/workflows/benchmark.yml‎

Lines changed: 249 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,249 @@
1+
name: Benchmark
2+
on:
3+
issue_comment:
4+
types: [created]
5+
pull_request:
6+
types: [labeled]
7+
8+
jobs:
9+
setup:
10+
name: Setup
11+
if: |
12+
(github.event_name == 'issue_comment' && github.event.issue.pull_request && contains(github.event.comment.body, '/benchmark')) ||
13+
(github.event_name == 'pull_request' && github.event.label.name == 'benchmark')
14+
runs-on: macos-26
15+
permissions:
16+
contents: read
17+
pull-requests: write
18+
outputs:
19+
matrix: ${{ steps.matrix.outputs.matrix }}
20+
pr_number: ${{ steps.pr-number.outputs.number }}
21+
head_sha: ${{ steps.pr.outputs.head_sha }}
22+
merge_base: ${{ steps.commits.outputs.merge_base }}
23+
steps:
24+
- name: Get PR number
25+
id: pr-number
26+
run: |
27+
if [ "${{ github.event_name }}" = "issue_comment" ]; then
28+
echo "number=${{ github.event.issue.number }}" >> "$GITHUB_OUTPUT"
29+
else
30+
echo "number=${{ github.event.pull_request.number }}" >> "$GITHUB_OUTPUT"
31+
fi
32+
33+
- name: Get PR details
34+
id: pr
35+
env:
36+
GH_TOKEN: ${{ github.token }}
37+
run: |
38+
PR_DATA=$(gh api repos/${{ github.repository }}/pulls/${{ steps.pr-number.outputs.number }})
39+
HEAD_SHA=$(echo "$PR_DATA" | jq -r '.head.sha')
40+
HEAD_REF=$(echo "$PR_DATA" | jq -r '.head.ref')
41+
BASE_REF=$(echo "$PR_DATA" | jq -r '.base.ref')
42+
echo "head_sha=$HEAD_SHA" >> "$GITHUB_OUTPUT"
43+
echo "head_ref=$HEAD_REF" >> "$GITHUB_OUTPUT"
44+
echo "base_ref=$BASE_REF" >> "$GITHUB_OUTPUT"
45+
46+
- name: Add reaction to comment
47+
if: github.event_name == 'issue_comment'
48+
env:
49+
GH_TOKEN: ${{ github.token }}
50+
run: |
51+
gh api repos/${{ github.repository }}/issues/comments/${{ github.event.comment.id }}/reactions \
52+
-f content='+1'
53+
54+
- uses: actions/checkout@master
55+
with:
56+
ref: ${{ steps.pr.outputs.head_sha }}
57+
fetch-depth: 0
58+
59+
- name: Get merge-base
60+
id: commits
61+
run: |
62+
git fetch origin ${{ steps.pr.outputs.base_ref }}
63+
MERGE_BASE=$(git merge-base HEAD origin/${{ steps.pr.outputs.base_ref }})
64+
echo "merge_base=$MERGE_BASE" >> "$GITHUB_OUTPUT"
65+
66+
- name: Generate matrix
67+
id: matrix
68+
run: |
69+
# Prepend hardcoded periphery (self) project to the list from config
70+
PERIPHERY='{"name": "periphery", "self": true}'
71+
MATRIX=$(jq -c --argjson periphery "$PERIPHERY" '{project: ([$periphery] + .projects)}' .github/benchmark-projects.json)
72+
echo "matrix=$MATRIX" >> "$GITHUB_OUTPUT"
73+
74+
- name: Select Xcode version
75+
run: sudo xcode-select -s /Applications/Xcode_26.2.0.app
76+
77+
- name: Build HEAD
78+
run: swift build -c release --product periphery
79+
80+
- name: Upload HEAD build
81+
uses: actions/upload-artifact@v4
82+
with:
83+
name: periphery-head
84+
path: .build/release/periphery
85+
86+
- name: Build merge-base
87+
run: |
88+
git checkout ${{ steps.commits.outputs.merge_base }}
89+
swift build -c release --product periphery
90+
91+
- name: Upload merge-base build
92+
uses: actions/upload-artifact@v4
93+
with:
94+
name: periphery-base
95+
path: .build/release/periphery
96+
97+
benchmark:
98+
name: Benchmark (${{ matrix.project.name }})
99+
needs: setup
100+
runs-on: macos-26
101+
permissions:
102+
contents: read
103+
strategy:
104+
fail-fast: false
105+
matrix: ${{ fromJson(needs.setup.outputs.matrix) }}
106+
steps:
107+
- uses: actions/checkout@master
108+
if: ${{ matrix.project.self }}
109+
with:
110+
ref: ${{ needs.setup.outputs.head_sha }}
111+
fetch-depth: 0
112+
113+
- name: Select Xcode version
114+
run: sudo xcode-select -s /Applications/Xcode_26.2.0.app
115+
116+
- name: Download HEAD build
117+
uses: actions/download-artifact@v4
118+
with:
119+
name: periphery-head
120+
path: .build/release
121+
122+
- name: Make binary executable
123+
run: chmod +x .build/release/periphery
124+
125+
- name: Install hyperfine
126+
run: brew install hyperfine
127+
128+
- name: Clone target project
129+
if: ${{ !matrix.project.self }}
130+
run: |
131+
git clone --depth 1 --branch ${{ matrix.project.ref }} ${{ matrix.project.url }} /tmp/target-project
132+
133+
- name: Build project
134+
working-directory: ${{ matrix.project.self && '.' || '/tmp/target-project' }}
135+
run: swift build
136+
137+
- name: Benchmark HEAD (self)
138+
if: ${{ matrix.project.self }}
139+
run: |
140+
hyperfine --show-output --warmup 3 --export-json head-benchmark.json \
141+
'./.build/release/periphery scan --quiet --skip-build'
142+
143+
- name: Benchmark HEAD (external)
144+
if: ${{ !matrix.project.self }}
145+
working-directory: /tmp/target-project
146+
run: |
147+
hyperfine --show-output --warmup 3 --export-json ${{ github.workspace }}/head-benchmark.json \
148+
'${{ github.workspace }}/.build/release/periphery scan --quiet --skip-build'
149+
150+
- name: Download merge-base build
151+
uses: actions/download-artifact@v4
152+
with:
153+
name: periphery-base
154+
path: .build/release
155+
156+
- name: Make binary executable
157+
run: chmod +x .build/release/periphery
158+
159+
- name: Benchmark merge-base (self)
160+
if: ${{ matrix.project.self }}
161+
run: |
162+
hyperfine --show-output --warmup 3 --export-json base-benchmark.json \
163+
'./.build/release/periphery scan --quiet --skip-build'
164+
165+
- name: Benchmark merge-base (external)
166+
if: ${{ !matrix.project.self }}
167+
working-directory: /tmp/target-project
168+
run: |
169+
hyperfine --show-output --warmup 3 --export-json ${{ github.workspace }}/base-benchmark.json \
170+
'${{ github.workspace }}/.build/release/periphery scan --quiet --skip-build'
171+
172+
- name: Generate result summary
173+
run: |
174+
HEAD_MEAN=$(jq '.results[0].mean' head-benchmark.json)
175+
BASE_MEAN=$(jq '.results[0].mean' base-benchmark.json)
176+
HEAD_STDDEV=$(jq '.results[0].stddev' head-benchmark.json)
177+
BASE_STDDEV=$(jq '.results[0].stddev' base-benchmark.json)
178+
CHANGE=$(echo "scale=2; (($HEAD_MEAN - $BASE_MEAN) / $BASE_MEAN) * 100" | bc)
179+
180+
jq -n \
181+
--arg name "${{ matrix.project.name }}" \
182+
--argjson head_mean "$HEAD_MEAN" \
183+
--argjson base_mean "$BASE_MEAN" \
184+
--argjson head_stddev "$HEAD_STDDEV" \
185+
--argjson base_stddev "$BASE_STDDEV" \
186+
--argjson change "$CHANGE" \
187+
'{name: $name, head_mean: $head_mean, base_mean: $base_mean, head_stddev: $head_stddev, base_stddev: $base_stddev, change: $change}' \
188+
> result-${{ matrix.project.name }}.json
189+
190+
- name: Upload benchmark results
191+
uses: actions/upload-artifact@v4
192+
with:
193+
name: benchmark-${{ matrix.project.name }}
194+
path: |
195+
head-benchmark.json
196+
base-benchmark.json
197+
result-${{ matrix.project.name }}.json
198+
199+
summary:
200+
name: Post Results
201+
needs: [setup, benchmark]
202+
runs-on: ubuntu-latest
203+
permissions:
204+
pull-requests: write
205+
steps:
206+
- name: Download all artifacts
207+
uses: actions/download-artifact@v4
208+
with:
209+
pattern: benchmark-*
210+
merge-multiple: true
211+
212+
- name: Generate comment
213+
id: comment
214+
run: |
215+
echo "## Benchmark Results" > comment.md
216+
echo "" >> comment.md
217+
echo "Comparing \`${{ needs.setup.outputs.merge_base }}\` (merge-base) → \`${{ needs.setup.outputs.head_sha }}\` (HEAD)" >> comment.md
218+
echo "" >> comment.md
219+
echo "| Project | Merge-base (s) | HEAD (s) | Change |" >> comment.md
220+
echo "|---------|----------------|----------|--------|" >> comment.md
221+
222+
for f in result-*.json; do
223+
NAME=$(jq -r '.name' "$f")
224+
BASE_MEAN=$(jq -r '.base_mean' "$f")
225+
HEAD_MEAN=$(jq -r '.head_mean' "$f")
226+
BASE_STDDEV=$(jq -r '.base_stddev' "$f")
227+
HEAD_STDDEV=$(jq -r '.head_stddev' "$f")
228+
CHANGE=$(jq -r '.change' "$f")
229+
230+
printf "| %s | %.3f ±%.3f | %.3f ±%.3f | %+.1f%% |\n" \
231+
"$NAME" "$BASE_MEAN" "$BASE_STDDEV" "$HEAD_MEAN" "$HEAD_STDDEV" "$CHANGE" >> comment.md
232+
done
233+
234+
- name: Post comment
235+
env:
236+
GH_TOKEN: ${{ github.token }}
237+
run: |
238+
gh pr comment ${{ needs.setup.outputs.pr_number }} \
239+
--repo ${{ github.repository }} \
240+
--body-file comment.md
241+
242+
- name: Remove benchmark label
243+
if: github.event_name == 'pull_request'
244+
env:
245+
GH_TOKEN: ${{ github.token }}
246+
run: |
247+
gh pr edit ${{ needs.setup.outputs.pr_number }} \
248+
--repo ${{ github.repository }} \
249+
--remove-label benchmark

0 commit comments

Comments
 (0)