Skip to content
This repository was archived by the owner on Sep 9, 2025. It is now read-only.

Commit 879b026

Browse files
author
Hendrik van Antwerpen
authored
Merge pull request #219 from github/perf
Add CI job to measure and report performance impact of PRs
2 parents 6c09f75 + 267af82 commit 879b026

File tree

3 files changed

+354
-0
lines changed

3 files changed

+354
-0
lines changed

‎.github/workflows/perf.yml‎

Lines changed: 303 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,303 @@
1+
name: Performance testing
2+
on:
3+
pull_request:
4+
paths:
5+
- 'stack-graphs/**'
6+
env:
7+
BASE_REPO: ${{ github.event.pull_request.base.repo.owner.login }}/${{ github.event.pull_request.base.repo.name }}
8+
BASE_SHA: ${{ github.event.pull_request.base.sha }}
9+
BASE_DIR: base
10+
BASE_ARTIFACT: base-perf-results
11+
HEAD_REPO: ${{ github.event.pull_request.head.repo.owner.login }}/${{ github.event.pull_request.head.repo.name }}
12+
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
13+
HEAD_DIR: head
14+
HEAD_ARTIFACT: head-perf-results
15+
TEST_REPO: microsoft/TypeScript
16+
TEST_REF: v4.9.5
17+
TEST_DIR: test
18+
TEST_SRC: src/compiler
19+
MASSIF_OUT: perf.out
20+
MASSIF_REPORT: perf.txt
21+
TSSG_TS: tree-sitter-stack-graphs-typescript
22+
23+
jobs:
24+
##
25+
## Determine relevant changes
26+
##
27+
changes:
28+
runs-on: ubuntu-latest
29+
outputs:
30+
base-sha: ${{ env.BASE_SHA }}
31+
head-sha: ${{ env.HEAD_SHA }}
32+
done: ${{ steps.done.outputs.cache-hit }}
33+
steps:
34+
- name: "Checkout base code"
35+
uses: actions/checkout@v3
36+
with:
37+
repository: ${{ env.BASE_REPO }}
38+
ref: ${{ env.BASE_SHA }}
39+
path: ${{ env.BASE_DIR }}
40+
fetch-depth: 0
41+
- name: Find last relevant base commit
42+
run: |
43+
printf 'BASE_SHA=%s\n' "$(git rev-list -1 ${{ env.BASE_SHA }} -- stack-graphs)" >> $GITHUB_ENV
44+
working-directory: ${{ env.BASE_DIR }}
45+
- name: "Checkout head code"
46+
uses: actions/checkout@v3
47+
with:
48+
repository: ${{ env.HEAD_REPO }}
49+
ref: ${{ env.HEAD_SHA }}
50+
path: ${{ env.HEAD_DIR }}
51+
fetch-depth: 0
52+
- name: "Find last relevant head commit"
53+
run: |
54+
printf 'HEAD_SHA=%s\n' "$(git rev-list -1 ${{ env.HEAD_SHA }} -- stack-graphs)" >> $GITHUB_ENV
55+
working-directory: ${{ env.HEAD_DIR }}
56+
- name: "Check cached status"
57+
id: done
58+
uses: actions/cache/restore@v3
59+
with:
60+
path: done
61+
key: ${{ runner.os }}-perf-tested-${{ env.BASE_REPO }}@${{ env.BASE_SHA }}-${{ env.HEAD_REPO }}@${{ env.HEAD_SHA }}-${{ env.TEST_REPO }}@${{ env.TEST_REF }}/${{ env.TEST_SRC }}
62+
lookup-only: 'true'
63+
64+
##
65+
## Base performance
66+
##
67+
base-perf:
68+
runs-on: ubuntu-latest
69+
needs: changes
70+
if: needs.changes.outputs.done != 'true'
71+
env:
72+
BASE_SHA: ${{ needs.changes.outputs.base-sha }}
73+
steps:
74+
#
75+
# Install tools
76+
#
77+
- name: Install Rust environment
78+
uses: hecrj/setup-rust-action@v1
79+
with:
80+
rust-version: stable
81+
- name: Cache Rust dependencies
82+
uses: actions/cache@v3
83+
with:
84+
path: |
85+
~/.cargo
86+
key: ${{ runner.OS }}-cargo-home
87+
- name: Install valgrind
88+
run: |
89+
sudo apt-get update
90+
sudo apt-get install -y valgrind
91+
#
92+
# Cache results
93+
#
94+
- name: "Cache base result"
95+
id: cache-base-result
96+
uses: actions/cache@v3
97+
with:
98+
path: |
99+
${{ env.MASSIF_OUT }}
100+
${{ env.MASSIF_REPORT }}
101+
key: ${{ runner.os }}-perf-result-${{ env.BASE_REPO }}@${{ env.BASE_SHA }}-${{ env.TEST_REPO }}@${{ env.TEST_REF }}/${{ env.TEST_SRC }}
102+
#
103+
# Checkout code
104+
#
105+
- name: "Checkout base code"
106+
if: steps.cache-base-result.outputs.cache-hit != 'true'
107+
uses: actions/checkout@v3
108+
with:
109+
repository: ${{ env.BASE_REPO }}
110+
ref: ${{ env.BASE_SHA }}
111+
path: ${{ env.BASE_DIR }}
112+
#
113+
# Build code
114+
#
115+
- name: "Build base CLI"
116+
if: steps.cache-base-result.outputs.cache-hit != 'true'
117+
run: cargo build --package ${{ env.TSSG_TS }} --features cli --release
118+
working-directory: ${{ env.BASE_DIR }}
119+
env:
120+
CARGO_PROFILE_RELEASE_DEBUG: true
121+
#
122+
# Test performance
123+
#
124+
- name: Checkout test code
125+
if: steps.cache-base-result.outputs.cache-hit != 'true'
126+
uses: actions/checkout@v3
127+
with:
128+
repository: ${{ env.TEST_REPO }}
129+
ref: ${{ env.TEST_REF }}
130+
path: ${{ env.TEST_DIR }}
131+
- name: Profile base memory
132+
if: steps.cache-base-result.outputs.cache-hit != 'true'
133+
run: |
134+
valgrind \
135+
--tool=massif \
136+
--massif-out-file=${{ env.MASSIF_OUT }} \
137+
${{ env.BASE_DIR }}/target/release/${{ env.TSSG_TS }} \
138+
analyze --max-file-time=30 --hide-error-details -- ${{ env.TEST_DIR }}/${{ env.TEST_SRC }}
139+
ms_print ${{ env.MASSIF_OUT }} > ${{ env.MASSIF_REPORT }}
140+
#
141+
# Upload results
142+
#
143+
- name: Upload results
144+
uses: actions/upload-artifact@v3
145+
with:
146+
name: ${{ env.BASE_ARTIFACT }}
147+
path: |
148+
${{ env.MASSIF_OUT }}
149+
${{ env.MASSIF_REPORT }}
150+
151+
##
152+
## Head performance
153+
##
154+
head-perf:
155+
runs-on: ubuntu-latest
156+
needs: changes
157+
if: needs.changes.outputs.done != 'true'
158+
env:
159+
HEAD_SHA: ${{ needs.changes.outputs.head-sha }}
160+
steps:
161+
#
162+
# Install tools
163+
#
164+
- name: Install Rust environment
165+
uses: hecrj/setup-rust-action@v1
166+
with:
167+
rust-version: stable
168+
- name: Cache Rust dependencies
169+
uses: actions/cache@v3
170+
with:
171+
path: |
172+
~/.cargo
173+
key: ${{ runner.OS }}-cargo-home
174+
- name: Install valgrind
175+
run: |
176+
sudo apt-get update
177+
sudo apt-get install -y valgrind
178+
#
179+
# Cache results
180+
#
181+
- name: "Cache head result"
182+
id: cache-head-result
183+
uses: actions/cache@v3
184+
with:
185+
path: |
186+
${{ env.MASSIF_OUT }}
187+
${{ env.MASSIF_REPORT }}
188+
key: ${{ runner.os }}-perf-result-${{ env.HEAD_REPO }}@${{ env.HEAD_SHA }}-${{ env.TEST_REPO }}@${{ env.TEST_REF }}/${{ env.TEST_SRC }}
189+
#
190+
# Checkout code
191+
#
192+
- name: "Checkout head code"
193+
if: steps.cache-head-result.outputs.cache-hit != 'true'
194+
uses: actions/checkout@v3
195+
with:
196+
repository: ${{ env.HEAD_REPO }}
197+
ref: ${{ env.HEAD_SHA }}
198+
path: ${{ env.HEAD_DIR }}
199+
#
200+
# Build code
201+
#
202+
- name: "Build head CLI"
203+
if: steps.cache-head-result.outputs.cache-hit != 'true'
204+
run: cargo build --package ${{ env.TSSG_TS }} --features cli --release
205+
working-directory: ${{ env.HEAD_DIR }}
206+
env:
207+
CARGO_PROFILE_RELEASE_DEBUG: true
208+
#
209+
# Test performance
210+
#
211+
- name: Checkout test code
212+
if: steps.cache-head-result.outputs.cache-hit != 'true'
213+
uses: actions/checkout@v3
214+
with:
215+
repository: ${{ env.TEST_REPO }}
216+
ref: ${{ env.TEST_REF }}
217+
path: ${{ env.TEST_DIR }}
218+
- name: Profile head memory
219+
if: steps.cache-head-result.outputs.cache-hit != 'true'
220+
run: |
221+
valgrind \
222+
--tool=massif \
223+
--massif-out-file=${{ env.MASSIF_OUT }} \
224+
${{ env.HEAD_DIR }}/target/release/${{ env.TSSG_TS }} \
225+
analyze --max-file-time=30 --hide-error-details -- ${{ env.TEST_DIR }}/${{ env.TEST_SRC }}
226+
ms_print ${{ env.MASSIF_OUT }} > ${{ env.MASSIF_REPORT }}
227+
#
228+
# Upload results
229+
#
230+
- name: Upload results
231+
uses: actions/upload-artifact@v3
232+
with:
233+
name: ${{ env.HEAD_ARTIFACT }}
234+
path: |
235+
${{ env.MASSIF_OUT }}
236+
${{ env.MASSIF_REPORT }}
237+
238+
##
239+
## Performance summary
240+
##
241+
perf-summary:
242+
runs-on: ubuntu-latest
243+
needs:
244+
- changes
245+
- base-perf
246+
- head-perf
247+
if: needs.changes.outputs.done != 'true'
248+
env:
249+
BASE_SHA: ${{ needs.changes.outputs.base-sha }}
250+
HEAD_SHA: ${{ needs.changes.outputs.head-sha }}
251+
SRC_DIR: src
252+
COMMENT_JSON: perf-summary.json
253+
steps:
254+
#
255+
# Install tools
256+
#
257+
- name: Install valgrind
258+
run: |
259+
sudo apt-get update
260+
sudo apt-get install -y valgrind
261+
#
262+
# Download results
263+
#
264+
- name: Download base results
265+
uses: actions/download-artifact@v3
266+
with:
267+
name: ${{ env.BASE_ARTIFACT }}
268+
path: ${{ env.BASE_ARTIFACT }}
269+
- name: Download head results
270+
uses: actions/download-artifact@v3
271+
with:
272+
name: ${{ env.HEAD_ARTIFACT }}
273+
path: ${{ env.HEAD_ARTIFACT }}
274+
#
275+
# Create report
276+
#
277+
- name: "Checkout code"
278+
uses: actions/checkout@v3
279+
with:
280+
path: ${{ env.SRC_DIR }}
281+
- name: Generate summary
282+
run: |
283+
${{ env.SRC_DIR }}/script/ci-perf-summary-md \
284+
${{ env.BASE_ARTIFACT }}/${{ env.MASSIF_OUT }} \
285+
${{ env.HEAD_ARTIFACT }}/${{ env.MASSIF_OUT }} \
286+
'Comparing base ${{ env.BASE_REPO }}@${{ env.BASE_SHA }} with head ${{ env.HEAD_REPO }}@${{ env.HEAD_SHA }} on [${{ env.TEST_REPO }}@${{ env.TEST_REF }}](${{ github.server_url }}/${{ env.TEST_REPO }}/tree/${{ env.TEST_REF }}). For details see [workflow](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) artifacts. _Note that performance is tested on the last commits with changes in `stack-graphs`, not on every commit._' \
287+
| ${{ env.SRC_DIR }}/script/ci-comment-json > ${{ env.COMMENT_JSON }}
288+
- name: Add summary comment to PR
289+
run: |
290+
curl \
291+
-X POST \
292+
-H "Accept: application/vnd.github+json" \
293+
-H "Authorization: Bearer ${{ github.token }}" \
294+
-H "X-GitHub-Api-Version: 2022-11-28" \
295+
${{ github.event.pull_request.comments_url }} \
296+
-d '@${{ env.COMMENT_JSON }}'
297+
- name: Create status marker
298+
run: touch done
299+
- name: "Cache status"
300+
uses: actions/cache/save@v3
301+
with:
302+
path: done
303+
key: ${{ runner.os }}-perf-tested-${{ env.BASE_REPO }}@${{ env.BASE_SHA }}-${{ env.HEAD_REPO }}@${{ env.HEAD_SHA }}-${{ env.TEST_REPO }}@${{ env.TEST_REF }}/${{ env.TEST_SRC }}

‎script/ci-comment-json‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
#!/usr/bin/env bash
2+
3+
set -eu
4+
5+
printf '{"body":"'
6+
cat | awk -v ORS='\\n' '1'
7+
printf '"}'

‎script/ci-perf-summary-md‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
#!/usr/bin/env bash
2+
3+
set -eu
4+
5+
graph_w=72
6+
graph_h=12
7+
8+
summary_lines=$((graph_h + 10))
9+
10+
if [ $# -lt 2 ]; then
11+
echo "Usage: $0 MASSIF_OUT_BEFORE MASSIF_OUT_AFTER DESC?" 1>&2
12+
exit 1
13+
fi
14+
massif_before="$1"
15+
massif_after="$2"
16+
shift 2
17+
18+
desc=
19+
if [ $# -gt 0 ]; then
20+
desc="$1"
21+
fi
22+
shift 1
23+
24+
printf '## Performance Summary\n'
25+
printf '\n'
26+
if [ -n "$desc" ]; then
27+
printf '%s\n' "$desc"
28+
printf '\n'
29+
fi
30+
printf '<details>\n'
31+
printf '<summary>Before</summary>\n'
32+
printf '\n'
33+
printf '```\n'
34+
ms_print --x="$graph_w" --y="$graph_h" "$massif_before" | head -n +$summary_lines
35+
printf '```\n'
36+
printf '</details>\n'
37+
printf '\n'
38+
printf '<details>\n'
39+
printf '<summary>After</summary>\n'
40+
printf '\n'
41+
printf '```\n'
42+
ms_print --x="$graph_w" --y="$graph_h" "$massif_after" | head -n +$summary_lines
43+
printf '```\n'
44+
printf '</details>\n'

0 commit comments

Comments
 (0)