Skip to content

Commit d281064

Browse files
dhusebyDave Grantham
dhuseby
and
Dave Grantham
authored
dhuseby/feat/perf baseline iperf (#766)
* adding iperf baseline test Signed-off-by: Dave Grantham <dwg@linuxprogrammer.org> * fixed the impls measurements Signed-off-by: Dave Grantham <dwg@linuxprogrammer.org> * fixing action and workflows to support perf properly Signed-off-by: Dave Grantham <dwg@linuxprogrammer.org> * clean up DEBUG and Rust perf warnings Signed-off-by: Dave Grantham <dwg@linuxprogrammer.org> * remove redundant detect-changes Signed-off-by: Dave Grantham <dwg@linuxprogrammer.org> * add box plotting Signed-off-by: Dave Grantham <dwg@linuxprogrammer.org> * remove box-plot.json generation Signed-off-by: Dave Grantham <dwg@linuxprogrammer.org> * fixed baselines and graphing Signed-off-by: Dave Grantham <dwg@linuxprogrammer.org> --------- Signed-off-by: Dave Grantham <dwg@linuxprogrammer.org> Co-authored-by: Dave Grantham <dwg@linuxprogrammer.org>
1 parent bd598a7 commit d281064

File tree

25 files changed

+1760
-391
lines changed

25 files changed

+1760
-391
lines changed

‎.github/actions/detect-changed-impls/action.yml‎

Lines changed: 56 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,9 @@ outputs:
2020
changed-routers:
2121
description: "Pipe-separated list of changed router IDs (hole-punch only)"
2222
value: ${{ steps.detect.outputs.changed-routers }}
23+
changed-baselines:
24+
description: "Pipe-separated list of changed baseline IDs (perf only)"
25+
value: ${{ steps.detect.outputs.changed-baselines }}
2326
has-changes:
2427
description: "Boolean indicating if any changes were detected"
2528
value: ${{ steps.detect.outputs.has-changes }}
@@ -66,6 +69,7 @@ runs:
6669
CHANGED_IMPLS=""
6770
CHANGED_RELAYS=""
6871
CHANGED_ROUTERS=""
72+
CHANGED_BASELINES=""
6973
IMPLS_YAML_CHANGED="false"
7074
IMPLS_FOLDER_CHANGED="false"
7175
@@ -140,6 +144,29 @@ runs:
140144
done
141145
fi
142146
147+
# For perf: also check baselines
148+
if [ "$TEST_TYPE" = "perf" ]; then
149+
# Check baselines
150+
OLD_BASELINES=$(yq eval '.baselines[].id' /tmp/impls-old.yaml 2>/dev/null | sort || echo "")
151+
NEW_BASELINES=$(yq eval '.baselines[].id' /tmp/impls-new.yaml 2>/dev/null | sort || echo "")
152+
153+
for baseline_id in $NEW_BASELINES; do
154+
if echo "$OLD_BASELINES" | grep -q "^${baseline_id}$"; then
155+
# Exists in old, check if content changed
156+
OLD_BASELINE=$(yq eval ".baselines[] | select(.id == \"$baseline_id\")" /tmp/impls-old.yaml 2>/dev/null || echo "")
157+
NEW_BASELINE=$(yq eval ".baselines[] | select(.id == \"$baseline_id\")" /tmp/impls-new.yaml 2>/dev/null || echo "")
158+
if [ "$OLD_BASELINE" != "$NEW_BASELINE" ]; then
159+
CHANGED_BASELINES="${CHANGED_BASELINES:+$CHANGED_BASELINES|}$baseline_id"
160+
echo " → Changed baseline: $baseline_id"
161+
fi
162+
else
163+
# New baseline
164+
CHANGED_BASELINES="${CHANGED_BASELINES:+$CHANGED_BASELINES|}$baseline_id"
165+
echo " → New baseline: $baseline_id"
166+
fi
167+
done
168+
fi
169+
143170
rm -f /tmp/impls-old.yaml /tmp/impls-new.yaml
144171
fi
145172
@@ -226,20 +253,47 @@ runs:
226253
fi
227254
done
228255
fi
256+
257+
# For perf: also check baselines
258+
if [ "$TEST_TYPE" = "perf" ]; then
259+
# Check baselines
260+
BASELINE_COUNT=$(yq eval '.baselines | length' "$IMPLS_FILE" 2>/dev/null || echo "0")
261+
for ((i=0; i<BASELINE_COUNT; i++)); do
262+
SOURCE_TYPE=$(yq eval ".baselines[$i].source.type" "$IMPLS_FILE" 2>/dev/null)
263+
if [ "$SOURCE_TYPE" = "local" ]; then
264+
SOURCE_PATH=$(yq eval ".baselines[$i].source.path" "$IMPLS_FILE")
265+
FULL_SOURCE_PATH="perf/$SOURCE_PATH"
266+
267+
if [[ "$changed_file" == "$FULL_SOURCE_PATH"* ]]; then
268+
BASELINE_ID=$(yq eval ".baselines[$i].id" "$IMPLS_FILE")
269+
if [[ "|$CHANGED_BASELINES|" != *"|$BASELINE_ID|"* ]]; then
270+
CHANGED_BASELINES="${CHANGED_BASELINES:+$CHANGED_BASELINES|}$BASELINE_ID"
271+
echo " ✓ Matched to baseline: $BASELINE_ID"
272+
fi
273+
fi
274+
fi
275+
done
276+
fi
229277
done <<< "$IMPLS_FOLDER_FILES"
230278
fi
231279
232280
# Output all results
233281
echo "changed-impls=$CHANGED_IMPLS" >> $GITHUB_OUTPUT
234282
echo "changed-relays=$CHANGED_RELAYS" >> $GITHUB_OUTPUT
235283
echo "changed-routers=$CHANGED_ROUTERS" >> $GITHUB_OUTPUT
284+
echo "changed-baselines=$CHANGED_BASELINES" >> $GITHUB_OUTPUT
236285
echo "impls-yaml-changed=$IMPLS_YAML_CHANGED" >> $GITHUB_OUTPUT
237286
echo "impls-folder-changed=$IMPLS_FOLDER_CHANGED" >> $GITHUB_OUTPUT
238287
239288
# Set has-changes
240-
if [ -n "$CHANGED_IMPLS" ] || [ -n "$CHANGED_RELAYS" ] || [ -n "$CHANGED_ROUTERS" ]; then
289+
if [ -n "$CHANGED_IMPLS" ] || [ -n "$CHANGED_RELAYS" ] || [ -n "$CHANGED_ROUTERS" ] || [ -n "$CHANGED_BASELINES" ]; then
241290
echo "has-changes=true" >> $GITHUB_OUTPUT
242-
echo "→ Summary: impls=$CHANGED_IMPLS, relays=$CHANGED_RELAYS, routers=$CHANGED_ROUTERS"
291+
# Update summary to include baselines
292+
if [ "$TEST_TYPE" = "perf" ]; then
293+
echo "→ Summary: impls=$CHANGED_IMPLS, relays=$CHANGED_RELAYS, routers=$CHANGED_ROUTERS, baselines=$CHANGED_BASELINES"
294+
else
295+
echo "→ Summary: impls=$CHANGED_IMPLS, relays=$CHANGED_RELAYS, routers=$CHANGED_ROUTERS"
296+
fi
243297
else
244298
echo "has-changes=false" >> $GITHUB_OUTPUT
245299
echo "→ No relevant changes detected"

‎.github/actions/run-bash-perf-test/action.yml‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -171,9 +171,18 @@ runs:
171171
${{ steps.find-test-pass.outputs.test-pass-dir }}/results.yaml
172172
${{ steps.find-test-pass.outputs.test-pass-dir }}/results.md
173173
${{ steps.find-test-pass.outputs.test-pass-dir }}/results.html
174-
${{ steps.find-test-pass.outputs.test-pass-dir }}/boxplot-data.json
175174
if-no-files-found: warn
176175

176+
- name: Upload box plot images
177+
if: always()
178+
uses: actions/upload-artifact@v4
179+
with:
180+
name: perf-test-box-plot-${{ steps.find-test-pass.outputs.test-pass-name }}
181+
path: |
182+
${{ steps.find-test-pass.outputs.test-pass-dir }}/boxplot-*.png
183+
if-no-files-found: ignore
184+
compression-level: 0
185+
177186
- name: Upload snapshot archive
178187
if: always() && inputs.snapshot == 'true'
179188
uses: actions/upload-artifact@v4

‎.github/workflows/daily-full-interop.yml‎

Lines changed: 0 additions & 117 deletions
Original file line numberDiff line numberDiff line change
@@ -120,45 +120,6 @@ on:
120120
type: boolean
121121

122122
jobs:
123-
check-transport-changes:
124-
if: github.event_name != 'workflow_dispatch'
125-
runs-on: [self-hosted, linux, x64, ephemeral]
126-
outputs:
127-
has-changes: ${{ steps.check.outputs.has-changes }}
128-
impls-yaml-changed: ${{ steps.check.outputs.impls-yaml-changed }}
129-
steps:
130-
- name: Checkout code
131-
uses: actions/checkout@v4
132-
with:
133-
fetch-depth: 0
134-
135-
- name: Check transport changes in last 24 hours
136-
id: check
137-
shell: bash
138-
run: |
139-
# Check if anything in transport/ changed in last 24 hours
140-
CHANGES=$(git log --oneline --since="24 hours ago" -- "transport/**" | wc -l)
141-
142-
if [ "$CHANGES" -eq 0 ]; then
143-
echo "has-changes=false" >> $GITHUB_OUTPUT
144-
echo "impls-yaml-changed=false" >> $GITHUB_OUTPUT
145-
echo "→ No changes in transport/ in last 24 hours, skipping tests"
146-
exit 0
147-
fi
148-
149-
echo "has-changes=true" >> $GITHUB_OUTPUT
150-
151-
# Check specifically if impls.yaml changed in last 24 hours
152-
IMPLS_CHANGES=$(git log --oneline --since="24 hours ago" -- "transport/impls.yaml" | wc -l)
153-
154-
if [ "$IMPLS_CHANGES" -gt 0 ]; then
155-
echo "impls-yaml-changed=true" >> $GITHUB_OUTPUT
156-
echo "→ transport/impls.yaml changed in last 24h, will force matrix rebuild"
157-
else
158-
echo "impls-yaml-changed=false" >> $GITHUB_OUTPUT
159-
echo "→ Other transport files changed, using cached matrix if available"
160-
fi
161-
162123
resolve-transport-parameters:
163124
runs-on: [self-hosted, linux, x64, ephemeral]
164125
outputs:
@@ -219,45 +180,6 @@ jobs:
219180
echo "DEBUG: should-run-tests output = '${{ steps.resolve.outputs.should-run-tests }}'"
220181
echo "DEBUG: github.event_name = '${{ github.event_name }}'"
221182
222-
check-hole-punch-changes:
223-
if: github.event_name != 'workflow_dispatch'
224-
runs-on: [self-hosted, linux, x64, ephemeral]
225-
outputs:
226-
has-changes: ${{ steps.check.outputs.has-changes }}
227-
impls-yaml-changed: ${{ steps.check.outputs.impls-yaml-changed }}
228-
steps:
229-
- name: Checkout code
230-
uses: actions/checkout@v4
231-
with:
232-
fetch-depth: 0
233-
234-
- name: Check hole-punch changes in last 24 hours
235-
id: check
236-
shell: bash
237-
run: |
238-
# Check if anything in hole-punch/ changed in last 24 hours
239-
CHANGES=$(git log --oneline --since="24 hours ago" -- "hole-punch/**" | wc -l)
240-
241-
if [ "$CHANGES" -eq 0 ]; then
242-
echo "has-changes=false" >> $GITHUB_OUTPUT
243-
echo "impls-yaml-changed=false" >> $GITHUB_OUTPUT
244-
echo "→ No changes in hole-punch/ in last 24 hours, skipping tests"
245-
exit 0
246-
fi
247-
248-
echo "has-changes=true" >> $GITHUB_OUTPUT
249-
250-
# Check specifically if impls.yaml changed in last 24 hours
251-
IMPLS_CHANGES=$(git log --oneline --since="24 hours ago" -- "hole-punch/impls.yaml" | wc -l)
252-
253-
if [ "$IMPLS_CHANGES" -gt 0 ]; then
254-
echo "impls-yaml-changed=true" >> $GITHUB_OUTPUT
255-
echo "→ hole-punch/impls.yaml changed in last 24h, will force matrix rebuild"
256-
else
257-
echo "impls-yaml-changed=false" >> $GITHUB_OUTPUT
258-
echo "→ Other hole-punch files changed, using cached matrix if available"
259-
fi
260-
261183
resolve-hole-punch-parameters:
262184
runs-on: [self-hosted, linux, x64, ephemeral]
263185
outputs:
@@ -324,45 +246,6 @@ jobs:
324246
fi
325247
fi
326248
327-
check-perf-changes:
328-
if: github.event_name != 'workflow_dispatch'
329-
runs-on: [self-hosted, linux, x64, ephemeral]
330-
outputs:
331-
has-changes: ${{ steps.check.outputs.has-changes }}
332-
impls-yaml-changed: ${{ steps.check.outputs.impls-yaml-changed }}
333-
steps:
334-
- name: Checkout code
335-
uses: actions/checkout@v4
336-
with:
337-
fetch-depth: 0
338-
339-
- name: Check perf changes in last 24 hours
340-
id: check
341-
shell: bash
342-
run: |
343-
# Check if anything in perf/ changed in last 24 hours
344-
CHANGES=$(git log --oneline --since="24 hours ago" -- "perf/**" | wc -l)
345-
346-
if [ "$CHANGES" -eq 0 ]; then
347-
echo "has-changes=false" >> $GITHUB_OUTPUT
348-
echo "impls-yaml-changed=false" >> $GITHUB_OUTPUT
349-
echo "→ No changes in perf/ in last 24 hours, skipping tests"
350-
exit 0
351-
fi
352-
353-
echo "has-changes=true" >> $GITHUB_OUTPUT
354-
355-
# Check specifically if impls.yaml changed in last 24 hours
356-
IMPLS_CHANGES=$(git log --oneline --since="24 hours ago" -- "perf/impls.yaml" | wc -l)
357-
358-
if [ "$IMPLS_CHANGES" -gt 0 ]; then
359-
echo "impls-yaml-changed=true" >> $GITHUB_OUTPUT
360-
echo "→ perf/impls.yaml changed in last 24h, will force matrix rebuild"
361-
else
362-
echo "impls-yaml-changed=false" >> $GITHUB_OUTPUT
363-
echo "→ Other perf files changed, using cached matrix if available"
364-
fi
365-
366249
resolve-perf-parameters:
367250
runs-on: [self-hosted, linux, x64, ephemeral]
368251
outputs:

‎.github/workflows/hole-punch-interop-pr.yml‎

Lines changed: 0 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -65,28 +65,6 @@ concurrency:
6565
cancel-in-progress: true
6666

6767
jobs:
68-
detect-changes:
69-
if: github.event_name != 'workflow_dispatch'
70-
runs-on: [self-hosted, linux, x64, ephemeral]
71-
outputs:
72-
changed-impls: ${{ steps.detect.outputs.changed-impls }}
73-
changed-relays: ${{ steps.detect.outputs.changed-relays }}
74-
changed-routers: ${{ steps.detect.outputs.changed-routers }}
75-
has-changes: ${{ steps.detect.outputs.has-changes }}
76-
impls-yaml-changed: ${{ steps.detect.outputs.impls-yaml-changed }}
77-
impls-folder-changed: ${{ steps.detect.outputs.impls-folder-changed }}
78-
steps:
79-
- name: Checkout code
80-
uses: actions/checkout@v4
81-
with:
82-
fetch-depth: 0
83-
84-
- name: Detect changed implementations/relays/routers
85-
id: detect
86-
uses: ./.github/actions/detect-changed-impls
87-
with:
88-
test-type: hole-punch
89-
9068
resolve-parameters:
9169
runs-on: [self-hosted, linux, x64, ephemeral]
9270
outputs:

‎.github/workflows/perf-pr.yml‎

Lines changed: 2 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -60,26 +60,6 @@ concurrency:
6060
cancel-in-progress: true
6161

6262
jobs:
63-
detect-changes:
64-
if: github.event_name != 'workflow_dispatch'
65-
runs-on: [self-hosted, linux, x64, ephemeral]
66-
outputs:
67-
changed-impls: ${{ steps.detect.outputs.changed-impls }}
68-
has-changes: ${{ steps.detect.outputs.has-changes }}
69-
impls-yaml-changed: ${{ steps.detect.outputs.impls-yaml-changed }}
70-
impls-folder-changed: ${{ steps.detect.outputs.impls-folder-changed }}
71-
steps:
72-
- name: Checkout code
73-
uses: actions/checkout@v4
74-
with:
75-
fetch-depth: 0
76-
77-
- name: Detect changed implementations
78-
id: detect
79-
uses: ./.github/actions/detect-changed-impls
80-
with:
81-
test-type: perf
82-
8363
resolve-parameters:
8464
runs-on: [self-hosted, linux, x64, ephemeral]
8565
outputs:
@@ -127,8 +107,8 @@ jobs:
127107
# Automatic trigger - use change detection outputs
128108
echo "test-select=${{ steps.detect.outputs.changed-impls }}" >> $GITHUB_OUTPUT
129109
echo "test-ignore=!${{ steps.detect.outputs.changed-impls }}" >> $GITHUB_OUTPUT
130-
echo "baseline-select=" >> $GITHUB_OUTPUT
131-
echo "baseline-ignore=" >> $GITHUB_OUTPUT
110+
echo "baseline-select=${{ steps.detect.outputs.changed-baselines }}" >> $GITHUB_OUTPUT
111+
echo "baseline-ignore=!${{ steps.detect.outputs.changed-baselines }}" >> $GITHUB_OUTPUT
132112
echo "iterations=10" >> $GITHUB_OUTPUT
133113
echo "force-matrix-rebuild=${{ steps.detect.outputs.impls-yaml-changed }}" >> $GITHUB_OUTPUT
134114
# Force image rebuild if impls.yaml changed (commit SHA changes require new images)

‎.github/workflows/transport-interop-pr.yml‎

Lines changed: 0 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -45,26 +45,6 @@ concurrency:
4545
cancel-in-progress: true
4646

4747
jobs:
48-
detect-changes:
49-
if: github.event_name != 'workflow_dispatch'
50-
runs-on: [self-hosted, linux, x64, ephemeral]
51-
outputs:
52-
changed-impls: ${{ steps.detect.outputs.changed-impls }}
53-
has-changes: ${{ steps.detect.outputs.has-changes }}
54-
impls-yaml-changed: ${{ steps.detect.outputs.impls-yaml-changed }}
55-
impls-folder-changed: ${{ steps.detect.outputs.impls-folder-changed }}
56-
steps:
57-
- name: Checkout code
58-
uses: actions/checkout@v4
59-
with:
60-
fetch-depth: 0
61-
62-
- name: Detect changed implementations
63-
id: detect
64-
uses: ./.github/actions/detect-changed-impls
65-
with:
66-
test-type: transport
67-
6848
resolve-parameters:
6949
runs-on: [self-hosted, linux, x64, ephemeral]
7050
outputs:

0 commit comments

Comments
 (0)