11name : Profiling Changes
22
33on :
4- pull_request : {}
4+ pull_request :
5+ paths :
6+ - ' node-graph/**'
7+ - ' Cargo.toml'
8+ - ' Cargo.lock'
59
610env :
711 CARGO_TERM_COLOR : always
@@ -50,14 +54,23 @@ jobs:
5054 id : master-sha
5155 run : echo "sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT
5256
57+ - name : Get CPU info
58+ id : cpu-info
59+ run : |
60+ # Get CPU model and create a short hash for cache key
61+ CPU_MODEL=$(cat /proc/cpuinfo | grep "model name" | head -1 | cut -d: -f2 | xargs)
62+ CPU_HASH=$(echo "$CPU_MODEL" | sha256sum | cut -c1-8)
63+ echo "cpu-hash=$CPU_HASH" >> $GITHUB_OUTPUT
64+ echo "CPU: $CPU_MODEL (hash: $CPU_HASH)"
65+
5366 - name : Cache benchmark baselines
5467 id : cache-benchmark-baselines
5568 uses : actions/cache@v4
5669 with :
5770 path : target/iai
58- key : ${{ runner.os }}-benchmark-baselines-master-${{ steps.master-sha.outputs.sha }}
71+ key : ${{ runner.os }}-${{ runner.arch }}-${{ steps.cpu-info.outputs.cpu-hash }}- benchmark-baselines-master-${{ steps.master-sha.outputs.sha }}
5972 restore-keys : |
60- ${{ runner.os }}-benchmark-baselines-master-
73+ ${{ runner.os }}-${{ runner.arch }}-${{ steps.cpu-info.outputs.cpu-hash }}- benchmark-baselines-master-
6174
6275 - name : Run baseline benchmarks
6376 if : steps.cache-benchmark-baselines.outputs.cache-hit != 'true'
8598 cargo bench --bench run_cached_iai -- --baseline=master --output-format=json | jq -sc | sed 's/\\"//g' > /tmp/run_cached_output.json
8699
87100 - name : Make old comments collapsed by default
101+ # Only run if we have write permissions (not a fork)
102+ if : github.event.pull_request.head.repo.full_name == github.repository
88103 uses : actions/github-script@v7
89104 with :
90105 github-token : ${{secrets.GITHUB_TOKEN}}
@@ -109,7 +124,67 @@ jobs:
109124 });
110125 }
111126
127+ - name : Analyze profiling changes
128+ id : analyze
129+ uses : actions/github-script@v7
130+ with :
131+ script : |
132+ const fs = require('fs');
133+
134+ function isSignificantChange(diffPct, absoluteChange, benchmarkType) {
135+ const meetsPercentageThreshold = Math.abs(diffPct) > 5;
136+ const meetsAbsoluteThreshold = absoluteChange > 200000;
137+ const isCachedExecution = benchmarkType === 'run_cached' ||
138+ benchmarkType.includes('Cached Execution');
139+
140+ return isCachedExecution
141+ ? (meetsPercentageThreshold && meetsAbsoluteThreshold)
142+ : meetsPercentageThreshold;
143+ }
144+
145+ const allOutputs = [
146+ JSON.parse(fs.readFileSync('/tmp/compile_output.json', 'utf8')),
147+ JSON.parse(fs.readFileSync('/tmp/update_output.json', 'utf8')),
148+ JSON.parse(fs.readFileSync('/tmp/run_once_output.json', 'utf8')),
149+ JSON.parse(fs.readFileSync('/tmp/run_cached_output.json', 'utf8'))
150+ ];
151+ const outputNames = ['compile', 'update', 'run_once', 'run_cached'];
152+ const sectionTitles = ['Compilation', 'Update', 'Run Once', 'Cached Execution'];
153+
154+ let hasSignificantChanges = false;
155+ let regressionDetails = [];
156+
157+ for (let i = 0; i < allOutputs.length; i++) {
158+ const benchmarkOutput = allOutputs[i];
159+ const outputName = outputNames[i];
160+ const sectionTitle = sectionTitles[i];
161+
162+ for (const benchmark of benchmarkOutput) {
163+ if (benchmark.profiles?.[0]?.summaries?.parts?.[0]?.metrics_summary?.Callgrind?.Ir?.diffs?.diff_pct) {
164+ const diffPct = parseFloat(benchmark.profiles[0].summaries.parts[0].metrics_summary.Callgrind.Ir.diffs.diff_pct);
165+ const oldValue = benchmark.profiles[0].summaries.parts[0].metrics_summary.Callgrind.Ir.metrics.Both[1].Int;
166+ const newValue = benchmark.profiles[0].summaries.parts[0].metrics_summary.Callgrind.Ir.metrics.Both[0].Int;
167+ const absoluteChange = Math.abs(newValue - oldValue);
168+
169+ if (isSignificantChange(diffPct, absoluteChange, outputName)) {
170+ hasSignificantChanges = true;
171+ regressionDetails.push({
172+ module_path: benchmark.module_path,
173+ id: benchmark.id,
174+ diffPct,
175+ absoluteChange,
176+ sectionTitle
177+ });
178+ }
179+ }
180+ }
181+ }
182+
183+ core.setOutput('has-significant-changes', hasSignificantChanges);
184+ core.setOutput('regression-details', JSON.stringify(regressionDetails));
185+
112186 - name : Comment PR
187+ if : github.event.pull_request.head.repo.full_name == github.repository
113188 uses : actions/github-script@v7
114189 with :
115190 github-token : ${{secrets.GITHUB_TOKEN}}
@@ -121,7 +196,7 @@ jobs:
121196 const runOnceOutput = JSON.parse(fs.readFileSync('/tmp/run_once_output.json', 'utf8'));
122197 const runCachedOutput = JSON.parse(fs.readFileSync('/tmp/run_cached_output.json', 'utf8'));
123198
124- let significantChanges = false ;
199+ const hasSignificantChanges = '${{ steps.analyze.outputs.has-significant-changes }}' === 'true' ;
125200 let commentBody = "";
126201
127202 function formatNumber(num) {
@@ -145,6 +220,17 @@ jobs:
145220 let sectionBody = "";
146221 let hasResults = false;
147222 let hasSignificantChanges = false;
223+
224+ function isSignificantChange(diffPct, absoluteChange, benchmarkType) {
225+ const meetsPercentageThreshold = Math.abs(diffPct) > 5;
226+ const meetsAbsoluteThreshold = absoluteChange > 200000;
227+ const isCachedExecution = benchmarkType === 'run_cached' ||
228+ benchmarkType.includes('Cached Execution');
229+
230+ return isCachedExecution
231+ ? (meetsPercentageThreshold && meetsAbsoluteThreshold)
232+ : meetsPercentageThreshold;
233+ }
148234
149235 for (const benchmark of benchmarkOutput) {
150236 if (benchmark.profiles && benchmark.profiles.length > 0) {
@@ -181,8 +267,8 @@ jobs:
181267 }
182268
183269 sectionBody += "```\n</details>\n\n";
184-
185- if (Math.abs(irDiff.diff_pct) > 5 ) {
270+
271+ if (isSignificantChange(irDiff.diff_pct, Math.abs(irDiff.new - irDiff.old), sectionTitle) ) {
186272 significantChanges = true;
187273 hasSignificantChanges = true;
188274 }
@@ -232,7 +318,7 @@ jobs:
232318 if (commentBody.length > 0) {
233319 const output = `<details open>\n<summary>Performance Benchmark Results</summary>\n\n${commentBody}\n</details>`;
234320
235- if (significantChanges ) {
321+ if (hasSignificantChanges ) {
236322 github.rest.issues.createComment({
237323 issue_number: context.issue.number,
238324 owner: context.repo.owner,
@@ -246,3 +332,13 @@ jobs:
246332 } else {
247333 console.log("No benchmark results to display.");
248334 }
335+
336+ - name : Fail on significant regressions
337+ if : steps.analyze.outputs.has-significant-changes == 'true'
338+ uses : actions/github-script@v7
339+ with :
340+ script : |
341+ const regressionDetails = JSON.parse('${{ steps.analyze.outputs.regression-details }}');
342+ const firstRegression = regressionDetails[0];
343+
344+ core.setFailed(`Significant performance regression detected: ${firstRegression.module_path} ${firstRegression.id} increased by ${firstRegression.absoluteChange.toLocaleString()} instructions (${firstRegression.diffPct.toFixed(2)}%)`);
0 commit comments