Skip to content

Commit 7151fb5

Browse files
huydhnhuydhn
authored
Fix summary table for same commit on different devices (#6343)
This is a bug reported by @malfet where the dashboard only shows the value from the new commit (right) when the left commit and the right commit are the same but on different devices. This defeats the purpose of comparing different devices. The bug here is that the `l → r` display worked under the assumption that same commit mean same benchmark, which is not true anymore. Same benchmark now means same commit plus same device. ### Preview https://torchci-git-fork-huydhn-fix-cross-device-sa-e8d1e5-fbopensource.vercel.app/benchmark/compilers?dashboard=torchinductor&startTime=Wed%2C%2019%20Feb%202025%2021%3A06%3A10%20GMT&stopTime=Wed%2C%2026%20Feb%202025%2021%3A06%3A10%20GMT&granularity=hour&mode=inference&dtype=bfloat16&lDeviceName=rocm%20(mi300x)&rDeviceName=cuda%20(a100)&lBranch=main&lCommit=d0adff761ea4d299528c89bee23008dbc622846e&rBranch=main&rCommit=d0adff761ea4d299528c89bee23008dbc622846e
1 parent 413da2a commit 7151fb5

File tree

2 files changed

+65
-22
lines changed

2 files changed

+65
-22
lines changed

‎torchci/components/benchmark/compilers/ModelPanel.tsx‎

Lines changed: 39 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -75,7 +75,10 @@ export function ModelPanel({
7575
});
7676

7777
// Combine with right data
78-
if (lCommit !== rCommit && rData !== undefined) {
78+
if (
79+
(lDeviceName !== rDeviceName || lCommit !== rCommit) &&
80+
rData !== undefined
81+
) {
7982
rData.forEach((record: CompilerPerformanceData) => {
8083
if (record.name in dataGroupedByModel) {
8184
dataGroupedByModel[record.name]["r"] = record;
@@ -241,7 +244,7 @@ export function ModelPanel({
241244
return "";
242245
}
243246

244-
if (lCommit === rCommit) {
247+
if (lDeviceName === rDeviceName && lCommit === rCommit) {
245248
return PASSING_ACCURACY.includes(v.l) ? "" : styles.warning;
246249
} else {
247250
if (
@@ -277,7 +280,10 @@ export function ModelPanel({
277280
{v.l} (<strong>NEW!</strong>)
278281
</>
279282
);
280-
} else if (lCommit === rCommit || v.l === v.r) {
283+
} else if (
284+
(lDeviceName === rDeviceName && lCommit === rCommit) ||
285+
v.l === v.r
286+
) {
281287
return v.l;
282288
} else {
283289
return `${v.r} → ${v.l}`;
@@ -297,7 +303,7 @@ export function ModelPanel({
297303
const l = Number(v.l);
298304
const r = Number(v.r);
299305

300-
if (lCommit === rCommit) {
306+
if (lDeviceName === rDeviceName && lCommit === rCommit) {
301307
return l >= SPEEDUP_THRESHOLD ? "" : styles.warning;
302308
} else {
303309
// l is the new value, r is the old value
@@ -334,7 +340,11 @@ export function ModelPanel({
334340
const l = Number(v.l).toFixed(SCALE);
335341
const r = Number(v.r).toFixed(SCALE);
336342

337-
if (lCommit === rCommit || l === r || v.r === 0) {
343+
if (
344+
(lDeviceName === rDeviceName && lCommit === rCommit) ||
345+
l === r ||
346+
v.r === 0
347+
) {
338348
return l;
339349
} else {
340350
return `${r} → ${l}`;
@@ -354,7 +364,7 @@ export function ModelPanel({
354364
const l = Number(v.l);
355365
const r = Number(v.r);
356366

357-
if (lCommit === rCommit) {
367+
if (lDeviceName === rDeviceName && lCommit === rCommit) {
358368
return "";
359369
} else {
360370
if (l === 0 || l === r) {
@@ -384,7 +394,11 @@ export function ModelPanel({
384394
const l = Number(v.l).toFixed(0);
385395
const r = Number(v.r).toFixed(0);
386396

387-
if (lCommit === rCommit || l === r || v.r === 0) {
397+
if (
398+
(lDeviceName === rDeviceName && lCommit === rCommit) ||
399+
l === r ||
400+
v.r === 0
401+
) {
388402
return l;
389403
} else {
390404
return `${r} → ${l}`;
@@ -404,7 +418,7 @@ export function ModelPanel({
404418
const l = Number(v.l);
405419
const r = Number(v.r);
406420

407-
if (lCommit === rCommit) {
421+
if (lDeviceName === rDeviceName && lCommit === rCommit) {
408422
return l >= COMPRESSION_RATIO_THRESHOLD ? "" : styles.warning;
409423
} else {
410424
if (l === 0 || l === r) {
@@ -438,7 +452,11 @@ export function ModelPanel({
438452
const l = Number(v.l).toFixed(SCALE);
439453
const r = Number(v.r).toFixed(SCALE);
440454

441-
if (lCommit === rCommit || l === r || v.r === 0) {
455+
if (
456+
(lDeviceName === rDeviceName && lCommit === rCommit) ||
457+
l === r ||
458+
v.r === 0
459+
) {
442460
return l;
443461
} else {
444462
return `${r} → ${l}`;
@@ -458,7 +476,7 @@ export function ModelPanel({
458476
const l = Number(v.l);
459477
const r = Number(v.r);
460478

461-
if (lCommit === rCommit) {
479+
if (lDeviceName === rDeviceName && lCommit === rCommit) {
462480
return "";
463481
} else {
464482
if (l === 0 || l === r) {
@@ -488,7 +506,11 @@ export function ModelPanel({
488506
const l = Number(v.l).toFixed(SCALE);
489507
const r = Number(v.r).toFixed(SCALE);
490508

491-
if (lCommit === rCommit || l === r || v.r === 0) {
509+
if (
510+
(lDeviceName === rDeviceName && lCommit === rCommit) ||
511+
l === r ||
512+
v.r === 0
513+
) {
492514
return l;
493515
} else {
494516
return `${r} → ${l}`;
@@ -508,7 +530,7 @@ export function ModelPanel({
508530
const l = Number(v.l);
509531
const r = Number(v.r);
510532

511-
if (lCommit === rCommit) {
533+
if (lDeviceName === rDeviceName && lCommit === rCommit) {
512534
return "";
513535
} else {
514536
if (l === 0 || l === r) {
@@ -538,7 +560,11 @@ export function ModelPanel({
538560
const l = Number(v.l).toFixed(2);
539561
const r = Number(v.r).toFixed(2);
540562

541-
if (lCommit === rCommit || l === r || v.r === 0) {
563+
if (
564+
(lDeviceName === rDeviceName && lCommit === rCommit) ||
565+
l === r ||
566+
v.r === 0
567+
) {
542568
return l;
543569
} else {
544570
return `${r} → ${l}`;

‎torchci/components/benchmark/compilers/SummaryPanel.tsx‎

Lines changed: 26 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -65,8 +65,10 @@ function processSummaryData(
6565

6666
function combineLeftAndRight(
6767
lCommit: string,
68+
lDeviceName: string,
6869
lData: { [k: string]: any },
6970
rCommit: string,
71+
rDeviceName: string,
7072
rData: { [k: string]: any },
7173
suites: string[]
7274
) {
@@ -84,7 +86,7 @@ function combineLeftAndRight(
8486
});
8587

8688
// Combine with right data
87-
if (lCommit !== rCommit) {
89+
if (lCommit !== rCommit || lDeviceName !== rDeviceName) {
8890
Object.keys(rData).forEach((compiler: string) => {
8991
if (!(compiler in data)) {
9092
data[compiler] = {
@@ -179,29 +181,37 @@ export function SummaryPanel({
179181
// Combine both sides
180182
const passrate = combineLeftAndRight(
181183
lCommit,
184+
lDeviceName,
182185
lPassrate,
183186
rCommit,
187+
rDeviceName,
184188
rPassrate,
185189
suites
186190
);
187191
const geomean = combineLeftAndRight(
188192
lCommit,
193+
lDeviceName,
189194
lGeomean,
190195
rCommit,
196+
rDeviceName,
191197
rGeomean,
192198
suites
193199
);
194200
const compTime = combineLeftAndRight(
195201
lCommit,
202+
lDeviceName,
196203
lCompTime,
197204
rCommit,
205+
rDeviceName,
198206
rCompTime,
199207
suites
200208
);
201209
const memory = combineLeftAndRight(
202210
lCommit,
211+
lDeviceName,
203212
lMemory,
204213
rCommit,
214+
rDeviceName,
205215
rMemory,
206216
suites
207217
);
@@ -259,7 +269,11 @@ export function SummaryPanel({
259269
return "";
260270
}
261271

262-
if (lCommit === rCommit || l === r || r == undefined) {
272+
if (
273+
(lDeviceName === rDeviceName && lCommit === rCommit) ||
274+
l === r ||
275+
r == undefined
276+
) {
263277
return <a href={url}>{v.l}</a>;
264278
} else {
265279
return (
@@ -282,7 +296,10 @@ export function SummaryPanel({
282296
return "";
283297
}
284298

285-
if (lCommit === rCommit || r === undefined) {
299+
if (
300+
(lDeviceName === rDeviceName && lCommit === rCommit) ||
301+
r === undefined
302+
) {
286303
return l >= ACCURACY_THRESHOLD ? "" : styles.warning;
287304
} else {
288305
if (l === r) {
@@ -348,7 +365,7 @@ export function SummaryPanel({
348365
const r = Number(v.r).toFixed(SCALE);
349366

350367
if (
351-
lCommit === rCommit ||
368+
(lDeviceName === rDeviceName && lCommit === rCommit) ||
352369
l === r ||
353370
v.r === undefined ||
354371
v.r === ""
@@ -377,7 +394,7 @@ export function SummaryPanel({
377394
const l = Number(v.l);
378395
const r = Number(v.r);
379396

380-
if (lCommit === rCommit) {
397+
if (lDeviceName === rDeviceName && lCommit === rCommit) {
381398
return l >= SPEEDUP_THRESHOLD ? "" : styles.warning;
382399
} else {
383400
if (l === r) {
@@ -443,7 +460,7 @@ export function SummaryPanel({
443460
const r = Number(v.r).toFixed(0);
444461

445462
if (
446-
lCommit === rCommit ||
463+
(lDeviceName === rDeviceName && lCommit === rCommit) ||
447464
l === r ||
448465
v.r === undefined ||
449466
v.r === ""
@@ -472,7 +489,7 @@ export function SummaryPanel({
472489
const l = Number(v.l);
473490
const r = Number(v.r);
474491

475-
if (lCommit === rCommit) {
492+
if (lDeviceName === rDeviceName && lCommit === rCommit) {
476493
return "";
477494
} else {
478495
if (l === r) {
@@ -534,7 +551,7 @@ export function SummaryPanel({
534551
const r = Number(v.r).toFixed(SCALE);
535552

536553
if (
537-
lCommit === rCommit ||
554+
(lDeviceName === rDeviceName && lCommit === rCommit) ||
538555
l === r ||
539556
v.r === undefined ||
540557
v.r === ""
@@ -563,7 +580,7 @@ export function SummaryPanel({
563580
const l = Number(v.l);
564581
const r = Number(v.r);
565582

566-
if (lCommit === rCommit) {
583+
if (lDeviceName === rDeviceName && lCommit === rCommit) {
567584
return l >= COMPRESSION_RATIO_THRESHOLD
568585
? ""
569586
: styles.warning;

0 commit comments

Comments
 (0)