Skip to content

Commit c7e5294

Browse files
huydhnhuydhn
authored
Add ExecuTorch benchmark dashboard (#5758)
This bases on the current benchmark dashboard on PyTorch with only a different repo name `pytorch/executorch`. The dashboard is still using `benchmark.oss_ci_benchmark_v2` as I'm setting up the initial version of the dashboard first before looking into consolidating the database schema. ### Testing https://torchci-git-fork-huydhn-executorch-benchmark-fbopensource.vercel.app/benchmark/llms?repoName=pytorch%2Fexecutorch **Please switch to ClickHouse for testing as it has the latest queries, I will submit a PR to clean up Rockset to avoid further discrepancy**
1 parent 8a8fa00 commit c7e5294

File tree

12 files changed

+116
-43
lines changed

12 files changed

+116
-43
lines changed

‎torchci/clickhouse_queries/oss_ci_benchmark_branches/params.json‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,10 @@
11
{
2+
"deviceArch": "String",
3+
"dtypes": "Array(String)",
4+
"excludedMetrics": "Array(String)",
25
"filenames": "Array(String)",
6+
"names": "Array(String)",
37
"repo": "String",
4-
"deviceArch": "String",
58
"startTime": "DateTime64(3)",
69
"stopTime": "DateTime64(3)"
710
}

‎torchci/clickhouse_queries/oss_ci_benchmark_branches/query.sql‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,10 @@ WHERE
1616
has({filenames: Array(String) }, o.filename)
1717
OR empty({filenames: Array(String) })
1818
)
19+
AND (
20+
has({names: Array(String) }, o.name)
21+
OR empty({names: Array(String) })
22+
)
1923
-- NB: DEVICE (ARCH) is the display format used by HUD when grouping together these two fields
2024
AND (
2125
CONCAT(
@@ -26,6 +30,14 @@ WHERE
2630
) = {deviceArch: String }
2731
OR {deviceArch: String } = ''
2832
)
33+
AND (
34+
has({dtypes: Array(String) }, o.dtype)
35+
OR empty({dtypes: Array(String) })
36+
)
37+
AND (
38+
NOT has({excludedMetrics: Array(String) }, o.metric)
39+
OR empty({excludedMetrics: Array(String) })
40+
)
2941
AND notEmpty(o.metric)
3042
AND w.html_url LIKE CONCAT('%', {repo: String }, '%')
3143
AND notEmpty(o.dtype)

‎torchci/clickhouse_queries/oss_ci_benchmark_llms/params.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
"commits": "Array(String)",
44
"deviceArch": "String",
55
"dtypes": "Array(String)",
6+
"excludedMetrics": "Array(String)",
67
"filenames": "Array(String)",
78
"getJobId": "Bool",
89
"granularity": "String",

‎torchci/clickhouse_queries/oss_ci_benchmark_llms/query.sql‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,8 @@ SELECT
77
IF({getJobId: Bool}, o.job_id, '') AS job_id,
88
o.name,
99
o.metric,
10-
o.actual AS actual,
11-
o.target AS target,
10+
floor(toFloat64(o.actual), 2) AS actual,
11+
floor(toFloat64(o.target), 2) AS target,
1212
DATE_TRUNC(
1313
{granularity: String },
1414
fromUnixTimestamp64Milli(o.timestamp)
@@ -53,6 +53,10 @@ WHERE
5353
has({dtypes: Array(String) }, o.dtype)
5454
OR empty({dtypes: Array(String) })
5555
)
56+
AND (
57+
NOT has({excludedMetrics: Array(String) }, o.metric)
58+
OR empty({excludedMetrics: Array(String) })
59+
)
5660
AND notEmpty(o.metric)
5761
AND notEmpty(o.dtype)
5862
AND notEmpty(o.device)

‎torchci/clickhouse_queries/oss_ci_benchmark_names/params.json‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,9 @@
11
{
2+
"deviceArch": "String",
3+
"dtypes": "Array(String)",
4+
"excludedMetrics": "Array(String)",
25
"filenames": "Array(String)",
6+
"names": "Array(String)",
37
"repo": "String",
48
"startTime": "DateTime64(3)",
59
"stopTime": "DateTime64(3)"

‎torchci/clickhouse_queries/oss_ci_benchmark_names/query.sql‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,28 @@ WHERE
1717
has({filenames: Array(String) }, o.filename)
1818
OR empty({filenames: Array(String) })
1919
)
20+
AND (
21+
has({names: Array(String) }, o.name)
22+
OR empty({names: Array(String) })
23+
)
24+
-- NB: DEVICE (ARCH) is the display format used by HUD when grouping together these two fields
25+
AND (
26+
CONCAT(
27+
o.device,
28+
' (',
29+
IF(empty(o.arch), 'NVIDIA A100-SXM4-40GB', o.arch),
30+
')'
31+
) = {deviceArch: String }
32+
OR {deviceArch: String } = ''
33+
)
34+
AND (
35+
has({dtypes: Array(String) }, o.dtype)
36+
OR empty({dtypes: Array(String) })
37+
)
38+
AND (
39+
NOT has({excludedMetrics: Array(String) }, o.metric)
40+
OR empty({excludedMetrics: Array(String) })
41+
)
2042
AND notEmpty(o.metric)
2143
AND w.html_url LIKE CONCAT('%', {repo: String }, '%')
2244
AND notEmpty(o.dtype)

‎torchci/components/NavBar.tsx‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,11 @@ function NavBar() {
5151
},
5252
{
5353
name: "LLMs",
54-
href: "/benchmark/llms",
54+
href: "/benchmark/llms?repoName=pytorch%2Fpytorch",
55+
},
56+
{
57+
name: "ExecuTorch",
58+
href: "/benchmark/llms?repoName=pytorch%2Fexecutorch",
5559
},
5660
];
5761

‎torchci/components/benchmark/llms/ModelGraphPanel.tsx‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -76,13 +76,17 @@ export function GraphPanel({
7676
// Clamp to the nearest granularity (e.g. nearest hour) so that the times will
7777
// align with the data we get from the database
7878
const startTime = useClickHouse
79-
? (queryParams as { [key: string]: any })["startTime"].startOf(granularity)
79+
? dayjs((queryParams as { [key: string]: any })["startTime"]).startOf(
80+
granularity
81+
)
8082
: dayjs(
8183
(queryParams as RocksetParam[]).find((p) => p.name === "startTime")
8284
?.value
8385
).startOf(granularity);
8486
const stopTime = useClickHouse
85-
? (queryParams as { [key: string]: any })["stopTime"].startOf(granularity)
87+
? dayjs((queryParams as { [key: string]: any })["stopTime"]).startOf(
88+
granularity
89+
)
8690
: dayjs(
8791
(queryParams as RocksetParam[]).find((p) => p.name === "stopTime")
8892
?.value

‎torchci/components/benchmark/llms/SummaryPanel.tsx‎

Lines changed: 17 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ export function SummaryPanel({
1919
startTime,
2020
stopTime,
2121
granularity,
22+
repoName,
2223
modelName,
2324
metricNames,
2425
lPerfData,
@@ -27,6 +28,7 @@ export function SummaryPanel({
2728
startTime: dayjs.Dayjs;
2829
stopTime: dayjs.Dayjs;
2930
granularity: Granularity;
31+
repoName: string;
3032
modelName: string;
3133
metricNames: string[];
3234
lPerfData: BranchAndCommitPerfData;
@@ -74,7 +76,9 @@ export function SummaryPanel({
7476
return `Invalid dtype for model ${name}`;
7577
}
7678

77-
const url = `/benchmark/llms?startTime=${startTime}&stopTime=${stopTime}&granularity=${granularity}&lBranch=${lBranch}&lCommit=${lCommit}&rBranch=${rBranch}&rCommit=${rCommit}&modelName=${encodeURIComponent(
79+
const url = `/benchmark/llms?startTime=${startTime}&stopTime=${stopTime}&granularity=${granularity}&lBranch=${lBranch}&lCommit=${lCommit}&rBranch=${rBranch}&rCommit=${rCommit}&repoName=${encodeURIComponent(
80+
repoName
81+
)}&modelName=${encodeURIComponent(
7882
name
7983
)}&dtypeName=${encodeURIComponent(
8084
dtype
@@ -165,18 +169,22 @@ export function SummaryPanel({
165169

166170
// Compute the percentage
167171
const target = v.r.target;
168-
const lPercent = target
169-
? `(${Number((l * 100) / target).toFixed(0)}%)`
170-
: "";
171-
const rPercent = target
172-
? `(${Number((r * 100) / target).toFixed(0)}%)`
173-
: "";
172+
const lPercent =
173+
target && target != 0
174+
? `(${Number((l * 100) / target).toFixed(0)}%)`
175+
: "";
176+
const rPercent =
177+
target && target != 0
178+
? `(${Number((r * 100) / target).toFixed(0)}%)`
179+
: "";
180+
const showTarget =
181+
target && target != 0 ? `[target = ${target}]` : "";
174182
const isNewModel = l === 0 ? "(NEW!)" : "";
175183

176184
if (lCommit === rCommit || l === r) {
177-
return `${r} ${rPercent} [target = ${target}]`;
185+
return `${r} ${rPercent} ${showTarget}`;
178186
} else {
179-
return `${l} ${lPercent} → ${r} ${rPercent} [target = ${target}] ${isNewModel} `;
187+
return `${l} ${lPercent} → ${r} ${rPercent} ${showTarget} ${isNewModel} `;
180188
}
181189
},
182190
};

‎torchci/components/benchmark/llms/common.tsx‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,11 @@
11
import { BranchAndCommit } from "lib/types";
22

3-
export const BENCHMARKS = ["gpt_fast_benchmark"];
3+
export const REPOS = ["pytorch/pytorch", "pytorch/executorch"];
4+
export const REPO_TO_BENCHMARKS: { [k: string]: string[] } = {
5+
"pytorch/pytorch": ["gpt_fast_benchmark"],
6+
"pytorch/executorch": ["android-perf", "apple-perf"],
7+
};
8+
export const EXCLUDED_METRICS: string[] = ["load_status"];
49
export const DEFAULT_MODEL_NAME = "All Models";
510
export const SCALE = 2;
611
export const METRIC_DISPLAY_HEADERS: { [k: string]: string } = {

0 commit comments

Comments
 (0)