Skip to content

Commit a43a148

Browse files
huydhnhuydhn
authored
[CH] Migrate TorchInductor benchmark page (#5769)
This takes much longer than I expect mainly because Rockset has some default values while CH doesn't, so there are many discrepancies in the results to churn through. ### Testing https://torchci-git-fork-huydhn-ch-migrate-remainin-10ef8e-fbopensource.vercel.app/benchmark/compilers v.s https://hud.pytorch.org/benchmark/compilers
1 parent 5510fd0 commit a43a148

File tree

10 files changed

+287
-382
lines changed

10 files changed

+287
-382
lines changed

‎torchci/clickhouse_queries/compilers_benchmark_performance/params.json‎

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,14 @@
11
{
2-
"branches": "String",
3-
"commits": "String",
4-
"compilers": "String",
2+
"branches": "Array(String)",
3+
"commits": "Array(String)",
4+
"compilers": "Array(String)",
55
"device": "String",
66
"dtypes": "String",
77
"getJobId": "Bool",
88
"granularity": "String",
99
"mode": "String",
1010
"startTime": "DateTime64(3)",
1111
"stopTime": "DateTime64(3)",
12-
"suites": "String",
13-
"timezone": "String",
12+
"suites": "Array(String)",
1413
"workflowId": "Int64"
15-
}
14+
}

‎torchci/clickhouse_queries/compilers_benchmark_performance/query.sql‎

Lines changed: 181 additions & 143 deletions
Original file line numberDiff line numberDiff line change
@@ -1,152 +1,190 @@
1-
-- !!! Query is not converted to CH syntax yet. Delete this line when it gets converted
1+
-- This query is used to get the PT2 benchmark results from different experiments
2+
-- to powers the TorchInductor benchmark dashboard
23
WITH performance_results AS (
3-
SELECT
4-
name,
5-
IF(speedup = 'infra_error', NULL, speedup) AS speedup, -- Handle the recent burst of infra error
6-
REPLACE(
7-
filename,
8-
CONCAT(
9-
'_', : dtypes, '_', : mode, '_', : device,
10-
'_performance'
11-
)
12-
) AS filename,
13-
compilation_latency,
14-
compression_ratio,
15-
abs_latency,
16-
mfu,
17-
memory_bandwidth,
18-
dynamo_peak_mem,
19-
eager_peak_mem,
20-
workflow_id,
21-
CAST(job_id AS INT) AS job_id,
22-
FROM
23-
inductor.torch_dynamo_perf_stats_v2
24-
WHERE
25-
filename LIKE CONCAT(
26-
'%_', : dtypes, '_', : mode, '_', : device,
27-
'_performance%'
28-
)
29-
AND TIMESTAMP_MILLIS(timestamp) >= PARSE_DATETIME_ISO8601(:startTime)
30-
AND TIMESTAMP_MILLIS(timestamp) < PARSE_DATETIME_ISO8601(:stopTime)
31-
AND (workflow_id = :workflowId OR :workflowId = 0)
4+
SELECT
5+
name,
6+
IF(speedup = 'infra_error', '', speedup) AS speedup,
7+
-- Handle the recent burst of infra error
8+
REPLACE(
9+
filename,
10+
CONCAT(
11+
'_',
12+
{ dtypes: String },
13+
'_',
14+
{ mode: String },
15+
'_',
16+
{device: String },
17+
'_performance'
18+
),
19+
''
20+
) AS replaced_filename,
21+
compilation_latency,
22+
compression_ratio,
23+
abs_latency,
24+
dynamo_peak_mem,
25+
eager_peak_mem,
26+
workflow_id,
27+
toInt64(job_id) AS job_id,
28+
timestamp
29+
FROM
30+
benchmark.inductor_torch_dynamo_perf_stats
31+
WHERE
32+
filename LIKE CONCAT(
33+
'%_',
34+
{ dtypes: String },
35+
'_',
36+
{mode: String },
37+
'_',
38+
{device: String },
39+
'_performance%'
40+
)
41+
AND timestamp >= toUnixTimestamp64Milli({startTime: DateTime64(3) })
42+
AND timestamp < toUnixTimestamp64Milli({stopTime: DateTime64(3) })
43+
AND (
44+
workflow_id = { workflowId: Int64 }
45+
OR { workflowId: Int64 } = 0
46+
)
3247
),
3348
accuracy_results AS (
34-
SELECT
35-
name,
36-
accuracy,
37-
REPLACE(
38-
filename,
39-
CONCAT(
40-
'_', : dtypes, '_', : mode, '_', : device,
41-
'_accuracy'
42-
)
43-
) AS filename,
44-
workflow_id,
45-
CAST(job_id AS INT) AS job_id,
46-
FROM
47-
inductor.torch_dynamo_perf_stats_v2
48-
WHERE
49-
filename LIKE CONCAT(
50-
'%_', : dtypes, '_', : mode, '_', : device,
51-
'_accuracy%'
52-
)
53-
AND TIMESTAMP_MILLIS(timestamp) >= PARSE_DATETIME_ISO8601(:startTime)
54-
AND TIMESTAMP_MILLIS(timestamp) < PARSE_DATETIME_ISO8601(:stopTime)
55-
AND (workflow_id = :workflowId OR :workflowId = 0)
56-
AND accuracy != 'model_fail_to_load'
57-
AND accuracy != 'eager_fail_to_run'
49+
SELECT
50+
name,
51+
accuracy,
52+
REPLACE(
53+
filename,
54+
CONCAT(
55+
'_',
56+
{ dtypes: String },
57+
'_',
58+
{mode: String },
59+
'_',
60+
{device: String },
61+
'_accuracy'
62+
),
63+
''
64+
) AS replaced_filename,
65+
workflow_id,
66+
toInt64(job_id) AS job_id,
67+
timestamp
68+
FROM
69+
benchmark.inductor_torch_dynamo_perf_stats
70+
WHERE
71+
filename LIKE CONCAT(
72+
'%_',
73+
{ dtypes: String },
74+
'_',
75+
{mode: String },
76+
'_',
77+
{device: String },
78+
'_accuracy%'
79+
)
80+
AND timestamp >= toUnixTimestamp64Milli({startTime: DateTime64(3) })
81+
AND timestamp < toUnixTimestamp64Milli({stopTime: DateTime64(3) })
82+
AND (
83+
workflow_id = { workflowId: Int64 }
84+
OR { workflowId: Int64 } = 0
85+
)
86+
AND accuracy != 'model_fail_to_load'
87+
AND accuracy != 'eager_fail_to_run'
5888
),
5989
results AS (
60-
SELECT
61-
accuracy_results.workflow_id AS workflow_id,
62-
accuracy_results.job_id AS job_id,
63-
CASE
64-
WHEN accuracy_results.filename LIKE '%_torchbench' THEN 'torchbench'
65-
WHEN accuracy_results.filename LIKE '%_timm_models' THEN 'timm_models'
66-
WHEN accuracy_results.filename LIKE '%_huggingface' THEN 'huggingface'
67-
ELSE NULL
68-
END AS suite,
69-
CASE
70-
WHEN accuracy_results.filename LIKE '%_torchbench' THEN REPLACE(
71-
accuracy_results.filename, '_torchbench'
72-
)
73-
WHEN accuracy_results.filename LIKE '%_timm_models' THEN REPLACE(
74-
accuracy_results.filename, '_timm_models'
75-
)
76-
WHEN accuracy_results.filename LIKE '%_huggingface' THEN REPLACE(
77-
accuracy_results.filename, '_huggingface'
78-
)
79-
ELSE NULL
80-
END AS compiler,
81-
accuracy_results.name,
82-
IF(TRY_CAST(speedup AS FLOAT) IS NOT NULL,
83-
CAST(speedup AS FLOAT),
84-
0.0
85-
) AS speedup,
86-
accuracy,
87-
IF(TRY_CAST(compilation_latency AS FLOAT) IS NOT NULL,
88-
CAST(compilation_latency AS FLOAT),
89-
0.0
90-
) AS compilation_latency,
91-
IF(TRY_CAST(compression_ratio AS FLOAT) IS NOT NULL,
92-
CAST(compression_ratio AS FLOAT),
93-
0.0
94-
) AS compression_ratio,
95-
IF(TRY_CAST(abs_latency AS FLOAT) IS NOT NULL,
96-
CAST(abs_latency AS FLOAT),
97-
0.0
98-
) AS abs_latency,
99-
IF(TRY_CAST(mfu AS FLOAT) IS NOT NULL,
100-
CAST(mfu AS FLOAT),
101-
0.0
102-
) AS mfu,
103-
IF(TRY_CAST(memory_bandwidth AS FLOAT) IS NOT NULL,
104-
CAST(memory_bandwidth AS FLOAT),
105-
0.0
106-
) AS memory_bandwidth,
107-
IF(TRY_CAST(dynamo_peak_mem AS FLOAT) IS NOT NULL,
108-
CAST(dynamo_peak_mem AS FLOAT),
109-
0.0
110-
) AS dynamo_peak_mem,
111-
IF(TRY_CAST(eager_peak_mem AS FLOAT) IS NOT NULL,
112-
CAST(eager_peak_mem AS FLOAT),
113-
0.0
114-
) AS eager_peak_mem,
115-
FROM
116-
accuracy_results
117-
LEFT JOIN performance_results ON performance_results.name = accuracy_results.name
118-
AND performance_results.filename = accuracy_results.filename
119-
AND performance_results.workflow_id = accuracy_results.workflow_id
90+
SELECT
91+
accuracy_results.workflow_id AS workflow_id,
92+
accuracy_results.job_id AS job_id,
93+
CASE
94+
WHEN accuracy_results.replaced_filename LIKE '%_torchbench' THEN 'torchbench'
95+
WHEN accuracy_results.replaced_filename LIKE '%_timm_models' THEN 'timm_models'
96+
WHEN accuracy_results.replaced_filename LIKE '%_huggingface' THEN 'huggingface'
97+
ELSE NULL
98+
END AS suite,
99+
CASE
100+
WHEN accuracy_results.replaced_filename LIKE '%_torchbench' THEN REPLACE(
101+
accuracy_results.replaced_filename,
102+
'_torchbench',
103+
''
104+
)
105+
WHEN accuracy_results.replaced_filename LIKE '%_timm_models' THEN REPLACE(
106+
accuracy_results.replaced_filename,
107+
'_timm_models',
108+
''
109+
)
110+
WHEN accuracy_results.replaced_filename LIKE '%_huggingface' THEN REPLACE(
111+
accuracy_results.replaced_filename,
112+
'_huggingface',
113+
''
114+
)
115+
ELSE NULL
116+
END AS compiler,
117+
accuracy_results.name,
118+
IF(speedup != '', toFloat32(speedup), 0.0) AS speedup,
119+
accuracy,
120+
IF(
121+
compilation_latency != '',
122+
toFloat32(compilation_latency),
123+
0.0
124+
) AS compilation_latency,
125+
IF(
126+
compression_ratio != '',
127+
toFloat32(compression_ratio),
128+
0.0
129+
) AS compression_ratio,
130+
IF(abs_latency != '', toFloat32(abs_latency), 0.0) AS abs_latency,
131+
IF(
132+
dynamo_peak_mem != '',
133+
toFloat32(dynamo_peak_mem),
134+
0.0
135+
) AS dynamo_peak_mem,
136+
IF(eager_peak_mem != '', toFloat32(eager_peak_mem), 0.0) AS eager_peak_mem,
137+
IF(
138+
performance_results.timestamp != 0,
139+
performance_results.timestamp,
140+
accuracy_results.timestamp
141+
) AS timestamp
142+
FROM
143+
accuracy_results
144+
LEFT JOIN performance_results ON performance_results.name = accuracy_results.name
145+
AND performance_results.replaced_filename = accuracy_results.replaced_filename
146+
AND performance_results.workflow_id = accuracy_results.workflow_id
120147
)
121-
SELECT DISTINCT
122-
results.workflow_id,
123-
-- As the JSON response is pretty big, only return the field if it's needed
124-
IF(:getJobId, results.job_id, NULL) AS job_id,
125-
results.suite,
126-
results.compiler,
127-
results.name,
128-
results.speedup,
129-
results.accuracy,
130-
results.compilation_latency,
131-
results.compression_ratio,
132-
results.abs_latency,
133-
results.mfu,
134-
results.memory_bandwidth,
135-
results.dynamo_peak_mem,
136-
results.eager_peak_mem,
137-
FORMAT_ISO8601(
138-
DATE_TRUNC(: granularity, w._event_time)
139-
) AS granularity_bucket,
148+
SELECT
149+
DISTINCT results.workflow_id,
150+
IF({getJobId: Bool}, results.job_id, 0) AS job_id,
151+
results.suite,
152+
results.compiler,
153+
results.name,
154+
results.speedup,
155+
results.accuracy,
156+
results.compilation_latency,
157+
results.compression_ratio,
158+
results.abs_latency,
159+
results.dynamo_peak_mem,
160+
results.eager_peak_mem,
161+
DATE_TRUNC(
162+
{granularity: String },
163+
fromUnixTimestamp64Milli(results.timestamp)
164+
) AS granularity_bucket
140165
FROM
141-
results LEFT JOIN commons.workflow_run w ON results.workflow_id = w.id
166+
results
167+
LEFT JOIN default .workflow_run w FINAL ON results.workflow_id = w.id
142168
WHERE
143-
ARRAY_CONTAINS(SPLIT(:suites, ','), LOWER(results.suite))
144-
AND (ARRAY_CONTAINS(SPLIT(:compilers, ','), LOWER(results.compiler)) OR :compilers = '')
145-
AND (ARRAY_CONTAINS(SPLIT(:branches, ','), head_branch) OR :branches = '')
146-
AND (ARRAY_CONTAINS(SPLIT(:commits, ','), head_sha) OR :commits = '')
169+
has({suites: Array(String) }, lower(results.suite))
170+
AND (
171+
has(
172+
{compilers: Array(String) },
173+
lower(results.compiler)
174+
)
175+
OR empty({compilers: Array(String) })
176+
)
177+
AND (
178+
has({branches: Array(String) }, head_branch)
179+
OR empty({branches: Array(String) })
180+
)
181+
AND (
182+
has({commits: Array(String) }, head_sha)
183+
OR empty({commits: Array(String) })
184+
)
147185
ORDER BY
148-
granularity_bucket DESC,
149-
workflow_id DESC,
150-
suite ASC,
151-
compiler ASC,
152-
name ASC
186+
granularity_bucket DESC,
187+
workflow_id DESC,
188+
suite ASC,
189+
compiler ASC,
190+
name ASC

‎torchci/clickhouse_queries/compilers_benchmark_performance_branches/params.json‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,7 @@
11
{
22
"device": "String",
33
"dtypes": "String",
4-
"granularity": "String",
54
"mode": "String",
65
"startTime": "DateTime64(3)",
76
"stopTime": "DateTime64(3)"
8-
}
7+
}

0 commit comments

Comments
 (0)