Skip to content

Commit b55fed9

Browse files
yangw-devyangw-dev
authored
[Add API] get_time_series (#7073)
Add api to get benchmark data as timeseries currently it only support request with name: compiler_precompute, will add general cases once we set up for torchao this will be used as end point of benchmark regression report
1 parent 646a667 commit b55fed9

File tree

8 files changed

+406
-1
lines changed

8 files changed

+406
-1
lines changed

‎.gitignore‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,3 +69,6 @@ aws/tools/cleanup-ssm/Cargo.lock
6969

7070
# These are backup files generated by rustfmt
7171
aws/tools/cleanup-ssm/**/*.rs.bk
72+
73+
# Remove the python version file from pyenv
74+
.python-version

‎torchci/clickhouse_queries/compilers_benchmark_performance_branches/query.sql‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ SELECT DISTINCT
77
toStartOfDay(fromUnixTimestamp(timestamp)) AS event_time
88
FROM
99
benchmark.oss_ci_benchmark_torchinductor
10-
WHERE
10+
PREWHERE
1111
timestamp >= toUnixTimestamp({startTime: DateTime64(3) })
1212
AND timestamp < toUnixTimestamp({stopTime: DateTime64(3) })
1313
-- TODO (huydhn): Clean up the output field and how it's used in the query

‎torchci/components/benchmark/compilers/SummaryGraphPanel.tsx‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -235,6 +235,7 @@ function SuiteGraphPanel({
235235
(id <= lWorkflowId && id >= rWorkflowId)
236236
);
237237
});
238+
238239
const peakMemorySeries = seriesWithInterpolatedTimes(
239240
peakMemory,
240241
startTime,

‎torchci/lib/benchmark/api_helper/compilers/precompute.ts‎

Lines changed: 173 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,173 @@
1+
import {
2+
computeGeomean,
3+
computeMemoryCompressionRatio,
4+
computePassrate,
5+
convertToCompilerPerformanceData,
6+
getPassingModels,
7+
} from "lib/benchmark/compilerUtils";
8+
import { queryClickhouseSaved } from "lib/clickhouse";
9+
import {
10+
BenchmarkTimeSeriesResponse,
11+
CommitRow,
12+
groupByBenchmarkData,
13+
toCommitRowMap,
14+
} from "../utils";
15+
16+
const BENCNMARK_TABLE_NAME = "compilers_benchmark_performance";
17+
const BENCNMARK_COMMIT_NAME = "compilers_benchmark_performance_branches";
18+
19+
// TODO(elainewy): improve the fetch performance
20+
export async function getCompilerBenchmarkData(inputparams: any) {
21+
const start = Date.now();
22+
const rows = await queryClickhouseSaved(BENCNMARK_TABLE_NAME, inputparams);
23+
const end = Date.now();
24+
console.log("time to get data", end - start);
25+
26+
const startc = Date.now();
27+
const commits = await queryClickhouseSaved(
28+
BENCNMARK_COMMIT_NAME,
29+
inputparams
30+
);
31+
const endc = Date.now();
32+
console.log("time to get commit data", endc - startc);
33+
const commitMap = toCommitRowMap(commits);
34+
35+
if (rows.length === 0) {
36+
const response: BenchmarkTimeSeriesResponse = {
37+
time_series: [],
38+
time_range: {
39+
start: "",
40+
end: "",
41+
},
42+
};
43+
return response;
44+
}
45+
46+
// TODO(elainewy): add logics to handle the case to return raw data
47+
const benchmark_time_series_response = toPrecomputeCompiler(
48+
rows,
49+
inputparams,
50+
commitMap,
51+
"time_series"
52+
);
53+
return benchmark_time_series_response;
54+
}
55+
56+
function toPrecomputeCompiler(
57+
rawData: any[],
58+
inputparams: any,
59+
commitMap: Record<string, CommitRow>,
60+
type: string = "time_series"
61+
) {
62+
const data = convertToCompilerPerformanceData(rawData);
63+
const models = getPassingModels(data);
64+
65+
const passrate = computePassrate(data, models);
66+
const geomean = computeGeomean(data, models);
67+
const peakMemory = computeMemoryCompressionRatio(data, models);
68+
69+
const all_data = [passrate, geomean, peakMemory].flat();
70+
71+
const earliest_timestamp = Math.min(
72+
...all_data.map((row) => new Date(row.granularity_bucket).getTime())
73+
);
74+
const latest_timestamp = Math.max(
75+
...all_data.map((row) => new Date(row.granularity_bucket).getTime())
76+
);
77+
78+
//TODO(elainewy): remove this after change the schema of compiler database to populate the fields directly
79+
all_data.map((row) => {
80+
row["dtype"] = inputparams["dtype"];
81+
row["arch"] = inputparams["arch"];
82+
row["device"] = inputparams["device"];
83+
row["mode"] = inputparams["mode"];
84+
// always keep this:
85+
row["commit"] = commitMap[row["workflow_id"]]?.head_sha;
86+
row["branch"] = commitMap[row["workflow_id"]]?.head_branch;
87+
});
88+
89+
let res: any[] = [];
90+
switch (type) {
91+
case "time_series":
92+
/**
93+
* Response of groupByBenchmarkData:
94+
* [
95+
* {
96+
* "group_info": {
97+
* "dtype": "fp32",
98+
* "arch": "sm80",
99+
* "device": "cuda",
100+
* "suite": "ads_10x",
101+
* "compiler": "gcc9.3.0",
102+
* "metric": "latency",
103+
* "mode": "eager"
104+
* },
105+
* "rows": [
106+
* "f123456": {
107+
* "group_info": {
108+
* "workflow_id": "f123456"
109+
* },
110+
* "data": [ # list of data that has the same group_info for group keys and sub group keys
111+
* {
112+
* "workflow_id": "f123456",
113+
* "granularity_bucket": "2022-10-01 00:00:00",
114+
* "value": 100
115+
* ...
116+
* }
117+
* ],
118+
* },
119+
* ]
120+
* }
121+
* ]
122+
*/
123+
const tsd = groupByBenchmarkData(
124+
all_data,
125+
["dtype", "arch", "device", "suite", "compiler", "metric", "mode"],
126+
["workflow_id"]
127+
);
128+
129+
res = tsd.map((group) => {
130+
const group_info = group.group_Info;
131+
const sub_group_data = group.rows;
132+
// extract the first data point for each sub group
133+
// since we only have one datapoint for each unique workflow id with the same group info
134+
const ts_list = Object.values(sub_group_data)
135+
.filter((item) => item.data.length > 0)
136+
.map((item) => item.data[0])
137+
.sort(
138+
(a, b) =>
139+
new Date(a.granularity_bucket).getTime() -
140+
new Date(b.granularity_bucket).getTime()
141+
);
142+
return {
143+
group_info,
144+
num_of_dp: ts_list.length,
145+
data: ts_list,
146+
};
147+
});
148+
break;
149+
case "table":
150+
res = groupByBenchmarkData(
151+
all_data,
152+
[
153+
"dtype",
154+
"arch",
155+
"device",
156+
"mode",
157+
"workflow_id",
158+
"granularity_bucket",
159+
],
160+
["metric", "compiler"]
161+
);
162+
break;
163+
}
164+
165+
const response: BenchmarkTimeSeriesResponse = {
166+
time_series: res,
167+
time_range: {
168+
start: new Date(earliest_timestamp).toISOString(),
169+
end: new Date(latest_timestamp).toISOString(),
170+
},
171+
};
172+
return response;
173+
}

‎torchci/lib/benchmark/api_helper/utils.ts‎

Lines changed: 155 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,155 @@
1+
// Utility to extract params from either GET or POST
2+
3+
import { NextApiRequest } from "next";
4+
5+
/**
6+
* Key-value map describing metadata for a group.
7+
* Example: { dtype: "fp32", arch: "sm80", device: "cuda" }
8+
*/
9+
type GroupInfo = Record<string, string>;
10+
11+
/**
12+
* Represents a subgroup within a larger group.
13+
* Contains its own metadata and a list of data items.
14+
*/
15+
type Subgroup<T> = {
16+
/** Metadata fields for this subgroup (e.g., workflow_id). */
17+
group_info: GroupInfo;
18+
19+
/** The actual list of data items belonging to this subgroup. */
20+
data: T[];
21+
};
22+
23+
/**
24+
* Represents a grouped item at the top level.
25+
* Contains group-level metadata and a collection of subgroups.
26+
*/
27+
type GroupedItem<T> = {
28+
/** Metadata fields for this group (e.g., dtype, arch, compiler). */
29+
group_Info: GroupInfo;
30+
31+
/**
32+
* Rows keyed by a unique identifier string,
33+
* derived from a distinct combination of subgroup `group_Info` fields.
34+
* Each entry corresponds to one subgroup that contains data points.
35+
*/
36+
rows: Record<string, Subgroup<T>>;
37+
};
38+
39+
/**
40+
* Generic parameters map passed into functions or queries.
41+
* Example: { startTime: "2025-08-24", device: "cuda", arch: "h100" }
42+
*/
43+
type Params = Record<string, any>;
44+
45+
// it accepts both ?parameters=<json string> and POST with JSON body
46+
export function readApiGetParams(req: NextApiRequest): Params {
47+
// 1) If POST with parsed JSON body
48+
if (req.method === "POST" && req.body && typeof req.body === "object") {
49+
return req.body as Params;
50+
}
51+
52+
// 2) If POST with raw string body
53+
if (
54+
req.method === "POST" &&
55+
typeof req.body === "string" &&
56+
req.body.trim()
57+
) {
58+
try {
59+
return JSON.parse(req.body) as Params;
60+
} catch {}
61+
}
62+
63+
// 3) If GET with ?parameters=<json string>
64+
const raw = req.query.parameters as string | undefined;
65+
if (raw) {
66+
try {
67+
return JSON.parse(raw) as Params;
68+
} catch {}
69+
}
70+
71+
// 4) Fallback: use query params directly
72+
const q: Params = {};
73+
Object.entries(req.query).forEach(([k, v]) => {
74+
if (k !== "parameters") q[k] = Array.isArray(v) ? v[0] : v;
75+
});
76+
return q;
77+
}
78+
79+
/**
80+
* Group benchmark data by `keys`, and inside each group further subgroup by `subGroupKeys`.
81+
* @param data - benchmark data
82+
* @param keys - keys to group by
83+
* @param subGroupKeys - keys to subgroup by (optional): if not provided, a single subgroup will be created with "_ALL_" data
84+
*/
85+
export function groupByBenchmarkData<T>(
86+
data: T[],
87+
keys: string[],
88+
subGroupKeys: string[] = []
89+
): GroupedItem<T>[] {
90+
const groups = new Map<string, Map<string, Subgroup<T>>>();
91+
const mainInfo = new Map<string, GroupInfo>();
92+
93+
for (const row of data as any[]) {
94+
// build main group key
95+
const mainKeyParts = keys.map((k) => String(getNestedField(row, k)));
96+
const mainKey = mainKeyParts.join("|");
97+
if (!mainInfo.has(mainKey)) {
98+
const info: GroupInfo = {};
99+
keys.forEach((k, i) => (info[k] = mainKeyParts[i]));
100+
mainInfo.set(mainKey, info);
101+
}
102+
103+
// build subgroup key
104+
const subKeyParts =
105+
subGroupKeys.length > 0
106+
? subGroupKeys.map((k) => String(getNestedField(row, k)))
107+
: ["__ALL__"]; // default single subgroup if none provided
108+
const subKey = subKeyParts.join("|");
109+
const subInfo: GroupInfo = {};
110+
111+
subGroupKeys.forEach((k, i) => (subInfo[k] = subKeyParts[i]));
112+
113+
if (!groups.has(mainKey)) groups.set(mainKey, new Map());
114+
const subMap = groups.get(mainKey)!;
115+
116+
if (!subMap.has(subKey)) {
117+
subMap.set(subKey, { group_info: subInfo, data: [] });
118+
}
119+
subMap.get(subKey)!.data.push(row as T);
120+
}
121+
122+
// build result array
123+
const result: GroupedItem<T>[] = [];
124+
for (const [mainKey, subMap] of groups.entries()) {
125+
const rowsObj = Object.fromEntries(subMap.entries());
126+
result.push({
127+
group_Info: mainInfo.get(mainKey)!,
128+
rows: rowsObj,
129+
});
130+
}
131+
return result;
132+
}
133+
134+
export function getNestedField(obj: any, path: string): any {
135+
return path.split(".").reduce((o, key) => (o && key in o ? o[key] : ""), obj);
136+
}
137+
138+
export type BenchmarkTimeSeriesResponse = {
139+
time_series: any[];
140+
time_range: { start: string; end: string };
141+
};
142+
143+
export type CommitRow = {
144+
head_branch: string;
145+
head_sha: string;
146+
id: string;
147+
};
148+
149+
export function toCommitRowMap(rows: CommitRow[]): Record<string, CommitRow> {
150+
const result: Record<string, CommitRow> = {};
151+
for (const row of rows) {
152+
result[row.id] = row;
153+
}
154+
return result;
155+
}

‎torchci/lib/benchmark/compilerUtils.ts‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -99,6 +99,8 @@ export function computePassrate(
9999

100100
const [bucket, workflowId, suite, compiler] = key.split("+");
101101
passrate.push({
102+
metric: "passrate",
103+
value: p,
102104
granularity_bucket: bucket,
103105
workflow_id: workflowId,
104106
suite: suite,
@@ -163,6 +165,8 @@ export function computeGeomean(
163165

164166
const [bucket, workflowId, suite, compiler] = key.split("+");
165167
returnedGeomean.push({
168+
metric: "geomean",
169+
value: Number(gm),
166170
granularity_bucket: bucket,
167171
workflow_id: workflowId,
168172
suite: suite,
@@ -274,6 +278,7 @@ export function computeCompilationTime(
274278

275279
const [bucket, workflowId, suite, compiler] = key.split("+");
276280
returnedCompTime.push({
281+
metric: "compilation_latency",
277282
granularity_bucket: bucket,
278283
workflow_id: workflowId,
279284
suite: suite,
@@ -328,6 +333,8 @@ export function computeMemoryCompressionRatio(
328333

329334
const [bucket, workflowId, suite, compiler] = key.split("+");
330335
returnedMemory.push({
336+
metric: "compression_ratio",
337+
value: Number(m.toFixed(SCALE)),
331338
granularity_bucket: bucket,
332339
workflow_id: workflowId,
333340
suite: suite,
@@ -379,6 +386,8 @@ export function computePeakMemoryUsage(
379386

380387
const [bucket, workflowId, suite, compiler] = key.split("+");
381388
returnedMemory.push({
389+
metric: "dynamo_peak_mem",
390+
value: Number(m.toFixed(SCALE)),
382391
granularity_bucket: bucket,
383392
workflow_id: workflowId,
384393
suite: suite,

0 commit comments

Comments
 (0)