Skip to content

Commit 1d728cf

Browse files
rzabarazeshrzabarazesh
committed
vllm - add CI runtime and review cycle metrics
1 parent 38bb33b commit 1d728cf

File tree

8 files changed

+729
-81
lines changed

8 files changed

+729
-81
lines changed

‎torchci/clickhouse_queries/vllm/ci_run_duration/params.json‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
{
2+
"params": {
3+
"repo": "String",
4+
"pipelineName": "String",
5+
"startTime": "DateTime64(3)",
6+
"stopTime": "DateTime64(3)"
7+
},
8+
"tests": [
9+
{
10+
"repo": "vllm-project/vllm",
11+
"pipelineName": "CI",
12+
"startTime": "2025-09-26T00:00:00.000",
13+
"stopTime": "2025-10-03T00:00:00.000"
14+
}
15+
]
16+
}

‎torchci/clickhouse_queries/vllm/ci_run_duration/query.sql‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
-- vLLM CI run durations (Buildkite builds)
2+
-- Lists per-build durations based on build.started_at and build.finished_at
3+
4+
WITH b AS (
5+
SELECT
6+
tupleElement(pipeline, 'repository') AS repository,
7+
tupleElement(pipeline, 'name') AS pipeline_name,
8+
toUInt32(tupleElement(build, 'number')) AS build_number,
9+
tupleElement(build, 'started_at') AS build_started_at,
10+
tupleElement(build, 'finished_at') AS build_finished_at,
11+
tupleElement(build, 'state') AS build_state
12+
FROM vllm.vllm_buildkite_jobs
13+
WHERE
14+
tupleElement(pipeline, 'repository') = {repo: String }
15+
AND tupleElement(pipeline, 'name') = {pipelineName: String }
16+
AND tupleElement(build, 'started_at') IS NOT NULL
17+
AND tupleElement(build, 'finished_at') IS NOT NULL
18+
AND tupleElement(build, 'started_at') >= {startTime: DateTime64(3) }
19+
AND tupleElement(build, 'started_at') < {stopTime: DateTime64(3) }
20+
)
21+
22+
SELECT
23+
pipeline_name,
24+
build_number,
25+
max(build_started_at) AS started_at,
26+
max(build_finished_at) AS finished_at,
27+
any(build_state) AS build_state,
28+
dateDiff('second', started_at, finished_at) AS duration_seconds,
29+
round(duration_seconds / 3600.0, 3) AS duration_hours
30+
FROM b
31+
GROUP BY pipeline_name, build_number
32+
ORDER BY started_at ASC

‎torchci/clickhouse_queries/vllm/pr_cycle_time_breakdown/params.json‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
{
2+
"params": {
3+
"repo": "String",
4+
"startTime": "DateTime64(3)",
5+
"stopTime": "DateTime64(3)"
6+
},
7+
"tests": [
8+
{
9+
"repo": "vllm-project/vllm",
10+
"startTime": "2025-09-22T00:00:00.000",
11+
"stopTime": "2025-09-29T00:00:00.000"
12+
}
13+
]
14+
}

‎torchci/clickhouse_queries/vllm/pr_cycle_time_breakdown/query.sql‎

Lines changed: 182 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,182 @@
1+
-- vLLM PR cycle time breakdown
2+
-- Computes P50 and P90 (hours) for:
3+
-- 1) Time to first (human) review: PR ready -> first human review
4+
-- 2) Time to approval: first human review -> first approval
5+
-- 3) Time in merge queue: first approval -> merge time
6+
-- Notes:
7+
-- - "Ready" is derived from the first time the 'ready' label was applied.
8+
-- - Reviews excluded if state = 'DISMISSED' and if reviewer looks like a bot.
9+
-- - Human review is approximated via author_association in an allowed set and reviewer != PR author.
10+
-- - Metrics only consider merged PRs within the window [startTime, stopTime).
11+
12+
WITH prs AS (
13+
SELECT
14+
number AS pr_number,
15+
user.login AS author,
16+
parseDateTimeBestEffort(created_at) AS created_at_ts,
17+
parseDateTimeBestEffort(closed_at) AS merged_at_ts
18+
FROM default.pull_request
19+
WHERE
20+
dynamoKey LIKE concat({repo: String }, '%')
21+
AND state = 'closed'
22+
AND closed_at != ''
23+
AND parseDateTimeBestEffort(closed_at) >= {startTime: DateTime64(3) }
24+
AND parseDateTimeBestEffort(closed_at) < {stopTime: DateTime64(3) }
25+
),
26+
27+
ready_events AS (
28+
SELECT
29+
ple.pr_number,
30+
minIf(
31+
ple.event_time,
32+
lowerUTF8(ple.label_name) = 'ready' AND ple.action = 'labeled'
33+
) AS first_ready_ts
34+
FROM default.pull_label_event ple
35+
WHERE
36+
ple.repo_name = {repo: String }
37+
GROUP BY ple.pr_number
38+
),
39+
40+
reviews_raw AS (
41+
SELECT
42+
toUInt32(
43+
extractGroups(review.'pull_request_url', 'pulls/([0-9]+)')[1]
44+
) AS pr_number,
45+
review.'user'.'login' AS reviewer,
46+
review.'state' AS state,
47+
review.'author_association' AS author_association,
48+
review.'submitted_at' AS submitted_at_ts
49+
FROM default.pull_request_review
50+
WHERE
51+
dynamoKey LIKE concat({repo: String }, '%')
52+
AND review.'submitted_at' IS NOT NULL
53+
),
54+
55+
-- Filter to human reviews and exclude dismissed ones and bot reviewers
56+
human_reviews AS (
57+
SELECT
58+
r.pr_number,
59+
r.reviewer,
60+
r.state,
61+
r.author_association,
62+
r.submitted_at_ts
63+
FROM reviews_raw r
64+
WHERE
65+
lowerUTF8(r.state) != 'dismissed'
66+
AND r.author_association IN (
67+
'MEMBER', 'OWNER', 'COLLABORATOR', 'CONTRIBUTOR'
68+
)
69+
AND r.reviewer NOT LIKE '%[bot]'
70+
AND lowerUTF8(r.reviewer) NOT LIKE '%bot%'
71+
),
72+
73+
first_human_review AS (
74+
SELECT
75+
pr.pr_number,
76+
-- Define "first review" as first non-approved human review (commented/changes_requested)
77+
minIf(
78+
hr.submitted_at_ts,
79+
hr.reviewer != pr.author
80+
AND lowerUTF8(hr.state) IN ('commented', 'changes_requested')
81+
) AS first_review_ts
82+
FROM prs pr
83+
LEFT JOIN human_reviews hr ON pr.pr_number = hr.pr_number
84+
GROUP BY pr.pr_number
85+
),
86+
87+
first_approval AS (
88+
SELECT
89+
pr.pr_number,
90+
-- Only count approvals from maintainers (exclude contributor approvals)
91+
minIf(
92+
hr.submitted_at_ts,
93+
lowerUTF8(hr.state) = 'approved'
94+
AND hr.reviewer != pr.author
95+
AND hr.author_association IN ('MEMBER', 'OWNER', 'COLLABORATOR')
96+
) AS first_approval_ts
97+
FROM prs pr
98+
LEFT JOIN human_reviews hr ON pr.pr_number = hr.pr_number
99+
GROUP BY pr.pr_number
100+
),
101+
102+
durations AS (
103+
SELECT
104+
pr.pr_number,
105+
coalesce(re.first_ready_ts, pr.created_at_ts) AS ready_ts,
106+
fr.first_review_ts,
107+
fa.first_approval_ts,
108+
pr.merged_at_ts,
109+
-- Durations in hours
110+
if(
111+
fr.first_review_ts IS NULL
112+
OR fr.first_review_ts
113+
< coalesce(re.first_ready_ts, pr.created_at_ts),
114+
NULL,
115+
dateDiff(
116+
'second',
117+
coalesce(re.first_ready_ts, pr.created_at_ts),
118+
fr.first_review_ts
119+
)
120+
/ 3600.0
121+
) AS time_to_first_review_hours,
122+
123+
if(
124+
fa.first_approval_ts IS NULL
125+
OR fr.first_review_ts IS NULL
126+
OR fa.first_approval_ts < fr.first_review_ts,
127+
NULL,
128+
dateDiff('second', fr.first_review_ts, fa.first_approval_ts)
129+
/ 3600.0
130+
) AS time_to_approval_hours,
131+
132+
if(
133+
fa.first_approval_ts IS NULL
134+
OR pr.merged_at_ts < fa.first_approval_ts,
135+
NULL,
136+
dateDiff('second', fa.first_approval_ts, pr.merged_at_ts) / 3600.0
137+
) AS time_in_merge_queue_hours
138+
FROM prs pr
139+
LEFT JOIN ready_events re ON pr.pr_number = re.pr_number
140+
LEFT JOIN first_human_review fr ON pr.pr_number = fr.pr_number
141+
LEFT JOIN first_approval fa ON pr.pr_number = fa.pr_number
142+
),
143+
144+
filtered AS (
145+
SELECT *
146+
FROM durations
147+
WHERE
148+
(
149+
time_to_first_review_hours IS NULL
150+
OR (
151+
time_to_first_review_hours >= 0
152+
AND time_to_first_review_hours < 24 * 30
153+
)
154+
)
155+
AND (
156+
time_to_approval_hours IS NULL
157+
OR (
158+
time_to_approval_hours >= 0 AND time_to_approval_hours < 24 * 30
159+
)
160+
)
161+
AND (
162+
time_in_merge_queue_hours IS NULL
163+
OR (
164+
time_in_merge_queue_hours >= 0
165+
AND time_in_merge_queue_hours < 24 * 30
166+
)
167+
)
168+
)
169+
170+
SELECT
171+
round(quantile(0.5) (time_to_first_review_hours), 2)
172+
AS time_to_first_review_p50,
173+
round(quantile(0.9) (time_to_first_review_hours), 2)
174+
AS time_to_first_review_p90,
175+
round(quantile(0.5) (time_to_approval_hours), 2) AS time_to_approval_p50,
176+
round(quantile(0.9) (time_to_approval_hours), 2) AS time_to_approval_p90,
177+
round(quantile(0.5) (time_in_merge_queue_hours), 2)
178+
AS time_in_merge_queue_p50,
179+
round(quantile(0.9) (time_in_merge_queue_hours), 2)
180+
AS time_in_merge_queue_p90
181+
FROM filtered
182+
-- Quantiles ignore NULLs implicitly; if a column is entirely NULL in window, result will be NULL

‎torchci/components/metrics/panels/ScalarPanel.tsx‎

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,11 +15,14 @@ export function ScalarPanelWithValue({
1515
valueRenderer,
1616
// Callback to decide whether the scalar value is "bad" and should be displayed red.
1717
badThreshold,
18+
// Optional styles to apply to the Paper
19+
paperSx,
1820
}: {
1921
title: string;
2022
value: any;
2123
valueRenderer: (_value: any) => string;
2224
badThreshold: (_value: any) => boolean;
25+
paperSx?: any;
2326
}) {
2427
if (value === undefined) {
2528
return <Skeleton variant={"rectangular"} height={"100%"} />;
@@ -28,14 +31,24 @@ export function ScalarPanelWithValue({
2831
let fontColor = badThreshold(value) ? "#ee6666" : "inherit";
2932

3033
return (
31-
<Paper sx={{ p: 2 }} elevation={3}>
34+
<Paper sx={{ p: 2, ...(paperSx || {}) }} elevation={3}>
3235
<Box
3336
sx={{
3437
display: "flex",
3538
flexDirection: "column",
3639
}}
3740
>
38-
<Typography sx={{ fontSize: "1rem", fontWeight: "bold" }}>
41+
<Typography
42+
sx={{
43+
fontSize: "1rem",
44+
fontWeight: "bold",
45+
whiteSpace: "nowrap",
46+
overflow: "hidden",
47+
textOverflow: "ellipsis",
48+
}}
49+
noWrap
50+
title={title}
51+
>
3952
{title}
4053
</Typography>
4154
<Typography

0 commit comments

Comments
 (0)