Skip to content

Show input and output length on vLLM dashboard #6992

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Aug 12, 2025
+60 −51
Merged

Show input and output length on vLLM dashboard #6992

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
c24325b
Show input and output length on vLLM dashboard
huydhn Aug 9, 2025
e4ea1b9
Get input and output lengths for latency benchmark
huydhn Aug 12, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 20 additions & 0 deletions torchci/clickhouse_queries/oss_ci_benchmark_llms/query.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,26 @@ WITH benchmarks AS (
tupleElement(o.benchmark, 'extra_info')['args'],
'tensor_parallel_size'
),
'random_input_len',
JSONExtractString(
tupleElement(benchmark, 'extra_info')['args'],
'random_input_len'
),
'random_output_len',
JSONExtractString(
tupleElement(benchmark, 'extra_info')['args'],
'random_output_len'
),
'input_len',
JSONExtractString(
tupleElement(benchmark, 'extra_info')['args'],
'input_len'
),
'output_len',
JSONExtractString(
tupleElement(benchmark, 'extra_info')['args'],
'output_len'
),
-- Used by Cachebench
'is_dynamic',
IF(
Expand Down
39 changes: 16 additions & 23 deletions torchci/components/benchmark/llms/components/LLMsGraphPanel.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -170,31 +170,24 @@ export default function LLMsGraphPanel({
const metric = record.metric;

if (repoName === "vllm-project/vllm") {
let requestRate = record.extra!["request_rate"];
// TODO (huydhn): Fix the invalid JSON on vLLM side
if (
metric.includes("itl") ||
metric.includes("tpot") ||
metric.includes("ttft")
) {
requestRate = requestRate !== "" ? requestRate : "Inf";
}
const requestRate = record.extra!["request_rate"];
const tensorParallel = record.extra!["tensor_parallel_size"];
const inputLen = record.extra!["random_input_len"]
? record.extra!["random_input_len"]
: record.extra!["input_len"];
const outputLen = record.extra!["random_output_len"]
? record.extra!["random_output_len"]
: record.extra!["output_len"];

let tensorParallel = record.extra!["tensor_parallel_size"];
// TODO (huydhn): Fix the passing of tensor_parallel_size to the benchmark
// script on vLLM side
if (model.includes("8B")) {
tensorParallel = tensorParallel !== "" ? tensorParallel : "1";
} else if (model.includes("70B")) {
tensorParallel = tensorParallel !== "" ? tensorParallel : "4";
} else if (model.includes("8x7B")) {
tensorParallel = tensorParallel !== "" ? tensorParallel : "2";
record.display = `${model} / tp${tensorParallel}`;
if (requestRate) {
record.display = `${record.display} / qps_${requestRate}`;
}

if (requestRate !== "") {
record.display = `${model} / tp${tensorParallel} / qps_${requestRate}`;
} else {
record.display = `${model} / tp${tensorParallel}`;
if (inputLen) {
record.display = `${record.display} / in_${inputLen}`;
}
if (outputLen) {
record.display = `${record.display} / out_${outputLen}`;
}
} else if (
repoName === "pytorch/pytorch" &&
Expand Down
18 changes: 18 additions & 0 deletions torchci/components/benchmark/llms/components/LLMsSummaryPanel.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -186,6 +186,24 @@ export default function LLMsSummaryPanel({
return `${params.value}`;
},
});

columns.push({
field: "input_len",
headerName: "Input len.",
flex: 1,
renderCell: (params: GridRenderCellParams<any>) => {
return `${params.value}`;
},
});

columns.push({
field: "output_len",
headerName: "Output len.",
flex: 1,
renderCell: (params: GridRenderCellParams<any>) => {
return `${params.value}`;
},
});
}

if (
Expand Down
34 changes: 6 additions & 28 deletions torchci/lib/benchmark/llms/utils/llmUtils.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -293,37 +293,15 @@ const toRowData = (
if (repoName === "vllm-project/vllm") {
// These fields are only available on vLLM benchmark
const extraInfo = JSON.parse(extra);
// TODO (huydhn): Fix the invalid JSON on vLLM side
if (
metric.includes("itl") ||
metric.includes("tpot") ||
metric.includes("ttft")
) {
extraInfo["request_rate"] =
extraInfo["request_rate"] !== "" ? extraInfo["request_rate"] : "Inf";
}
// TODO (huydhn): Fix the passing of tensor_parallel_size to the benchmark
// script on vLLM side
if (model.includes("8B")) {
extraInfo["tensor_parallel_size"] =
extraInfo["tensor_parallel_size"] !== ""
? extraInfo["tensor_parallel_size"]
: 1;
} else if (model.includes("70B")) {
extraInfo["tensor_parallel_size"] =
extraInfo["tensor_parallel_size"] !== ""
? extraInfo["tensor_parallel_size"]
: 4;
} else if (model.includes("8x7B")) {
extraInfo["tensor_parallel_size"] =
extraInfo["tensor_parallel_size"] !== ""
? extraInfo["tensor_parallel_size"]
: 2;
}

row["extra"] = extraInfo;
row["tensor_parallel_size"] = extraInfo["tensor_parallel_size"];
row["request_rate"] = extraInfo["request_rate"];
row["input_len"] = extraInfo["random_input_len"]
? extraInfo["random_input_len"]
: extraInfo["input_len"];
row["output_len"] = extraInfo["random_output_len"]
? extraInfo["random_input_len"]
: extraInfo["output_len"];
}

if (
Expand Down