diff --git a/src/guidellm/presentation/data_models.py b/src/guidellm/presentation/data_models.py index fefcf7d8..ceceea86 100644 --- a/src/guidellm/presentation/data_models.py +++ b/src/guidellm/presentation/data_models.py @@ -5,6 +5,8 @@ from pydantic import BaseModel, computed_field +from guidellm.scheduler.strategy import SchedulingStrategy + if TYPE_CHECKING: from guidellm.benchmark.benchmark import GenerativeBenchmark @@ -212,11 +214,29 @@ class BenchmarkDatum(BaseModel): ttft: TabularDistributionSummary throughput: TabularDistributionSummary time_per_request: TabularDistributionSummary + strategy_display_str: str + + @classmethod + def get_strategy_display_str(cls, strategy: SchedulingStrategy): + strategy_type = strategy if isinstance(strategy, str) else strategy.type_ + strategy_instance = ( + strategy if isinstance(strategy, SchedulingStrategy) else None + ) + + if strategy_type == "concurrent": + rate = f"@{strategy.streams}" if strategy_instance else "@##" # type: ignore[attr-defined] + elif strategy_type in ("constant", "poisson"): + rate = f"@{strategy.rate:.2f}" if strategy_instance else "@#.##" # type: ignore[attr-defined] + else: + rate = "" + return f"{strategy_type}{rate}" @classmethod def from_benchmark(cls, bm: "GenerativeBenchmark"): + rps = bm.metrics.requests_per_second.successful.mean return cls( - requests_per_second=bm.metrics.requests_per_second.successful.mean, + strategy_display_str=cls.get_strategy_display_str(bm.args.strategy), + requests_per_second=rps, itl=TabularDistributionSummary.from_distribution_summary( bm.metrics.inter_token_latency_ms.successful ), diff --git a/src/ui/lib/components/Charts/MetricLine/MetricLine.component.tsx b/src/ui/lib/components/Charts/MetricLine/MetricLine.component.tsx index 8b1b4df2..eb123593 100644 --- a/src/ui/lib/components/Charts/MetricLine/MetricLine.component.tsx +++ b/src/ui/lib/components/Charts/MetricLine/MetricLine.component.tsx @@ -1,5 +1,6 @@ -import { useTheme } from '@mui/material'; -import { ResponsiveLine } from '@nivo/line'; +import { Typography, useTheme } from '@mui/material'; +import { PointTooltipProps, ResponsiveLine } from '@nivo/line'; +import { BasicTooltip } from '@nivo/tooltip'; import React, { FC } from 'react'; import { useColor } from '@/lib/hooks/useColor'; @@ -49,11 +50,30 @@ export const Component: FC = ({ reverse: false, }; } + type PointTooltipPropsWithLabel = PointTooltipProps & { + point: { + data: { + label: string; + }; + }; + }; return ( ( + + {(point as PointTooltipPropsWithLabel).point.data.label} + + } + color={point.point.color} + enableChip={true} + /> + )} + pointSize={10} colors={[selectedColor]} margin={{ top: 20, right: 10, bottom: 20, left: 35.5 }} xScale={{ type: 'linear', min: minX }} @@ -92,7 +112,6 @@ export const Component: FC = ({ }} enableGridX={false} enableGridY={false} - pointSize={0} useMesh={true} layers={[ CustomAxes, @@ -115,6 +134,9 @@ export const Component: FC = ({ ), 'axes', 'lines', + 'points', + 'markers', + 'mesh', ]} theme={lineTheme} /> diff --git a/src/ui/lib/components/MetricsSummary/MetricsSummary.component.tsx b/src/ui/lib/components/MetricsSummary/MetricsSummary.component.tsx index 8f5e3aed..0a453d1f 100644 --- a/src/ui/lib/components/MetricsSummary/MetricsSummary.component.tsx +++ b/src/ui/lib/components/MetricsSummary/MetricsSummary.component.tsx @@ -102,7 +102,7 @@ export const Component = () => { return ( <> - + diff --git a/src/ui/lib/store/benchmarksWindowData.ts b/src/ui/lib/store/benchmarksWindowData.ts index 87faf7bc..5bbbcfaa 100644 --- a/src/ui/lib/store/benchmarksWindowData.ts +++ b/src/ui/lib/store/benchmarksWindowData.ts @@ -1,5 +1,6 @@ export const benchmarksScript = `window.benchmarks = [ { + strategyDisplayStr: "synchronous", requestsPerSecond: 11.411616848282272, itl: { mean: 8.758024845683707, @@ -171,6 +172,7 @@ export const benchmarksScript = `window.benchmarks = [ }, }, { + strategyDisplayStr: "constant@36.28", requestsPerSecond: 36.289181300710815, itl: { mean: 588.0161376137819, @@ -342,6 +344,7 @@ export const benchmarksScript = `window.benchmarks = [ }, }, { + strategyDisplayStr: "constant@20.75", requestsPerSecond: 20.752070927855794, itl: { mean: 116.28360712595156, @@ -513,6 +516,7 @@ export const benchmarksScript = `window.benchmarks = [ }, }, { + strategyDisplayStr: "constant@26.81", requestsPerSecond: 26.81917480361788, itl: { mean: 299.7306064613554, @@ -684,6 +688,7 @@ export const benchmarksScript = `window.benchmarks = [ }, }, { + strategyDisplayStr: "constant@26.82", requestsPerSecond: 26.823988819498975, itl: { mean: 683.8011571339198, @@ -855,6 +860,7 @@ export const benchmarksScript = `window.benchmarks = [ }, }, { + strategyDisplayStr: "constant@24.50", requestsPerSecond: 24.50047903792646, itl: { mean: 742.9258901891964, @@ -1026,6 +1032,7 @@ export const benchmarksScript = `window.benchmarks = [ }, }, { + strategyDisplayStr: "constant@25.61", requestsPerSecond: 25.617829792196602, itl: { mean: 663.3098317044122, @@ -1197,6 +1204,7 @@ export const benchmarksScript = `window.benchmarks = [ }, }, { + strategyDisplayStr: "constant@37.02", requestsPerSecond: 37.02892550982192, itl: { mean: 606.4144710877113, @@ -1368,6 +1376,7 @@ export const benchmarksScript = `window.benchmarks = [ }, }, { + strategyDisplayStr: "constant@37.29", requestsPerSecond: 37.29183354201869, itl: { mean: 603.3237551205925, @@ -1539,6 +1548,7 @@ export const benchmarksScript = `window.benchmarks = [ }, }, { + strategyDisplayStr: "throughput", requestsPerSecond: 37.45318312972309, itl: { mean: 600.7204526769262, diff --git a/src/ui/lib/store/slices/benchmarks/benchmarks.interfaces.ts b/src/ui/lib/store/slices/benchmarks/benchmarks.interfaces.ts index 2a5f319e..a17bfb5f 100644 --- a/src/ui/lib/store/slices/benchmarks/benchmarks.interfaces.ts +++ b/src/ui/lib/store/slices/benchmarks/benchmarks.interfaces.ts @@ -27,6 +27,7 @@ export interface BenchmarkMetrics { export interface Benchmark extends BenchmarkMetrics { requestsPerSecond: number; + strategyDisplayStr: string; } export type Benchmarks = Benchmark[]; diff --git a/src/ui/lib/store/slices/benchmarks/benchmarks.selectors.ts b/src/ui/lib/store/slices/benchmarks/benchmarks.selectors.ts index 9aa5fd81..778dc799 100644 --- a/src/ui/lib/store/slices/benchmarks/benchmarks.selectors.ts +++ b/src/ui/lib/store/slices/benchmarks/benchmarks.selectors.ts @@ -18,8 +18,10 @@ export const selectMetricsSummaryLineData = createSelector( ?.slice() ?.sort((bm1, bm2) => (bm1.requestsPerSecond > bm2.requestsPerSecond ? 1 : -1)); const selectedPercentile = sloState.enforcedPercentile; - - const lineData: { [K in keyof BenchmarkMetrics]: Point[] } = { + interface PointWithLabel extends Point { + label: string; + } + const lineData: { [K in keyof BenchmarkMetrics]: PointWithLabel[] } = { ttft: [], itl: [], timePerRequest: [], @@ -32,7 +34,7 @@ export const selectMetricsSummaryLineData = createSelector( 'throughput', ]; metrics.forEach((metric) => { - const data: Point[] = []; + const data: PointWithLabel[] = []; sortedByRPS?.forEach((benchmark) => { const percentile = benchmark[metric].percentileRows.find( (p) => p.percentile === selectedPercentile @@ -40,6 +42,7 @@ export const selectMetricsSummaryLineData = createSelector( data.push({ x: benchmark.requestsPerSecond, y: percentile?.value ?? 0, + label: benchmark.strategyDisplayStr, }); }); diff --git a/tests/unit/presentation/test_data_models.py b/tests/unit/presentation/test_data_models.py index c1663c43..e879406d 100644 --- a/tests/unit/presentation/test_data_models.py +++ b/tests/unit/presentation/test_data_models.py @@ -1,6 +1,7 @@ import pytest -from guidellm.presentation.data_models import Bucket +from guidellm.presentation.data_models import BenchmarkDatum, Bucket +from tests.unit.mock_benchmark import mock_generative_benchmark @pytest.mark.smoke @@ -18,3 +19,10 @@ def test_bucket_from_data(): assert buckets[1].value == 8.0 assert buckets[1].count == 5 assert bucket_width == 1 + + +@pytest.mark.smoke +def test_from_benchmark_includes_strategy_display_str(): + mock_bm = mock_generative_benchmark() + bm = BenchmarkDatum.from_benchmark(mock_bm) + assert bm.strategy_display_str == "synchronous"