Skip to content

Integrate model statistics #4929

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 7 commits into from
Oct 21, 2025
+251 −147
Merged

Integrate model statistics #4929

Show file tree
Hide file tree
Changes from 5 commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
72cae30
Integrate throughput and latency
jpggvilaca Oct 20, 2025
97dadaf
Remove SearchField
jpggvilaca Oct 20, 2025
87ebbf2
Update data collection
jpggvilaca Oct 20, 2025
8340820
Refactor
jpggvilaca Oct 20, 2025
542ba5c
Update inference spec
jpggvilaca Oct 20, 2025
47e058d
Copilot comments
jpggvilaca Oct 20, 2025
2d00395
Avoid retrying on error; Add empty case scenario; Update test
jpggvilaca Oct 21, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
108 changes: 77 additions & 31 deletions application/ui/src/features/inference/aside/data-collection.component.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,14 @@ import { $api } from 'src/api/client';
import { components } from 'src/api/openapi-spec';
import { useProjectIdentifier } from 'src/hooks/use-project-identifier.hook';

type FixedRateDataCollectionPolicy = components['schemas']['FixedRateDataCollectionPolicy'];
type FixedRatePolicy = components['schemas']['FixedRateDataCollectionPolicy'];
type ConfidenceThresholdPolicy = components['schemas']['ConfidenceThresholdDataCollectionPolicy'];

const DEFAULTS = {
RATE: 12,
CONFIDENCE_THRESHOLD: 0.5,
MIN_SAMPLING_INTERVAL: 2.5,
} as const;

export const DataCollection = () => {
const projectId = useProjectIdentifier();
Expand All @@ -26,32 +33,46 @@ export const DataCollection = () => {
},
});

const isAutoCapturingEnabled = pipelineQuery.data?.data_collection_policies[0]?.enabled ?? false;
const defaultRate = 12;
const serverRate =
(pipelineQuery.data?.data_collection_policies[0] as FixedRateDataCollectionPolicy)?.rate ?? defaultRate;
const policies = pipelineQuery.data?.data_collection_policies ?? [];
const ratePolicy = policies[0] as FixedRatePolicy | undefined;
const confidencePolicy = policies[1] as ConfidenceThresholdPolicy | undefined;

// TODO: add confidence_threshold slider
const serverRate = ratePolicy?.rate ?? DEFAULTS.RATE;
const serverConfidenceThreshold = confidencePolicy?.confidence_threshold ?? DEFAULTS.CONFIDENCE_THRESHOLD;

const [localRate, setLocalRate] = useState(serverRate);
const [localConfidenceThreshold, setLocalConfidenceThreshold] = useState(serverConfidenceThreshold);

useEffect(() => setLocalRate(serverRate), [serverRate]);
useEffect(() => setLocalConfidenceThreshold(serverConfidenceThreshold), [serverConfidenceThreshold]);

const updatePolicies = (updates: {
rateEnabled?: boolean;
rate?: number;
confidenceEnabled?: boolean;
confidenceThreshold?: number;
}) => {
const newPolicies = [
{
type: 'fixed_rate' as const,
rate: updates.rate ?? serverRate,
enabled: updates.rateEnabled ?? ratePolicy?.enabled ?? false,
},
{
type: 'confidence_threshold' as const,
confidence_threshold: updates.confidenceThreshold ?? serverConfidenceThreshold,
min_sampling_interval: confidencePolicy?.min_sampling_interval ?? DEFAULTS.MIN_SAMPLING_INTERVAL,
enabled: updates.confidenceEnabled ?? confidencePolicy?.enabled ?? false,
},
];

useEffect(() => {
setLocalRate(serverRate);
}, [serverRate]);

const toggleAutoCapturing = (isEnabled: boolean) => {
patchPipelineMutation.mutate({
params: { path: { project_id: projectId } },
body: { data_collection_policies: [{ rate: defaultRate, enabled: isEnabled }] },
body: { data_collection_policies: newPolicies },
});
};

const updateRate = (value: number) => {
patchPipelineMutation.mutate({
params: { path: { project_id: projectId } },
body: { data_collection_policies: [{ rate: value, enabled: isAutoCapturingEnabled }] },
});
};
const isDisabled = patchPipelineMutation.isPending || !canEditPipeline;

return (
<Flex
Expand All @@ -64,33 +85,58 @@ export const DataCollection = () => {
<Heading level={4}>Data collection</Heading>
</Flex>
<Flex direction={'column'} UNSAFE_style={{ overflow: 'hidden auto' }}>
<Heading level={3} margin={0}>
Capture rate
</Heading>

<Text marginY={'size-100'}>Capture frames while the stream is running</Text>

<Switch
isSelected={isAutoCapturingEnabled}
onChange={toggleAutoCapturing}
isDisabled={patchPipelineMutation.isPending || !canEditPipeline}
isSelected={ratePolicy?.enabled ?? false}
onChange={(enabled) => updatePolicies({ rateEnabled: enabled })}
isDisabled={isDisabled}
marginBottom={'size-200'}
>
Toggle auto capturing
</Switch>

<Text>Capture frames while the stream is running</Text>
<Slider
step={0.1}
minValue={0}
maxValue={60}
value={localRate}
onChange={setLocalRate}
onChangeEnd={(rate) => updatePolicies({ rate })}
label='Rate'
isDisabled={isDisabled || !ratePolicy?.enabled}
/>

<Divider marginY={'size-400'} size={'S'} />

<Heading level={4} margin={0}>
Capture rate
<Heading level={3} margin={0}>
Confidence threshold
</Heading>

<Text marginY={'size-100'}>Capture frames when confidence is below threshold</Text>

<Switch
isSelected={confidencePolicy?.enabled ?? false}
onChange={(enabled) => updatePolicies({ confidenceEnabled: enabled })}
isDisabled={isDisabled}
>
Confidence threshold
</Switch>

<Slider
step={0.1}
step={0.01}
minValue={0}
maxValue={60}
value={localRate}
onChange={setLocalRate}
onChangeEnd={updateRate}
maxValue={1}
value={localConfidenceThreshold}
onChange={setLocalConfidenceThreshold}
onChangeEnd={(confidenceThreshold) => updatePolicies({ confidenceThreshold })}
marginY={'size-200'}
label='Rate'
isDisabled={patchPipelineMutation.isPending || !canEditPipeline}
label='Threshold'
isDisabled={isDisabled || !confidencePolicy?.enabled}
/>
</Flex>
</Flex>
Expand Down
147 changes: 65 additions & 82 deletions application/ui/src/features/inference/aside/graphs.component.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,63 +4,69 @@
import { useEffect, useRef, useState } from 'react';

import { Flex, Grid, Heading, View } from '@geti/ui';
import { useProjectIdentifier } from 'hooks/use-project-identifier.hook';
import { CartesianGrid, Label, Line, LineChart, ReferenceLine, XAxis, YAxis } from 'recharts';

const generateData = (n: number) => {
const result = new Array(n);
result[0] = 100;

for (let idx = 1; idx < n; idx++) {
const dx = 1 + Math.random() * 0.3 - 0.15;
result[idx] = Math.max(0, Math.min(100, result[idx - 1] * dx));
}

return result;
};

const useData = () => {
const [data, setData] = useState(
generateData(60).map((value, idx) => {
return { name: `${idx}`, value };
})
import { $api } from '../../../api/client';

interface DataPoint {
name: string;
value: number;
}

const POLLING_INTERVAL = 5000;
const MAX_DATA_POINTS = 60; // Keep last 60 data points

const useMetricsData = (projectId: string) => {
const [latencyData, setLatencyData] = useState<DataPoint[]>([]);
const [throughputData, setThroughputData] = useState<DataPoint[]>([]);
const counterRef = useRef(0);

const { data: metrics } = $api.useQuery(
'get',
'/api/projects/{project_id}/pipeline/metrics',
{
params: { path: { project_id: projectId } },
},
{
refetchInterval: POLLING_INTERVAL,
}
);

const timeout = useRef<ReturnType<typeof setTimeout>>(undefined);

useEffect(() => {
timeout.current = setTimeout(
() => {
const newData = data.slice(1);
const previous = newData.at(-1);

if (!previous) {
return;
}

const dx = 1 + Math.random() * 0.3 - 0.15;
newData.push({
name: `${Number(previous.name) + 1}`,
value: Math.max(0, Math.min(100, previous.value * dx)),
});

setData(newData);
},
Math.random() * 100 + 250
);

return () => {
if (timeout.current) {
clearTimeout(timeout.current);
}
};
});

return data;
if (!metrics) return;

const dataPointName = `${counterRef.current++}`;

setLatencyData((prev) => {
const newData = [
...prev,
{
name: dataPointName,
value: metrics.inference.latency.avg_ms ?? 0,
},
];

// Keep only last MAX_DATA_POINTS
return newData.slice(-MAX_DATA_POINTS);
});

setThroughputData((prev) => {
const newData = [
...prev,
{
name: dataPointName,
value: metrics.inference.throughput.avg_requests_per_second ?? 0,
},
];
return newData.slice(-MAX_DATA_POINTS);
});
}, [metrics]);

return { latencyData, throughputData, metrics };
};

const Graph = ({ label }: { label: string }) => {
const data = useData();

const Graph = ({ label, data }: { label: string; data: DataPoint[] }) => {
return (
<View padding={{ top: 'size-250', left: 'size-200', right: 'size-200', bottom: 'size-125' }}>
<LineChart width={500} height={200} data={data}>
Expand All @@ -70,16 +76,8 @@ const Graph = ({ label }: { label: string }) => {
dataKey='name'
tickLine={false}
tickMargin={8}
// tickFormatter={(value) => {
// const date = new Date(value);
// return date.toLocaleDateString('en-US', {
// month: 'short',
// day: 'numeric',
// });
// }}
/>
<YAxis
// ...
tickLine={false}
stroke='var(--spectrum-global-color-gray-900)'
dataKey='value'
Expand All @@ -88,7 +86,6 @@ const Graph = ({ label }: { label: string }) => {
}}
>
<Label
// ...
angle={-90}
value={label}
position='insideLeft'
Expand All @@ -100,14 +97,10 @@ const Graph = ({ label }: { label: string }) => {
}}
/>
</YAxis>
{/* <CartesianAxis stroke='var(--spectrum-global-color-gray-800)' /> */}
<CartesianGrid stroke='var(--spectrum-global-color-gray-400)' />
<ReferenceLine
x={data[0].name}
// ...
stroke='var(--spectrum-global-color-gray-600)'
strokeWidth={2}
/>
{data.length > 0 && (
<ReferenceLine x={data[0].name} stroke='var(--spectrum-global-color-gray-600)' strokeWidth={2} />
)}
<Line
type='linear'
dataKey='value'
Expand All @@ -122,6 +115,9 @@ const Graph = ({ label }: { label: string }) => {
};

export const Graphs = () => {
const projectId = useProjectIdentifier();
const { latencyData, throughputData } = useMetricsData(projectId);

return (
<Grid
gridArea={'aside'}
Expand All @@ -136,30 +132,17 @@ export const Graphs = () => {
<Heading level={4}>Model statistics</Heading>
</Flex>
<View gridArea={'graphs'} UNSAFE_style={{ overflow: 'hidden auto' }}>
{/* TODO: Extract these into a shared component */}
<View paddingY='size-200'>
<Heading level={4} marginBottom={'size-300'}>
Throughput
</Heading>
<Graph label='fps' />
</View>
<View paddingY='size-200'>
<Heading level={4} marginBottom={'size-300'}>
Latency per frame
</Heading>
<Graph label='ms per frame' />
</View>
<View paddingY='size-200'>
<Heading level={4} marginBottom={'size-300'}>
Model confidence
</Heading>
<Graph label='Score' />
<Graph label='requests/sec' data={throughputData} />
</View>
<View paddingY='size-200'>
<Heading level={4} marginBottom={'size-300'}>
Resource utilization
Latency
</Heading>
<Graph label='CPU' />
<Graph label='ms' data={latencyData} />
</View>
</View>
</Grid>
Expand Down
3 changes: 1 addition & 2 deletions application/ui/src/features/models/page-header.component.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
// Copyright (C) 2025 Intel Corporation
// SPDX-License-Identifier: Apache-2.0

import { Flex, Grid, Item, Picker, SearchField, Text } from '@geti/ui';
import { Flex, Grid, Item, Picker, Text } from '@geti/ui';

export const ModelsHeader = () => {
return (
Expand All @@ -18,7 +18,6 @@ export const ModelsHeader = () => {
<Item key='architecture'>Sort: Architecture</Item>
</Picker>
</Flex>
<SearchField marginStart={'auto'} />
</Grid>
);
};
Loading
Loading