Skip to content

Support llama.cpp's cache_n in timings info #287

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 6 commits into from
Sep 6, 2025
Merged

Support llama.cpp's cache_n in timings info #287

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
570d527
show cached prompt tokens in metrics
mostlygeek Sep 6, 2025
9c6193e
tidy up typescript
mostlygeek Sep 6, 2025
32b73f9
add tooltip to headers in activity page
mostlygeek Sep 6, 2025
ae004ed
fix typescript build issues
mostlygeek Sep 6, 2025
3118f86
tweak activity now timestamp
mostlygeek Sep 6, 2025
a023624
add a space
mostlygeek Sep 6, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion proxy/metrics_middleware.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,6 @@ func MetricsMiddleware(pm *ProxyManager) gin.HandlerFunc {
} else {
writer.metricsRecorder.processNonStreamingResponse(writer.body)
}

}
}

Expand All @@ -73,6 +72,7 @@ func (rec *MetricsRecorder) parseAndRecordMetrics(jsonData gjson.Result) bool {
}

// default values
cachedTokens := -1 // unknown or missing data
outputTokens := 0
inputTokens := 0

Expand All @@ -93,11 +93,16 @@ func (rec *MetricsRecorder) parseAndRecordMetrics(jsonData gjson.Result) bool {
promptPerSecond = jsonData.Get("timings.prompt_per_second").Float()
tokensPerSecond = jsonData.Get("timings.predicted_per_second").Float()
durationMs = int(jsonData.Get("timings.prompt_ms").Float() + jsonData.Get("timings.predicted_ms").Float())

if cachedValue := jsonData.Get("timings.cache_n"); cachedValue.Exists() {
cachedTokens = int(cachedValue.Int())
}
}

rec.metricsMonitor.addMetrics(TokenMetrics{
Timestamp: time.Now(),
Model: rec.realModelName,
CachedTokens: cachedTokens,
InputTokens: inputTokens,
OutputTokens: outputTokens,
PromptPerSecond: promptPerSecond,
Expand Down
2 changes: 1 addition & 1 deletion proxy/metrics_monitor.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ type TokenMetrics struct {
ID int `json:"id"`
Timestamp time.Time `json:"timestamp"`
Model string `json:"model"`
CachedTokens int `json:"cache_tokens"`
InputTokens int `json:"input_tokens"`
OutputTokens int `json:"output_tokens"`
PromptPerSecond float64 `json:"prompt_per_second"`
Expand Down Expand Up @@ -61,7 +62,6 @@ func (mp *MetricsMonitor) addMetrics(metric TokenMetrics) {
if len(mp.metrics) > mp.maxMetrics {
mp.metrics = mp.metrics[len(mp.metrics)-mp.maxMetrics:]
}

event.Emit(TokenMetricsEvent{Metrics: metric})
}

Expand Down
1 change: 1 addition & 0 deletions ui/src/contexts/APIProvider.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ interface Metrics {
id: number;
timestamp: string;
model: string;
cache_tokens: number;
input_tokens: number;
output_tokens: number;
prompt_per_second: number;
Expand Down
73 changes: 64 additions & 9 deletions ui/src/pages/Activity.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,10 +1,6 @@
import { useMemo } from "react";
import { useAPI } from "../contexts/APIProvider";

const formatTimestamp = (timestamp: string): string => {
return new Date(timestamp).toLocaleString();
};

const formatSpeed = (speed: number): string => {
return speed < 0 ? "unknown" : speed.toFixed(2) + " t/s";
};
Expand All @@ -13,6 +9,33 @@ const formatDuration = (ms: number): string => {
return (ms / 1000).toFixed(2) + "s";
};

const formatRelativeTime = (timestamp: string): string => {
const now = new Date();
const date = new Date(timestamp);
const diffInSeconds = Math.floor((now.getTime() - date.getTime()) / 1000);

// Handle future dates by returning "just now"
if (diffInSeconds < 5) {
return "now";
}

if (diffInSeconds < 60) {
return `${diffInSeconds}s ago`;
}

const diffInMinutes = Math.floor(diffInSeconds / 60);
if (diffInMinutes < 60) {
return `${diffInMinutes}m ago`;
}

const diffInHours = Math.floor(diffInMinutes / 60);
if (diffInHours < 24) {
return `${diffInHours}h ago`;
}

return "a while ago";
};

const ActivityPage = () => {
const { metrics } = useAPI();
const sortedMetrics = useMemo(() => {
Expand All @@ -32,11 +55,16 @@ const ActivityPage = () => {
<table className="min-w-full divide-y">
<thead>
<tr>
<th className="px-4 py-3 text-left text-xs font-medium uppercase tracking-wider">Id</th>
<th className="px-6 py-3 text-left text-xs font-medium uppercase tracking-wider">Timestamp</th>
<th className="px-4 py-3 text-left text-xs font-medium uppercase tracking-wider">ID</th>
<th className="px-6 py-3 text-left text-xs font-medium uppercase tracking-wider">Time</th>
<th className="px-6 py-3 text-left text-xs font-medium uppercase tracking-wider">Model</th>
<th className="px-6 py-3 text-left text-xs font-medium uppercase tracking-wider">Input Tokens</th>
<th className="px-6 py-3 text-left text-xs font-medium uppercase tracking-wider">Output Tokens</th>
<th className="px-6 py-3 text-left text-xs font-medium uppercase tracking-wider">
Cached <Tooltip content="prompt tokens from cache" />
</th>
<th className="px-6 py-3 text-left text-xs font-medium uppercase tracking-wider">
Prompt <Tooltip content="new prompt tokens processed" />
</th>
<th className="px-6 py-3 text-left text-xs font-medium uppercase tracking-wider">Generated</th>
<th className="px-6 py-3 text-left text-xs font-medium uppercase tracking-wider">Prompt Processing</th>
<th className="px-6 py-3 text-left text-xs font-medium uppercase tracking-wider">Generation Speed</th>
<th className="px-6 py-3 text-left text-xs font-medium uppercase tracking-wider">Duration</th>
Expand All @@ -46,8 +74,11 @@ const ActivityPage = () => {
{sortedMetrics.map((metric) => (
<tr key={`metric_${metric.id}`}>
<td className="px-4 py-4 whitespace-nowrap text-sm">{metric.id + 1 /* un-zero index */}</td>
<td className="px-6 py-4 whitespace-nowrap text-sm">{formatTimestamp(metric.timestamp)}</td>
<td className="px-6 py-4 whitespace-nowrap text-sm">{formatRelativeTime(metric.timestamp)}</td>
<td className="px-6 py-4 whitespace-nowrap text-sm">{metric.model}</td>
<td className="px-6 py-4 whitespace-nowrap text-sm">
{metric.cache_tokens > 0 ? metric.cache_tokens.toLocaleString() : "-"}
</td>
<td className="px-6 py-4 whitespace-nowrap text-sm">{metric.input_tokens.toLocaleString()}</td>
<td className="px-6 py-4 whitespace-nowrap text-sm">{metric.output_tokens.toLocaleString()}</td>
<td className="px-6 py-4 whitespace-nowrap text-sm">{formatSpeed(metric.prompt_per_second)}</td>
Expand All @@ -63,4 +94,28 @@ const ActivityPage = () => {
);
};

interface TooltipProps {
content: string;
}

const Tooltip: React.FC<TooltipProps> = ({ content }) => {
return (
<div className="relative group inline-block">
ⓘ
<div
className="absolute top-full left-1/2 transform -translate-x-1/2 mt-2
px-3 py-2 bg-gray-900 text-white text-sm rounded-md
opacity-0 group-hover:opacity-100 transition-opacity
duration-200 pointer-events-none whitespace-nowrap z-50 normal-case"
>
{content}
<div
className="absolute bottom-full left-1/2 transform -translate-x-1/2
border-4 border-transparent border-b-gray-900"
></div>
</div>
</div>
);
};

export default ActivityPage;
Loading