Merge pull request #8505 from sagemathinc/admin-llm-timing

frontend/admin/llm: add a "Timings" column

Author: williamstein

src/packages/frontend/admin/llm/admin-llm-test.tsx

@@ -1,4 +1,13 @@
-import { Alert, Button, Input, Select, Space, Table } from "antd";
+import {
+  Alert,
+  Button,
+  Input,
+  Progress,
+  Select,
+  Space,
+  Table,
+  Tooltip,
+} from "antd";
 
 import {
   redux,
@@ -20,12 +29,13 @@ import { trunc_middle } from "@cocalc/util/misc";
 import { COLORS } from "@cocalc/util/theme";
 import { PROMPTS } from "./tests";
 import { Value } from "./value";
-
 interface TestResult {
   model: string;
   status: "pending" | "running" | "passed" | "failed";
   output: string;
   error?: string;
+  firstResponseTime?: number; // Time in milliseconds until first token
+  totalTime?: number; // Total time in milliseconds until completion
 }
 
 export function TestLLMAdmin() {
@@ -101,6 +111,10 @@ export function TestLLMAdmin() {
 
     return new Promise((resolve) => {
       try {
+        const startTime = Date.now();
+        let firstResponseTime: number | undefined;
+        let totalTime: number | undefined;
+
         const llmStream = webapp_client.openai_client.queryStream({
           input: prompt,
           project_id: null,
@@ -116,6 +130,10 @@ export function TestLLMAdmin() {
         llmStream.on("token", (token) => {
           console.log({ model, system, token });
           if (token != null) {
+            // Record first response time if this is the first token
+            if (firstResponseTime === undefined) {
+              firstResponseTime = Date.now() - startTime;
+            }
             reply += token;
             // Update the result in real-time
             setTestResults((prev) =>
@@ -125,22 +143,28 @@ export function TestLLMAdmin() {
             );
           } else {
             // Stream is complete (token is null)
+            totalTime = Date.now() - startTime;
             const passed = expectedRegex.test(reply);
             resolve({
               model,
               status: passed ? "passed" : "failed",
               output: reply,
+              firstResponseTime,
+              totalTime,
             });
           }
         });
 
         llmStream.on("error", (err) => {
+          totalTime = Date.now() - startTime;
           console.error(`Error in LLM stream for model ${model}:`, err);
           resolve({
             model,
             status: "failed",
             output: reply,
             error: err?.toString(),
+            firstResponseTime,
+            totalTime,
           });
         });
 
@@ -241,6 +265,57 @@ export function TestLLMAdmin() {
     }
   }
 
+  function formatTiming(timeMs: number | undefined): string {
+    if (timeMs === undefined) return "-";
+    return `${(timeMs / 1000).toFixed(1)}s`;
+  }
+
+  function renderTimingColumn(record: TestResult) {
+    const { firstResponseTime, totalTime, status } = record;
+
+    if (status === "pending" || status === "running") {
+      return <span style={{ color: COLORS.GRAY_M }}>-</span>;
+    }
+
+    if (firstResponseTime === undefined || totalTime === undefined) {
+      return <span style={{ color: COLORS.GRAY_M }}>-</span>;
+    }
+
+    // Calculate progress bar values (normalize to 10 seconds max)
+    const maxTime = Math.max(
+      10000,
+      ...testResults.filter((r) => r.totalTime).map((r) => r.totalTime!),
+    );
+    const totalPercent = Math.min(100, (totalTime / maxTime) * 100);
+
+    // Determine if this is one of the slowest (top 10% quantile)
+    const completedResults = testResults.filter(
+      (r) => r.totalTime !== undefined,
+    );
+    const sortedTimes = completedResults
+      .map((r) => r.totalTime!)
+      .sort((a, b) => b - a);
+    const slowThreshold =
+      sortedTimes[Math.floor(sortedTimes.length * 0.1)] || 0;
+    const isSlow = totalTime >= slowThreshold && completedResults.length > 1;
+
+    return (
+      <div>
+        <Tooltip title="First response time / Total completion time">
+          <div style={{ marginBottom: 2 }}>
+            {formatTiming(firstResponseTime)}/{formatTiming(totalTime)}
+          </div>
+        </Tooltip>
+        <Progress
+          percent={totalPercent}
+          size="small"
+          status={isSlow ? "exception" : "normal"}
+          showInfo={false}
+        />
+      </div>
+    );
+  }
+
   function renderTestResults() {
     if (testResults.length === 0) {
       return (
@@ -292,6 +367,12 @@ export function TestLLMAdmin() {
             <span style={{ color: COLORS.GRAY_M }}>-</span>
           ),
       },
+      {
+        title: "Timing",
+        key: "timing",
+        width: 120,
+        render: (_, record: TestResult) => renderTimingColumn(record),
+      },
       {
         title: "Test",
         key: "test",