feat: Ensure warnings / errors on ai analysis sessions get tracked and rendered

Author: bcanfield

apps/api/src/routes/dashboard/dashboard.mappers.ts

@@ -40,6 +40,7 @@ export function mapJobActivityTimelineToResponse(timeline: JobActivityTimeline):
       reasoning: event.reasoning,
       toolCalls: event.toolCalls,
       tokenUsage: event.tokenUsage,
+      warnings: event.warnings,
     })),
     total: timeline.total,
     summary: timeline.summary,

apps/api/src/routes/jobs/jobs.handlers.ts

@@ -495,6 +495,7 @@ export const getSession: AppRouteHandler<routes.GetSessionRoute> = async (c) =>
       reasoning: session.reasoning,
       tokenUsage: session.tokenUsage,
       durationMs: session.durationMs,
+      warnings: session.warnings,
     }, HTTPStatusCodes.OK);
   });
 };

apps/migrator/src/seed.ts

@@ -826,6 +826,7 @@ function generateAISessions(): Array<typeof schema.aiAnalysisSessions.$inferInse
     reasoning: string,
     actions: string[] = [],
     extraToolCalls: Array<Record<string, unknown>> = [],
+    warnings?: Array<{ code: string; message: string; meta?: Record<string, unknown> }>,
   ) => {
     const sessionTime = new Date(BLACK_FRIDAY_START.getTime() + minuteOffset * 60_000);
     const responseBody = getAISessionResponseBody(endpointId, minuteOffset);
@@ -845,6 +846,7 @@ function generateAISessions(): Array<typeof schema.aiAnalysisSessions.$inferInse
       tokenUsage: 800 + Math.floor(Math.random() * 500),
       durationMs: 250 + Math.floor(Math.random() * 200),
       nextAnalysisAt: new Date(sessionTime.getTime() + 20 * 60_000),
+      ...(warnings ? { warnings } : {}),
     });
   };
 
@@ -860,13 +862,20 @@ function generateAISessions(): Array<typeof schema.aiAnalysisSessions.$inferInse
   // Hour 2: 09:00-10:00 - Peak building (4 sessions)
   generateBFSession(60, "ep-traffic-monitor", "ALERT: Traffic spiking. Page load degrading.", [], [tc.proposeInterval(25_000, 60, "Critical traffic—max monitoring")]);
   generateBFSession(75, "ep-order-processor-health", "Order queue building. Failure rate rising.");
-  generateBFSession(90, "ep-inventory-sync-check", "Inventory sync lagging. Queue depth critical.");
+  generateBFSession(90, "ep-inventory-sync-check", "Inventory sync lagging. Queue depth critical.", [], [], [
+    { code: "output_truncated", message: "Step 2 output was truncated (finishReason: length). The model was analyzing inventory queue backlog history and hit the output token limit.", meta: { stepIndex: 1, finishReason: "length" } },
+  ]);
   generateBFSession(105, "ep-traffic-monitor", "PEAK TRAFFIC. Page load strained. All systems under load.", [], [tc.proposeInterval(20_000, 60, "Peak load—minimum interval")]);
 
   // Hour 3: 10:00-11:00 - Critical phase (4 sessions)
-  generateBFSession(120, "ep-slow-page-analyzer", "CRITICAL: p95 latency extremely high. Database is bottleneck.", ["propose_interval"], [tc.pauseUntil(null, "Activating recovery endpoints"), tc.proposeNext(5_000, 5, "Trigger cache warmup")]);
+  generateBFSession(120, "ep-slow-page-analyzer", "CRITICAL: p95 latency extremely high. Database is bottleneck.", ["propose_interval"], [tc.pauseUntil(null, "Activating recovery endpoints"), tc.proposeNext(5_000, 5, "Trigger cache warmup")], [
+    { code: "output_truncated", message: "Step 3 output was truncated (finishReason: length). The model was generating an extended root-cause analysis of database connection pool exhaustion and hit the output token limit.", meta: { stepIndex: 2, finishReason: "length" } },
+    { code: "missing_final_tool", message: "The model did not call submit_analysis. Analysis was extracted from the last tool call and raw model output." },
+  ]);
   generateBFSession(135, "ep-order-processor-health", "Order failure rate critical. Queue depth high. Recovery actions triggered.");
-  generateBFSession(150, "ep-inventory-sync-check", "Inventory lag critical. Failed syncs detected. Critical but stable.");
+  generateBFSession(150, "ep-inventory-sync-check", "Inventory lag critical. Failed syncs detected. Critical but stable.", [], [], [
+    { code: "missing_reasoning", message: "The model did not produce any reasoning text. It jumped directly to tool calls without a natural language explanation." },
+  ]);
   generateBFSession(165, "ep-traffic-monitor", "Sustained peak traffic. Systems holding under load.");
 
   // Hour 4: 11:00-12:00 - Sustained load (3 sessions)

apps/web/src/components/ai/session-item.tsx

@@ -1,15 +1,22 @@
-import { AlertCircle, Check, ChevronRight } from "lucide-react";
+import { AlertCircle, AlertTriangle, Check, ChevronRight } from "lucide-react";
 import { useEffect, useRef, useState } from "react";
 import { Button } from "@cronicorn/ui-library/components/button";
 import { cn } from "@cronicorn/ui-library/lib/utils";
 
+export type AISessionWarning = {
+    code: string;
+    message: string;
+    meta?: Record<string, unknown>;
+};
+
 export type AISession = {
     id: string;
     analyzedAt: string;
     reasoning: string;
     toolCalls: Array<{ tool: string; args?: unknown; result?: unknown }>;
     tokenUsage: number | null;
     durationMs: number | null;
+    warnings?: AISessionWarning[];
 };
 
 // Type guards and safe accessors
@@ -126,6 +133,15 @@ export function AISessionItem({
                         <span>{formattedDuration}</span>
                     </>
                 )}
+                {Array.isArray(session.warnings) && session.warnings.length > 0 && (
+                    <>
+                        <span className="text-border">•</span>
+                        <span className="inline-flex items-center gap-1 text-amber-600 dark:text-amber-400">
+                            <AlertTriangle className="h-3 w-3" />
+                            {session.warnings.length} {session.warnings.length === 1 ? "warning" : "warnings"}
+                        </span>
+                    </>
+                )}
             </div>
 
             {session.reasoning && (
@@ -151,6 +167,19 @@ export function AISessionItem({
                     )}
                 </div>
             )}
+            {Array.isArray(session.warnings) && session.warnings.length > 0 && (
+                <div className="mt-1.5 space-y-1">
+                    {session.warnings.map((warning, idx) => (
+                        <div
+                            key={idx}
+                            className="flex items-start gap-1.5 rounded-md border border-amber-200 bg-amber-50 px-2 py-1.5 text-xs text-amber-800 dark:border-amber-800 dark:bg-amber-950/30 dark:text-amber-300"
+                        >
+                            <AlertTriangle className="mt-0.5 h-3 w-3 shrink-0" />
+                            <span>{warning.message}</span>
+                        </div>
+                    ))}
+                </div>
+            )}
             {Array.isArray(session.toolCalls) && session.toolCalls.length > 0 && (
                 <div className="mt-1.5 space-y-2">
                     <div className="flex flex-wrap items-center gap-2 text-xs">

apps/web/src/components/dashboard-new/activity-event-item.tsx

@@ -1,6 +1,6 @@
 import { useEffect, useMemo, useRef, useState } from "react";
 import { Link } from "@tanstack/react-router";
-import { AlertCircle, Brain, Check, ChevronDown, ChevronUp, Clock, Play, Settings2, Zap } from "lucide-react";
+import { AlertCircle, AlertTriangle, Brain, Check, ChevronDown, ChevronUp, Clock, Play, Settings2, Zap } from "lucide-react";
 
 import { Badge } from "@cronicorn/ui-library/components/badge";
 import { Button } from "@cronicorn/ui-library/components/button";
@@ -18,6 +18,7 @@ export type ActivityEvent = {
     // Session-specific fields
     reasoning?: string;
     toolCalls?: Array<{ tool: string; args?: unknown; result?: unknown }>;
+    warnings?: Array<{ code: string; message: string; meta?: Record<string, unknown> }>;
 };
 
 /** Action tools that make scheduling changes */
@@ -118,6 +119,12 @@ export function ActivityEventItem({ event }: { event: ActivityEvent }) {
                                         {actionToolCount}
                                     </Badge>
                                 )}
+                                {Array.isArray(event.warnings) && event.warnings.length > 0 && (
+                                    <Badge variant="outline" className="text-[10px] px-1 py-0 gap-0.5 text-error border-error">
+                                        <AlertTriangle className="size-2.5" />
+                                        {event.warnings.length}
+                                    </Badge>
+                                )}
                             </>
                         )}
                     </div>

apps/web/src/routes/_authed/ai-sessions.$id.tsx

@@ -1,6 +1,6 @@
 import { useSuspenseQuery } from "@tanstack/react-query";
 import { Link, createFileRoute } from "@tanstack/react-router";
-import { Brain, Clock, ExternalLink, Zap } from "lucide-react";
+import { AlertTriangle, Brain, Clock, ExternalLink, Zap } from "lucide-react";
 
 import { Badge } from "@cronicorn/ui-library/components/badge";
 import { CodeDisplay } from "../../components/composed/code-display";
@@ -91,6 +91,25 @@ function AISessionDetailsPage() {
           </InfoGrid>
         </DetailSection>
 
+        {Array.isArray(session.warnings) && session.warnings.length > 0 && (
+          <DetailSection title="Warnings">
+            <div className="space-y-2">
+              {session.warnings.map((warning, idx) => (
+                <div
+                  key={idx}
+                  className="flex items-start gap-2 rounded-md border border-error bg-error/15 px-3 py-2 text-sm text-error"
+                >
+                  <AlertTriangle className="mt-0.5 size-4 shrink-0" />
+                  <div>
+                    <span className="font-medium">{warning.code}</span>
+                    <p className="text-error">{warning.message}</p>
+                  </div>
+                </div>
+              ))}
+            </div>
+          </DetailSection>
+        )}
+
         <DetailSection title="AI Reasoning">
           {session.reasoning ? (
             <div className="prose prose-sm dark:prose-invert max-w-none">

packages/adapter-ai/src/client.ts

@@ -1,6 +1,6 @@
 // Vercel AI SDK client implementation
 
-import type { AIClient, Tool } from "@cronicorn/domain";
+import type { AIClient, AISessionWarning, Tool } from "@cronicorn/domain";
 
 import { generateText, hasToolCall, stepCountIs, tool } from "ai";
 import { z } from "zod";
@@ -174,6 +174,31 @@ export function createVercelAiClient(config: VercelAiClientConfig): AIClient {
           totalTokens: result.totalUsage?.totalTokens,
         });
 
+        // Flag when any step was truncated by the output token limit
+        const truncatedSteps = result.steps
+          .map((step, i) => ({ step: i, finishReason: step.finishReason }))
+          .filter(s => s.finishReason === "length");
+
+        // Build warnings for upstream consumers (planner, UI)
+        const warnings: AISessionWarning[] = [];
+
+        if (truncatedSteps.length > 0) {
+          config.logger?.warn("Output token limit hit — model response was truncated", {
+            truncatedSteps,
+            maxOutputTokens: maxTokens || config.maxOutputTokens || 4096,
+            totalTokens: result.totalUsage?.totalTokens,
+          });
+
+          warnings.push({
+            code: "output_truncated",
+            message: `Output token limit hit on ${truncatedSteps.length} step(s) — model response was truncated`,
+            meta: {
+              truncatedSteps,
+              maxOutputTokens: maxTokens || config.maxOutputTokens || 4096,
+            },
+          });
+        }
+
         // Diagnostic logging when no tool calls captured — should not happen with prepareStep
         if (capturedToolCalls.length === 0) {
           config.logger?.warn("Zero tool calls captured", {
@@ -197,6 +222,7 @@ export function createVercelAiClient(config: VercelAiClientConfig): AIClient {
           reasoning: result.text,
           // totalUsage accumulates across all steps; usage is last step only
           tokenUsage: result.totalUsage?.totalTokens,
+          warnings: warnings.length > 0 ? warnings : undefined,
         };
       }
       catch (error) {

packages/adapter-drizzle/migrations/0024_bored_cammi.sql

@@ -0,0 +1 @@
+ALTER TABLE "ai_analysis_sessions" ADD COLUMN "warnings" jsonb;