diff --git a/mcpjam-inspector/client/src/components/ChatTabV2.tsx b/mcpjam-inspector/client/src/components/ChatTabV2.tsx index 3188c2c03..d4b6c1cf0 100644 --- a/mcpjam-inspector/client/src/components/ChatTabV2.tsx +++ b/mcpjam-inspector/client/src/components/ChatTabV2.tsx @@ -1,8 +1,16 @@ -import { FormEvent, useMemo, useState, useEffect, useCallback } from "react"; +import { + FormEvent, + useMemo, + useState, + useEffect, + useCallback, + useRef, +} from "react"; import { ArrowDown } from "lucide-react"; import { useAuth } from "@workos-inc/authkit-react"; import { useConvexAuth } from "convex/react"; -import type { ContentBlock } from "@modelcontextprotocol/sdk/types.js"; +import { toast } from "sonner"; + import { ModelDefinition } from "@/shared/types"; import { LoggerView } from "./logger-view"; import { @@ -42,6 +50,11 @@ import { useSharedAppState } from "@/state/app-state-context"; import { useWorkspaceServers } from "@/hooks/useViews"; import { HOSTED_MODE } from "@/lib/config"; import { buildOAuthTokensByServerId } from "@/lib/oauth/oauth-tokens"; +import { buildWidgetModelContextMessages } from "@/lib/mcp-ui/model-context-messages"; +import { + useWidgetStateSync, + type ModelContextItem, +} from "@/hooks/use-widget-state-sync"; interface ChatTabProps { connectedOrConnectingServerConfigs: Record; @@ -98,18 +111,7 @@ export function ChatTabV2({ ); const [fileAttachments, setFileAttachments] = useState([]); const [skillResults, setSkillResults] = useState([]); - const [widgetStateQueue, setWidgetStateQueue] = useState< - { toolCallId: string; state: unknown }[] - >([]); - const [modelContextQueue, setModelContextQueue] = useState< - { - toolCallId: string; - context: { - content?: ContentBlock[]; - structuredContent?: Record; - }; - }[] - >([]); + const resetWidgetSyncRef = useRef<() => void>(() => {}); const [elicitation, setElicitation] = useState( null, ); @@ -198,10 +200,20 @@ export function ChatTabV2({ minimalMode, onReset: () => { setInput(""); - setWidgetStateQueue([]); + resetWidgetSyncRef.current(); }, }); + const { + enqueueWidgetStateSync, + setWidgetStateQueue, + widgetStateSyncRef, + modelContextQueueRef, + setModelContextQueue, + resetWidgetSync, + } = useWidgetStateSync({ status, setMessages }); + resetWidgetSyncRef.current = resetWidgetSync; + // Check if thread is empty const isThreadEmpty = !messages.some( (msg) => msg.role === "user" || msg.role === "assistant", @@ -264,94 +276,19 @@ export function ChatTabV2({ onHasMessagesChange?.(!isThreadEmpty); }, [isThreadEmpty, onHasMessagesChange]); - // Widget state management - const applyWidgetStateUpdates = useCallback( - ( - prevMessages: typeof messages, - updates: { toolCallId: string; state: unknown }[], - ) => { - let nextMessages = prevMessages; - - for (const { toolCallId, state } of updates) { - const messageId = `widget-state-${toolCallId}`; - - if (state === null) { - const filtered = nextMessages.filter((msg) => msg.id !== messageId); - nextMessages = filtered; - continue; - } - - const stateText = `The state of widget ${toolCallId} is: ${JSON.stringify(state)}`; - const existingIndex = nextMessages.findIndex( - (msg) => msg.id === messageId, - ); - - if (existingIndex !== -1) { - const existingMessage = nextMessages[existingIndex]; - const existingText = - existingMessage.parts?.[0]?.type === "text" - ? (existingMessage.parts[0] as { text?: string }).text - : null; - - if (existingText === stateText) { - continue; - } - - const updatedMessages = [...nextMessages]; - updatedMessages[existingIndex] = { - id: messageId, - role: "assistant", - parts: [{ type: "text" as const, text: stateText }], - }; - nextMessages = updatedMessages; - continue; - } - - nextMessages = [ - ...nextMessages, - { - id: messageId, - role: "assistant", - parts: [{ type: "text" as const, text: stateText }], - }, - ]; - } - - return nextMessages; - }, - [], - ); - const handleWidgetStateChange = useCallback( (toolCallId: string, state: unknown) => { if (status === "ready") { - setMessages((prevMessages) => - applyWidgetStateUpdates(prevMessages, [{ toolCallId, state }]), - ); + void enqueueWidgetStateSync([{ toolCallId, state }]); } else { setWidgetStateQueue((prev) => [...prev, { toolCallId, state }]); } }, - [status, setMessages, applyWidgetStateUpdates], + [status, enqueueWidgetStateSync], ); - useEffect(() => { - if (status !== "ready" || widgetStateQueue.length === 0) return; - - setMessages((prevMessages) => - applyWidgetStateUpdates(prevMessages, widgetStateQueue), - ); - setWidgetStateQueue([]); - }, [status, widgetStateQueue, setMessages, applyWidgetStateUpdates]); - const handleModelContextUpdate = useCallback( - ( - toolCallId: string, - context: { - content?: ContentBlock[]; - structuredContent?: Record; - }, - ) => { + (toolCallId: string, context: ModelContextItem["context"]) => { // Queue model context to be included in next message setModelContextQueue((prev) => { // Remove any existing context from same widget (overwrite pattern per SEP-1865) @@ -359,7 +296,7 @@ export function ChatTabV2({ return [...filtered, { toolCallId, context }]; }); }, - [], + [setModelContextQueue], ); // Elicitation SSE listener @@ -478,64 +415,62 @@ export function ChatTabV2({ skillResults.length > 0 || fileAttachments.length > 0; if (hasContent && status === "ready" && !submitBlocked) { - posthog.capture("send_message", { - location: "chat_tab", - platform: detectPlatform(), - environment: detectEnvironment(), - model_id: selectedModel?.id ?? null, - model_name: selectedModel?.name ?? null, - model_provider: selectedModel?.provider ?? null, - }); + try { + // Ensure any async widget-state -> message conversion is complete + // before submitting the next user turn. + await widgetStateSyncRef.current; + + posthog.capture("send_message", { + location: "chat_tab", + platform: detectPlatform(), + environment: detectEnvironment(), + model_id: selectedModel?.id ?? null, + model_name: selectedModel?.name ?? null, + model_provider: selectedModel?.provider ?? null, + }); + + // Build messages from MCP prompts + const promptMessages = buildMcpPromptMessages(mcpPromptResults); + if (promptMessages.length > 0) { + setMessages((prev) => [...prev, ...(promptMessages as any[])]); + } - // Build messages from MCP prompts - const promptMessages = buildMcpPromptMessages(mcpPromptResults); - if (promptMessages.length > 0) { - setMessages((prev) => [...prev, ...(promptMessages as any[])]); - } + // Build messages from skills + const skillMessages = buildSkillToolMessages(skillResults); + if (skillMessages.length > 0) { + setMessages((prev) => [...prev, ...(skillMessages as any[])]); + } - // Build messages from skills - const skillMessages = buildSkillToolMessages(skillResults); - if (skillMessages.length > 0) { - setMessages((prev) => [...prev, ...(skillMessages as any[])]); - } + // Include any pending model context from widgets (SEP-1865 ui/update-model-context) + // Sent as hidden user messages; preserve image/audio blocks as file parts. + const contextMessages = await buildWidgetModelContextMessages( + modelContextQueueRef.current, + ); - // Include any pending model context from widgets (SEP-1865 ui/update-model-context) - // Sent as "user" messages for compatibility with model provider APIs - const contextMessages = modelContextQueue.map( - ({ toolCallId, context }) => ({ - id: `model-context-${toolCallId}-${Date.now()}`, - role: "user" as const, - parts: [ - { - type: "text" as const, - text: `Widget ${toolCallId} context: ${JSON.stringify(context)}`, - }, - ], - metadata: { - source: "widget-model-context", - toolCallId, - }, - }), - ); + if (contextMessages.length > 0) { + setMessages((prev) => [...prev, ...(contextMessages as any[])]); + } - if (contextMessages.length > 0) { - setMessages((prev) => [...prev, ...(contextMessages as any[])]); + // Convert file attachments to FileUIPart[] format for the AI SDK + const files = + fileAttachments.length > 0 + ? await attachmentsToFileUIParts(fileAttachments) + : undefined; + + sendMessage({ text: input, files }); + setInput(""); + setMcpPromptResults([]); + setSkillResults([]); + // Revoke object URLs and clear file attachments + revokeFileAttachmentUrls(fileAttachments); + setFileAttachments([]); + setModelContextQueue([]); // Clear after sending + } catch (err) { + console.error("[ChatTabV2] Submit failed:", err); + toast.error( + err instanceof Error ? err.message : "Failed to send message", + ); } - - // Convert file attachments to FileUIPart[] format for the AI SDK - const files = - fileAttachments.length > 0 - ? await attachmentsToFileUIParts(fileAttachments) - : undefined; - - sendMessage({ text: input, files }); - setInput(""); - setMcpPromptResults([]); - setSkillResults([]); - // Revoke object URLs and clear file attachments - revokeFileAttachmentUrls(fileAttachments); - setFileAttachments([]); - setModelContextQueue([]); // Clear after sending } }; diff --git a/mcpjam-inspector/client/src/components/ui-playground/PlaygroundMain.tsx b/mcpjam-inspector/client/src/components/ui-playground/PlaygroundMain.tsx index 2fabc01f7..aa56db5c7 100644 --- a/mcpjam-inspector/client/src/components/ui-playground/PlaygroundMain.tsx +++ b/mcpjam-inspector/client/src/components/ui-playground/PlaygroundMain.tsx @@ -11,10 +11,18 @@ * which manages PiP/fullscreen at the widget level. */ -import { FormEvent, useState, useEffect, useCallback, useMemo } from "react"; +import { + FormEvent, + useState, + useEffect, + useCallback, + useMemo, + useRef, +} from "react"; import { ArrowDown, Braces, Loader2, Trash2 } from "lucide-react"; import { useAuth } from "@workos-inc/authkit-react"; -import type { ContentBlock } from "@modelcontextprotocol/sdk/types.js"; +import { toast } from "sonner"; + import { ModelDefinition } from "@/shared/types"; import { cn } from "@/lib/utils"; import { Thread } from "@/components/chat-v2/thread"; @@ -62,6 +70,11 @@ import { ToolRenderOverride } from "@/components/chat-v2/thread/tool-render-over import { useConvexAuth } from "convex/react"; import { useWorkspaceServers } from "@/hooks/useViews"; import { buildOAuthTokensByServerId } from "@/lib/oauth/oauth-tokens"; +import { buildWidgetModelContextMessages } from "@/lib/mcp-ui/model-context-messages"; +import { + useWidgetStateSync, + type ModelContextItem, +} from "@/hooks/use-widget-state-sync"; /** Custom device config - dimensions come from store */ const CUSTOM_DEVICE_BASE = { @@ -184,15 +197,7 @@ export function PlaygroundMain({ ); const [fileAttachments, setFileAttachments] = useState([]); const [skillResults, setSkillResults] = useState([]); - const [modelContextQueue, setModelContextQueue] = useState< - { - toolCallId: string; - context: { - content?: ContentBlock[]; - structuredContent?: Record; - }; - }[] - >([]); + const resetWidgetSyncRef = useRef<() => void>(() => {}); const [showClearConfirm, setShowClearConfirm] = useState(false); const [xrayMode, setXrayMode] = useState(false); const [isWidgetFullscreen, setIsWidgetFullscreen] = useState(false); @@ -283,9 +288,20 @@ export function PlaygroundMain({ hostedOAuthTokens, onReset: () => { setInput(""); + resetWidgetSyncRef.current(); }, }); + const { + enqueueWidgetStateSync, + setWidgetStateQueue, + widgetStateSyncRef, + modelContextQueueRef, + setModelContextQueue, + resetWidgetSync, + } = useWidgetStateSync({ status, setMessages }); + resetWidgetSyncRef.current = resetWidgetSync; + // Set playground active flag for widget renderers to read const setPlaygroundActive = useUIPlaygroundStore( (s) => s.setPlaygroundActive, @@ -391,8 +407,14 @@ export function PlaygroundMain({ const handleWidgetStateChange = useCallback( (toolCallId: string, state: unknown) => { onWidgetStateChange?.(toolCallId, state); + + if (status === "ready") { + void enqueueWidgetStateSync([{ toolCallId, state }]); + } else { + setWidgetStateQueue((prev) => [...prev, { toolCallId, state }]); + } }, - [onWidgetStateChange], + [onWidgetStateChange, status, enqueueWidgetStateSync], ); // Handle follow-up messages from widgets @@ -405,13 +427,7 @@ export function PlaygroundMain({ // Handle model context updates from widgets (SEP-1865 ui/update-model-context) const handleModelContextUpdate = useCallback( - ( - toolCallId: string, - context: { - content?: ContentBlock[]; - structuredContent?: Record; - }, - ) => { + (toolCallId: string, context: ModelContextItem["context"]) => { // Queue model context to be included in next message setModelContextQueue((prev) => { // Remove any existing context from same widget (overwrite pattern per SEP-1865) @@ -419,7 +435,7 @@ export function PlaygroundMain({ return [...filtered, { toolCallId, context }]; }); }, - [], + [setModelContextQueue], ); // Handle clear chat @@ -427,8 +443,9 @@ export function PlaygroundMain({ resetChat(); clearLogs(); setInjectedToolRenderOverrides({}); + resetWidgetSync(); setShowClearConfirm(false); - }, [resetChat, clearLogs]); + }, [resetChat, clearLogs, resetWidgetSync]); const mergedToolRenderOverrides = useMemo( () => ({ @@ -465,54 +482,51 @@ export function PlaygroundMain({ const hasContent = input.trim() || mcpPromptResults.length > 0 || fileAttachments.length > 0; if (hasContent && status === "ready" && !submitBlocked) { - if (displayMode === "fullscreen" && isWidgetFullscreen) { - setIsFullscreenChatOpen(true); - } - posthog.capture("app_builder_send_message", { - location: "app_builder_tab", - platform: detectPlatform(), - environment: detectEnvironment(), - model_id: selectedModel?.id ?? null, - model_name: selectedModel?.name ?? null, - model_provider: selectedModel?.provider ?? null, - }); + try { + // Ensure async widget-state -> message conversion has finished before submit. + await widgetStateSyncRef.current; - // Include any pending model context from widgets (SEP-1865 ui/update-model-context) - // Sent as "user" messages for compatibility with model provider APIs - const contextMessages = modelContextQueue.map( - ({ toolCallId, context }) => ({ - id: `model-context-${toolCallId}-${Date.now()}`, - role: "user" as const, - parts: [ - { - type: "text" as const, - text: `Widget ${toolCallId} context: ${JSON.stringify(context)}`, - }, - ], - metadata: { - source: "widget-model-context", - toolCallId, - }, - }), - ); + if (displayMode === "fullscreen" && isWidgetFullscreen) { + setIsFullscreenChatOpen(true); + } + posthog.capture("app_builder_send_message", { + location: "app_builder_tab", + platform: detectPlatform(), + environment: detectEnvironment(), + model_id: selectedModel?.id ?? null, + model_name: selectedModel?.name ?? null, + model_provider: selectedModel?.provider ?? null, + }); + + // Include any pending model context from widgets (SEP-1865 ui/update-model-context) + // Sent as hidden user messages; preserve image/audio blocks as file parts. + const contextMessages = await buildWidgetModelContextMessages( + modelContextQueueRef.current, + ); + + if (contextMessages.length > 0) { + setMessages((prev) => [...prev, ...contextMessages]); + } - if (contextMessages.length > 0) { - setMessages((prev) => [...prev, ...contextMessages]); + // Convert file attachments to FileUIPart[] format for the AI SDK + const files = + fileAttachments.length > 0 + ? await attachmentsToFileUIParts(fileAttachments) + : undefined; + + sendMessage({ text: input, files }); + setInput(""); + setMcpPromptResults([]); + // Revoke object URLs and clear file attachments + revokeFileAttachmentUrls(fileAttachments); + setFileAttachments([]); + setModelContextQueue([]); // Clear after sending + } catch (err) { + console.error("[PlaygroundMain] Submit failed:", err); + toast.error( + err instanceof Error ? err.message : "Failed to send message", + ); } - - // Convert file attachments to FileUIPart[] format for the AI SDK - const files = - fileAttachments.length > 0 - ? await attachmentsToFileUIParts(fileAttachments) - : undefined; - - sendMessage({ text: input, files }); - setInput(""); - setMcpPromptResults([]); - // Revoke object URLs and clear file attachments - revokeFileAttachmentUrls(fileAttachments); - setFileAttachments([]); - setModelContextQueue([]); // Clear after sending } }; diff --git a/mcpjam-inspector/client/src/hooks/__tests__/use-widget-state-sync.test.ts b/mcpjam-inspector/client/src/hooks/__tests__/use-widget-state-sync.test.ts new file mode 100644 index 000000000..403c0437f --- /dev/null +++ b/mcpjam-inspector/client/src/hooks/__tests__/use-widget-state-sync.test.ts @@ -0,0 +1,273 @@ +import { describe, it, expect, vi, beforeEach } from "vitest"; +import { renderHook, act } from "@testing-library/react"; +import { mcpApiPresets } from "@/test/mocks/mcp-api"; +import { storePresets } from "@/test/mocks/stores"; +import { + applyClientRuntimePresets, + clientRuntimeMocks, +} from "@/test/mocks/widget-state-sync"; + +import { useWidgetStateSync } from "../use-widget-state-sync"; +import type { UIMessage } from "ai"; + +describe("useWidgetStateSync", () => { + let messages: UIMessage[]; + let setMessages: (updater: (prev: UIMessage[]) => UIMessage[]) => void; + + beforeEach(() => { + vi.clearAllMocks(); + messages = []; + setMessages = vi.fn((updater) => { + messages = updater(messages); + }); + + applyClientRuntimePresets({ + mcpApi: mcpApiPresets.allSuccess(), + appState: storePresets.empty(), + buildWidgetStateParts: async (toolCallId: string, state: unknown) => [ + { + type: "text", + text: `widget ${toolCallId}: ${JSON.stringify(state)}`, + }, + ], + }); + }); + + describe("enqueueWidgetStateSync", () => { + it("appends a new widget-state message", async () => { + const { result } = renderHook(() => + useWidgetStateSync({ status: "ready", setMessages }), + ); + + await act(async () => { + await result.current.enqueueWidgetStateSync([ + { toolCallId: "tool-1", state: { count: 1 } }, + ]); + }); + + expect(messages).toHaveLength(1); + expect(messages[0].id).toBe("widget-state-tool-1"); + expect(messages[0].role).toBe("user"); + expect(messages[0].parts[0]).toEqual({ + type: "text", + text: 'widget tool-1: {"count":1}', + }); + }); + + it("updates an existing widget-state message when parts change", async () => { + const { result } = renderHook(() => + useWidgetStateSync({ status: "ready", setMessages }), + ); + + await act(async () => { + await result.current.enqueueWidgetStateSync([ + { toolCallId: "tool-1", state: { count: 1 } }, + ]); + }); + + expect(messages).toHaveLength(1); + + await act(async () => { + await result.current.enqueueWidgetStateSync([ + { toolCallId: "tool-1", state: { count: 2 } }, + ]); + }); + + // Should still be 1 message, updated in-place + expect(messages).toHaveLength(1); + expect(messages[0].parts[0]).toEqual({ + type: "text", + text: 'widget tool-1: {"count":2}', + }); + }); + + it("removes message when state is null", async () => { + const { result } = renderHook(() => + useWidgetStateSync({ status: "ready", setMessages }), + ); + + await act(async () => { + await result.current.enqueueWidgetStateSync([ + { toolCallId: "tool-1", state: { count: 1 } }, + ]); + }); + + expect(messages).toHaveLength(1); + + await act(async () => { + await result.current.enqueueWidgetStateSync([ + { toolCallId: "tool-1", state: null }, + ]); + }); + + expect(messages).toHaveLength(0); + }); + + it("skips update when parts are identical (dedup)", async () => { + const { result } = renderHook(() => + useWidgetStateSync({ status: "ready", setMessages }), + ); + + await act(async () => { + await result.current.enqueueWidgetStateSync([ + { toolCallId: "tool-1", state: { same: true } }, + ]); + }); + + const firstCallCount = (setMessages as ReturnType).mock + .calls.length; + + await act(async () => { + await result.current.enqueueWidgetStateSync([ + { toolCallId: "tool-1", state: { same: true } }, + ]); + }); + + // setMessages was called again but the updater should return same array + const secondUpdater = (setMessages as ReturnType).mock + .calls[firstCallCount][0]; + const before = [...messages]; + const after = secondUpdater(before); + // referential equality — updater returned the same array (no mutation needed) + expect(after).toBe(before); + }); + }); + + describe("queue flush on status change", () => { + it("flushes queued updates when status becomes ready", async () => { + const { result, rerender } = renderHook( + ({ status }) => useWidgetStateSync({ status, setMessages }), + { initialProps: { status: "streaming" } }, + ); + + // Queue updates while not ready + act(() => { + result.current.setWidgetStateQueue((prev) => [ + ...prev, + { toolCallId: "tool-q1", state: { queued: true } }, + ]); + }); + + expect(messages).toHaveLength(0); + + // Switch to ready — should trigger flush + await act(async () => { + rerender({ status: "ready" }); + // Allow the async enqueue to complete + await result.current.widgetStateSyncRef.current; + }); + + expect(messages).toHaveLength(1); + expect(messages[0].id).toBe("widget-state-tool-q1"); + }); + }); + + describe("setModelContextQueue", () => { + it("keeps modelContextQueueRef in sync with state", () => { + const { result } = renderHook(() => + useWidgetStateSync({ status: "ready", setMessages }), + ); + + const item = { + toolCallId: "tool-mc1", + context: { content: [{ type: "text" as const, text: "hello" }] }, + }; + + act(() => { + result.current.setModelContextQueue([item]); + }); + + expect(result.current.modelContextQueueRef.current).toEqual([item]); + }); + + it("accepts a function updater", () => { + const { result } = renderHook(() => + useWidgetStateSync({ status: "ready", setMessages }), + ); + + const item1 = { + toolCallId: "tool-mc1", + context: { content: [{ type: "text" as const, text: "first" }] }, + }; + const item2 = { + toolCallId: "tool-mc2", + context: { content: [{ type: "text" as const, text: "second" }] }, + }; + + act(() => { + result.current.setModelContextQueue([item1]); + }); + + act(() => { + result.current.setModelContextQueue((prev) => [...prev, item2]); + }); + + expect(result.current.modelContextQueueRef.current).toEqual([ + item1, + item2, + ]); + }); + }); + + describe("resetWidgetSync", () => { + it("clears model context queue and ref", () => { + const { result } = renderHook(() => + useWidgetStateSync({ status: "ready", setMessages }), + ); + + act(() => { + result.current.setModelContextQueue([ + { + toolCallId: "tool-mc1", + context: { content: [{ type: "text" as const, text: "data" }] }, + }, + ]); + }); + + expect(result.current.modelContextQueueRef.current).toHaveLength(1); + + act(() => { + result.current.resetWidgetSync(); + }); + + expect(result.current.modelContextQueueRef.current).toHaveLength(0); + }); + + it("cancels in-flight async updates via epoch increment", async () => { + // Create a deferred promise so resolveSlowParts is assigned immediately + let resolveSlowParts!: (value: UIMessage["parts"]) => void; + const slowPromise = new Promise((resolve) => { + resolveSlowParts = resolve; + }); + clientRuntimeMocks.buildWidgetStatePartsMock.mockReturnValueOnce( + slowPromise, + ); + + const { result } = renderHook(() => + useWidgetStateSync({ status: "ready", setMessages }), + ); + + // Start an async update + let enqueuePromise: Promise; + act(() => { + enqueuePromise = result.current.enqueueWidgetStateSync([ + { toolCallId: "tool-stale", state: { old: true } }, + ]); + }); + + // Reset before the promise resolves — bumps the epoch + act(() => { + result.current.resetWidgetSync(); + }); + + // Now resolve the slow parts — should be ignored due to epoch mismatch + await act(async () => { + resolveSlowParts([{ type: "text", text: "stale data" }]); + await enqueuePromise!; + }); + + // No messages should have been added — the epoch was stale + expect(messages).toHaveLength(0); + }); + }); +}); diff --git a/mcpjam-inspector/client/src/hooks/use-widget-state-sync.ts b/mcpjam-inspector/client/src/hooks/use-widget-state-sync.ts new file mode 100644 index 000000000..ebc72527f --- /dev/null +++ b/mcpjam-inspector/client/src/hooks/use-widget-state-sync.ts @@ -0,0 +1,214 @@ +import { useState, useRef, useCallback, useEffect } from "react"; +import type { UIMessage } from "ai"; +import type { ContentBlock } from "@modelcontextprotocol/sdk/types.js"; +import { buildWidgetStateParts } from "@/lib/mcp-ui/openai-widget-state-messages"; + +type Part = UIMessage["parts"][number]; + +/** Lightweight structural equality check for message parts. */ +function partsEqual(a: Part[], b: Part[]): boolean { + if (a.length !== b.length) return false; + for (let i = 0; i < a.length; i++) { + const pa = a[i]; + const pb = b[i]; + if (pa.type !== pb.type) return false; + switch (pa.type) { + case "text": + if (pb.type !== "text" || pa.text !== pb.text) return false; + break; + case "file": + if ( + pb.type !== "file" || + pa.mediaType !== pb.mediaType || + pa.url !== pb.url + ) + return false; + break; + default: + // For any other/future part types, fall back to JSON comparison. + if (JSON.stringify(pa) !== JSON.stringify(pb)) return false; + } + } + return true; +} + +export type ModelContextItem = { + toolCallId: string; + context: { + content?: ContentBlock[]; + structuredContent?: Record; + }; +}; + +interface UseWidgetStateSyncOptions { + status: string; + setMessages: (updater: (prev: UIMessage[]) => UIMessage[]) => void; +} + +interface UseWidgetStateSyncReturn { + /** Async-serialised enqueue: resolves file IDs then updates messages. */ + enqueueWidgetStateSync: ( + updates: { toolCallId: string; state: unknown }[], + ) => Promise; + /** Enqueue pending updates while chat is not yet ready. */ + setWidgetStateQueue: React.Dispatch< + React.SetStateAction<{ toolCallId: string; state: unknown }[]> + >; + /** Awaitable promise for the in-flight widget-state resolution. */ + widgetStateSyncRef: React.MutableRefObject>; + /** Ref of pending model-context items for use inside async submit handlers. */ + modelContextQueueRef: React.MutableRefObject; + /** + * Setter that keeps modelContextQueueRef in sync automatically. + * Use this instead of setModelContextQueue + manual ref update. + */ + setModelContextQueue: ( + action: + | ModelContextItem[] + | ((prev: ModelContextItem[]) => ModelContextItem[]), + ) => void; + /** Reset all queues and cancel in-flight async updates. */ + resetWidgetSync: () => void; +} + +export function useWidgetStateSync({ + status, + setMessages, +}: UseWidgetStateSyncOptions): UseWidgetStateSyncReturn { + const [widgetStateQueue, setWidgetStateQueue] = useState< + { toolCallId: string; state: unknown }[] + >([]); + const [, setModelContextQueueState] = useState([]); + const modelContextQueueRef = useRef([]); + const widgetStateSyncRef = useRef>(Promise.resolve()); + const widgetStateEpochRef = useRef(0); + + const applyWidgetStateUpdates = useCallback( + async ( + updates: { toolCallId: string; state: unknown }[], + epoch: number, + ) => { + const resolvedUpdates = await Promise.all( + updates.map(async ({ toolCallId, state }) => { + const messageId = `widget-state-${toolCallId}`; + if (state === null) { + return { messageId, nextMessage: null as null }; + } + + const parts = await buildWidgetStateParts(toolCallId, state); + return { + messageId, + nextMessage: { + id: messageId, + // "user" (not "assistant") is required: model provider APIs only + // accept image/file attachments inside user-role messages. + role: "user" as const, + parts, + }, + }; + }), + ); + + if (epoch !== widgetStateEpochRef.current) return; + + setMessages((prevMessages) => { + if (epoch !== widgetStateEpochRef.current) return prevMessages; + + let nextMessages = prevMessages; + + for (const { messageId, nextMessage } of resolvedUpdates) { + if (!nextMessage) { + nextMessages = nextMessages.filter((msg) => msg.id !== messageId); + continue; + } + + const existingIndex = nextMessages.findIndex( + (msg) => msg.id === messageId, + ); + if (existingIndex !== -1) { + const existingMessage = nextMessages[existingIndex]; + if (partsEqual(existingMessage.parts, nextMessage.parts)) { + continue; + } + + const updatedMessages = [...nextMessages]; + updatedMessages[existingIndex] = nextMessage; + nextMessages = updatedMessages; + continue; + } + + nextMessages = [...nextMessages, nextMessage]; + } + + return nextMessages; + }); + }, + [setMessages], + ); + + const enqueueWidgetStateSync = useCallback( + (updates: { toolCallId: string; state: unknown }[]) => { + const epoch = widgetStateEpochRef.current; + widgetStateSyncRef.current = widgetStateSyncRef.current + .catch(() => undefined) + .then(() => applyWidgetStateUpdates(updates, epoch)); + return widgetStateSyncRef.current; + }, + [applyWidgetStateUpdates], + ); + + useEffect(() => { + if (status !== "ready" || widgetStateQueue.length === 0) return; + + const queueToFlush = widgetStateQueue; + void enqueueWidgetStateSync(queueToFlush) + .then(() => { + setWidgetStateQueue((currentQueue) => { + if (currentQueue.length < queueToFlush.length) return currentQueue; + + const startsWithFlushedItems = queueToFlush.every( + (queuedItem, index) => currentQueue[index] === queuedItem, + ); + if (!startsWithFlushedItems) return currentQueue; + + return currentQueue.slice(queueToFlush.length); + }); + }) + .catch((error) => { + console.error("Failed to flush widget state queue", error); + }); + }, [status, widgetStateQueue, enqueueWidgetStateSync, setWidgetStateQueue]); + + /** Setter that keeps modelContextQueueRef in sync automatically. */ + const setModelContextQueue = useCallback( + ( + action: + | ModelContextItem[] + | ((prev: ModelContextItem[]) => ModelContextItem[]), + ) => { + setModelContextQueueState((prev) => { + const next = typeof action === "function" ? action(prev) : action; + modelContextQueueRef.current = next; + return next; + }); + }, + [], + ); + + const resetWidgetSync = useCallback(() => { + setModelContextQueueState([]); + modelContextQueueRef.current = []; + setWidgetStateQueue([]); + widgetStateEpochRef.current += 1; + widgetStateSyncRef.current = Promise.resolve(); + }, []); + + return { + enqueueWidgetStateSync, + setWidgetStateQueue, + widgetStateSyncRef, + modelContextQueueRef, + setModelContextQueue, + resetWidgetSync, + }; +} diff --git a/mcpjam-inspector/client/src/lib/mcp-ui/__tests__/model-context-messages.test.ts b/mcpjam-inspector/client/src/lib/mcp-ui/__tests__/model-context-messages.test.ts new file mode 100644 index 000000000..ee4c8d0ad --- /dev/null +++ b/mcpjam-inspector/client/src/lib/mcp-ui/__tests__/model-context-messages.test.ts @@ -0,0 +1,226 @@ +import { describe, expect, it, vi, beforeEach } from "vitest"; +import { mcpApiPresets } from "@/test/mocks/mcp-api"; +import { storePresets } from "@/test/mocks/stores"; +import { + applyClientRuntimePresets, + clientRuntimeMocks, +} from "@/test/mocks/client-runtime"; + +vi.mock("@/hooks/use-app-state", () => ({ + useAppState: clientRuntimeMocks.useAppStateMock, +})); + +vi.mock("@/state/mcp-api", () => clientRuntimeMocks.mcpApiMock); + +import { buildWidgetModelContextMessages } from "../model-context-messages"; +import * as widgetStateMessages from "../openai-widget-state-messages"; + +const resolveFilePartSpy = vi.spyOn(widgetStateMessages, "resolveFilePart"); + +beforeEach(() => { + vi.clearAllMocks(); + applyClientRuntimePresets({ + mcpApi: mcpApiPresets.allSuccess(), + appState: storePresets.empty(), + }); + resolveFilePartSpy.mockResolvedValue(null); +}); + +describe("buildWidgetModelContextMessages", () => { + it("preserves text and image content blocks as text + file parts", async () => { + const messages = await buildWidgetModelContextMessages([ + { + toolCallId: "tool-1", + context: { + content: [ + { type: "text", text: "Selected image" }, + { type: "image", data: "aGVsbG8=", mimeType: "image/jpeg" }, + ], + }, + }, + ]); + + expect(messages).toHaveLength(1); + expect(messages[0].parts).toEqual([ + { type: "text", text: "Selected image" }, + { + type: "file", + mediaType: "image/jpeg", + url: "data:image/jpeg;base64,aGVsbG8=", + }, + ]); + }); + + it("keeps existing data URLs unchanged", async () => { + const dataUrl = "data:image/png;base64,aGVsbG8="; + const messages = await buildWidgetModelContextMessages([ + { + toolCallId: "tool-2", + context: { + content: [{ type: "image", data: dataUrl, mimeType: "image/png" }], + }, + }, + ]); + + expect(messages[0].parts).toEqual([ + { + type: "file", + mediaType: "image/png", + url: dataUrl, + }, + ]); + }); + + it("falls back to structured content text when no supported content blocks exist", async () => { + const messages = await buildWidgetModelContextMessages([ + { + toolCallId: "tool-3", + context: { + structuredContent: { selectedIds: [1, 2, 3] }, + }, + }, + ]); + + expect(messages[0].parts).toEqual([ + { + type: "text", + text: 'Widget tool-3 structured context: {"selectedIds":[1,2,3]}', + }, + ]); + }); + + it("strips privateContent from structured content fallback text", async () => { + const messages = await buildWidgetModelContextMessages([ + { + toolCallId: "tool-priv", + context: { + structuredContent: { + modelContent: "visible summary", + privateContent: { secret: "should not appear" }, + imageIds: [], + }, + }, + }, + ]); + + const text = (messages[0].parts[0] as { type: "text"; text: string }).text; + expect(text).toContain("visible summary"); + expect(text).not.toContain("privateContent"); + expect(text).not.toContain("should not appear"); + }); + + it("emits no text part when structuredContent contains only privateContent", async () => { + const messages = await buildWidgetModelContextMessages([ + { + toolCallId: "tool-priv-only", + context: { + structuredContent: { + privateContent: { secret: "should not appear" }, + }, + }, + }, + ]); + + expect(messages).toHaveLength(0); + }); + + it("appends image URL from structuredContent when content[] provides only text (native MCP Apps path)", async () => { + // Native MCP Apps send text metadata in content[] but image URLs only in + // structuredContent.privateContent.selectedImages. The old code returned + // early once content[] produced parts, silently dropping the image. + const imageUrl = "https://example.com/search-result.jpg"; + const messages = await buildWidgetModelContextMessages([ + { + toolCallId: "tool-5", + context: { + content: [{ type: "text", text: "1: Title: Cat photo" }], + structuredContent: { + modelContent: "1: Title: Cat photo", + privateContent: { + selectedImages: [{ imageUrl }], + }, + }, + }, + }, + ]); + + const parts = messages[0].parts; + expect(parts).toHaveLength(2); + expect(parts[0]).toEqual({ type: "text", text: "1: Title: Cat photo" }); + expect(parts[1]).toEqual({ + type: "file", + mediaType: "image/jpeg", + url: imageUrl, + }); + }); + + it("resolves image URL from structuredContent.privateContent.selectedImages when file fetch fails", async () => { + const imageUrl = "https://example.com/cat.jpg"; + const messages = await buildWidgetModelContextMessages([ + { + toolCallId: "tool-4", + context: { + structuredContent: { + modelContent: "1: Title: Cat", + privateContent: { + selectedImages: [{ imageUrl }], + rawFileIds: ["file_abc123"], + }, + imageIds: ["file_abc123"], + }, + }, + }, + ]); + + // resolveFilePart is mocked to return null, so we fall back to URL + const parts = messages[0].parts; + expect(parts).toHaveLength(2); + expect(parts[0]).toEqual({ + type: "text", + text: expect.stringContaining("tool-4"), + }); + expect(parts[1]).toEqual({ + type: "file", + mediaType: "image/jpeg", + url: imageUrl, + }); + }); + + it("skips fallback image URLs when at least one file ID resolves (avoids duplicates)", async () => { + const resolvedPart = { + type: "file" as const, + mediaType: "image/png", + url: "data:image/png;base64,cmVzb2x2ZWQ=", + }; + vi.mocked(widgetStateMessages.resolveFilePart) + .mockResolvedValueOnce(resolvedPart) + .mockResolvedValueOnce(null); + + const messages = await buildWidgetModelContextMessages([ + { + toolCallId: "tool-6", + context: { + structuredContent: { + imageIds: [ + "file_550e8400-e29b-41d4-a716-446655440000", + "file_550e8400-e29b-41d4-a716-446655440001", + ], + privateContent: { + selectedImages: [ + { imageUrl: "https://example.com/cat-1.jpg" }, + { imageUrl: "https://example.com/cat-2.jpg" }, + ], + }, + }, + }, + }, + ]); + + const parts = messages[0].parts; + // Only the resolved file part should be present — no fallback HTTP URLs + // which would duplicate the already-resolved image. + const fileParts = parts.filter((p) => p.type === "file"); + expect(fileParts).toHaveLength(1); + expect(fileParts[0]).toEqual(resolvedPart); + }); +}); diff --git a/mcpjam-inspector/client/src/lib/mcp-ui/__tests__/openai-widget-state-messages.hosted.test.ts b/mcpjam-inspector/client/src/lib/mcp-ui/__tests__/openai-widget-state-messages.hosted.test.ts new file mode 100644 index 000000000..9112e93d9 --- /dev/null +++ b/mcpjam-inspector/client/src/lib/mcp-ui/__tests__/openai-widget-state-messages.hosted.test.ts @@ -0,0 +1,64 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { mcpApiPresets } from "@/test/mocks/mcp-api"; +import { storePresets } from "@/test/mocks/stores"; +import { + applyClientRuntimePresets, + clientRuntimeMocks, +} from "@/test/mocks/client-runtime"; +import { resolveFilePart } from "../openai-widget-state-messages"; + +vi.mock("@/lib/session-token", () => ({ + authFetch: clientRuntimeMocks.authFetchMock, +})); + +vi.mock("@/lib/config", () => ({ + get HOSTED_MODE() { + return clientRuntimeMocks.hostedMode; + }, +})); + +vi.mock("@/hooks/use-app-state", () => ({ + useAppState: clientRuntimeMocks.useAppStateMock, +})); + +vi.mock("@/state/mcp-api", () => clientRuntimeMocks.mcpApiMock); + +describe("resolveFilePart (hosted mode)", () => { + beforeEach(() => { + vi.clearAllMocks(); + applyClientRuntimePresets({ + hostedMode: true, + mcpApi: mcpApiPresets.allSuccess(), + appState: storePresets.empty(), + }); + }); + + it("tries hosted web fallback when the primary endpoint throws", async () => { + const fileId = "file_550e8400-e29b-41d4-a716-446655440000"; + clientRuntimeMocks.authFetchMock + .mockRejectedValueOnce(new Error("network failure")) + .mockResolvedValueOnce({ + ok: true, + blob: async () => new Blob(["fallback"], { type: "image/png" }), + }); + + const part = await resolveFilePart(fileId); + + expect(clientRuntimeMocks.authFetchMock).toHaveBeenCalledTimes(2); + expect(clientRuntimeMocks.authFetchMock).toHaveBeenNthCalledWith( + 1, + `/api/apps/chatgpt-apps/file/${fileId}`, + ); + expect(clientRuntimeMocks.authFetchMock).toHaveBeenNthCalledWith( + 2, + `/api/web/apps/chatgpt-apps/file/${fileId}`, + ); + expect(part).toMatchObject({ + type: "file", + mediaType: "image/png", + }); + expect((part as { url: string }).url.startsWith("data:image/png;")).toBe( + true, + ); + }); +}); diff --git a/mcpjam-inspector/client/src/lib/mcp-ui/__tests__/openai-widget-state-messages.test.ts b/mcpjam-inspector/client/src/lib/mcp-ui/__tests__/openai-widget-state-messages.test.ts new file mode 100644 index 000000000..c2d895a40 --- /dev/null +++ b/mcpjam-inspector/client/src/lib/mcp-ui/__tests__/openai-widget-state-messages.test.ts @@ -0,0 +1,214 @@ +import { describe, expect, it, vi, beforeEach } from "vitest"; +import { mcpApiPresets } from "@/test/mocks/mcp-api"; +import { storePresets } from "@/test/mocks/stores"; +import { + applyClientRuntimePresets, + clientRuntimeMocks, +} from "@/test/mocks/client-runtime"; + +vi.mock("@/lib/session-token", () => ({ + authFetch: clientRuntimeMocks.authFetchMock, +})); + +vi.mock("@/lib/config", () => ({ + get HOSTED_MODE() { + return clientRuntimeMocks.hostedMode; + }, +})); + +vi.mock("@/hooks/use-app-state", () => ({ + useAppState: clientRuntimeMocks.useAppStateMock, +})); + +vi.mock("@/state/mcp-api", () => clientRuntimeMocks.mcpApiMock); + +import { + buildWidgetStateParts, + resolveFilePart, +} from "../openai-widget-state-messages"; + +function applyDefaultRuntimePresets() { + vi.clearAllMocks(); + applyClientRuntimePresets({ + hostedMode: false, + mcpApi: mcpApiPresets.allSuccess(), + appState: storePresets.empty(), + }); +} + +describe("buildWidgetStateParts", () => { + beforeEach(() => { + applyDefaultRuntimePresets(); + }); + + it("returns text-only parts when there are no uploaded file ids", async () => { + const state = { foo: "bar" }; + const parts = await buildWidgetStateParts("tool-1", state); + + expect(parts).toEqual([ + { + type: "text", + text: 'The state of widget tool-1 is: {"foo":"bar"}', + }, + ]); + expect(clientRuntimeMocks.authFetchMock).not.toHaveBeenCalled(); + }); + + it("resolves file ids from imageIds into file parts", async () => { + clientRuntimeMocks.authFetchMock.mockResolvedValue({ + ok: true, + blob: async () => new Blob(["hello"], { type: "image/jpeg" }), + }); + + // imageIds is the canonical source; privateContent is UI-only and not read. + const state = { + imageIds: ["file_550e8400-e29b-41d4-a716-446655440000"], + privateContent: { currentView: "image-viewer" }, + }; + + const parts = await buildWidgetStateParts("tool-2", state); + + expect(clientRuntimeMocks.authFetchMock).toHaveBeenCalledTimes(1); + expect(parts[0]).toEqual({ + type: "text", + text: 'The state of widget tool-2 is: {"imageIds":["file_550e8400-e29b-41d4-a716-446655440000"]}', + }); + expect(parts[1]).toMatchObject({ + type: "file", + mediaType: "image/jpeg", + }); + expect( + (parts[1] as { url: string }).url.startsWith("data:image/jpeg;"), + ).toBe(true); + }); + + it("strips privateContent when modelContent is missing", async () => { + const state = { + privateContent: { + secret: "should-not-leak", + }, + imageIds: [], + selectedTab: "results", + }; + + const parts = await buildWidgetStateParts("tool-2b", state); + + expect(clientRuntimeMocks.authFetchMock).not.toHaveBeenCalled(); + expect(parts).toEqual([ + { + type: "text", + text: 'The state of widget tool-2b is: {"imageIds":[],"selectedTab":"results"}', + }, + ]); + expect((parts[0] as { text: string }).text).not.toContain("privateContent"); + expect((parts[0] as { text: string }).text).not.toContain( + "should-not-leak", + ); + }); + + it("uses modelContent as text and does not expose privateContent", async () => { + const state = { + modelContent: "User uploaded an image from the file upload widget.", + privateContent: { + rawFileId: "file_8a45d63b-79d4-4eff-a178-c0ce4078c7c6", + }, + imageIds: [], + }; + + const parts = await buildWidgetStateParts("tool-3", state); + + expect(clientRuntimeMocks.authFetchMock).not.toHaveBeenCalled(); + expect(parts).toEqual([ + { + type: "text", + text: "User uploaded an image from the file upload widget.", + }, + ]); + }); + + it("handles undefined modelContent without producing an invalid text part", async () => { + const state = { + modelContent: undefined, + privateContent: { + rawFileId: "file_8a45d63b-79d4-4eff-a178-c0ce4078c7c6", + }, + imageIds: [], + }; + + const parts = await buildWidgetStateParts("tool-3b", state); + + expect(clientRuntimeMocks.authFetchMock).not.toHaveBeenCalled(); + expect(parts).toEqual([ + { + type: "text", + text: 'The state of widget tool-3b is: {"imageIds":[]}', + }, + ]); + }); + + it("resolves imageIds into file parts alongside modelContent text", async () => { + clientRuntimeMocks.authFetchMock.mockResolvedValue({ + ok: true, + blob: async () => new Blob(["img"], { type: "image/jpeg" }), + }); + + const state = { + modelContent: "User uploaded an image from the file upload widget.", + privateContent: { + rawFileId: "file_8a45d63b-79d4-4eff-a178-c0ce4078c7c6", + }, + imageIds: ["file_8a45d63b-79d4-4eff-a178-c0ce4078c7c6"], + }; + + const parts = await buildWidgetStateParts("tool-4", state); + + expect(clientRuntimeMocks.authFetchMock).toHaveBeenCalledTimes(1); + expect(parts[0]).toEqual({ + type: "text", + text: "User uploaded an image from the file upload widget.", + }); + expect(parts[1]).toMatchObject({ type: "file", mediaType: "image/jpeg" }); + expect( + (parts[1] as { url: string }).url.startsWith("data:image/jpeg;"), + ).toBe(true); + }); + + it("omits file parts when imageId resolution fails", async () => { + clientRuntimeMocks.authFetchMock.mockResolvedValue({ ok: false }); + + const state = { + modelContent: "User uploaded an image from the file upload widget.", + privateContent: { + rawFileId: "file_8a45d63b-79d4-4eff-a178-c0ce4078c7c6", + }, + imageIds: ["file_8a45d63b-79d4-4eff-a178-c0ce4078c7c6"], + }; + + const parts = await buildWidgetStateParts("tool-5", state); + + expect(clientRuntimeMocks.authFetchMock).toHaveBeenCalledTimes(1); + expect(parts).toEqual([ + { + type: "text", + text: "User uploaded an image from the file upload widget.", + }, + ]); + }); +}); + +describe("resolveFilePart", () => { + beforeEach(() => { + applyDefaultRuntimePresets(); + }); + + it("returns null when all endpoint requests throw", async () => { + clientRuntimeMocks.authFetchMock.mockRejectedValue( + new Error("network failure"), + ); + + await expect( + resolveFilePart("file_550e8400-e29b-41d4-a716-446655440000"), + ).resolves.toBeNull(); + expect(clientRuntimeMocks.authFetchMock).toHaveBeenCalledTimes(1); + }); +}); diff --git a/mcpjam-inspector/client/src/lib/mcp-ui/model-context-messages.ts b/mcpjam-inspector/client/src/lib/mcp-ui/model-context-messages.ts new file mode 100644 index 000000000..b97229ae8 --- /dev/null +++ b/mcpjam-inspector/client/src/lib/mcp-ui/model-context-messages.ts @@ -0,0 +1,172 @@ +import type { UIMessage } from "ai"; +import type { ContentBlock } from "@modelcontextprotocol/sdk/types.js"; +import { + extractUploadedFileIds, + extractImageUrls, + resolveFilePart, + guessImageMediaType, + safeStringify, +} from "./openai-widget-state-messages"; + +export interface WidgetModelContext { + content?: ContentBlock[]; + structuredContent?: Record; +} + +export interface WidgetModelContextQueueItem { + toolCallId: string; + context: WidgetModelContext; +} + +function toDataUrl(data: string, mimeType: string): string { + const trimmed = data.trim(); + if (trimmed.startsWith("data:")) return trimmed; + return `data:${mimeType};base64,${trimmed}`; +} + +function contextToSyncParts(context: WidgetModelContext): UIMessage["parts"] { + const parts: UIMessage["parts"] = []; + + for (const block of context.content ?? []) { + switch (block.type) { + case "text": + if (block.text.trim()) { + parts.push({ type: "text", text: block.text }); + } + break; + case "image": + if (block.data && block.mimeType) { + parts.push({ + type: "file", + mediaType: block.mimeType, + url: toDataUrl(block.data, block.mimeType), + }); + } + break; + case "audio": + if (block.data && block.mimeType) { + parts.push({ + type: "file", + mediaType: block.mimeType, + url: toDataUrl(block.data, block.mimeType), + }); + } + break; + default: + break; + } + } + + return parts; +} + +/** + * Build parts for a single widget context item. + * + * Handles two paths: + * 1. Native MCP Apps: sends `content` (ContentBlock[] with text/image blocks). + * Image data is inline as base64; no network fetch is required. + * 2. ChatGPT extension widgets that call `setWidgetState` with the structured + * shape recommended by the Apps SDK — `{ modelContent, privateContent, imageIds }`. + * The host forwards this payload as `structuredContent`. File IDs in `imageIds` + * must be fetched from the local file endpoint before they can be attached. + * See: https://developers.openai.com/apps-sdk/build/state-management#image-ids-in-widget-state-model-visible-images-chatgpt-extension + */ +async function contextToParts( + toolCallId: string, + context: WidgetModelContext, +): Promise { + // First, convert any typed ContentBlocks (text/image/audio) from content[]. + const parts = contextToSyncParts(context); + + // Handle the structured widget state shape from ChatGPT extension widgets + // ({ modelContent, privateContent, imageIds }). The host forwards this as + // structuredContent; file IDs in imageIds need to be fetched. + if (context.structuredContent !== undefined) { + // Only add a text summary when content[] produced nothing (ChatGPT extension path). + // Strip privateContent — it is UI-only state that must not be sent to the model. + if (parts.length === 0) { + const { privateContent: _, ...modelVisible } = context.structuredContent; + if (Object.keys(modelVisible).length > 0) { + parts.push({ + type: "text", + text: `Widget ${toolCallId} structured context: ${safeStringify(modelVisible)}`, + }); + } + } + + // Attempt to resolve uploaded file IDs embedded in structuredContent. + const fileIds = extractUploadedFileIds(context.structuredContent); + let useImageUrlsFallback = false; + + if (fileIds.length > 0) { + const resolved = await Promise.all( + fileIds.map(async (fileId) => { + try { + return await resolveFilePart(fileId); + } catch { + return null; + } + }), + ); + const resolvedParts = resolved.filter( + (part): part is NonNullable => part !== null, + ); + for (const filePart of resolvedParts) { + parts.push(filePart); + } + + // Only fall back to image URLs when the file endpoint was completely + // unavailable (no IDs resolved). When some resolve, adding HTTP URLs + // from selectedImages would duplicate already-resolved images since + // data URLs and HTTP URLs can't be deduped against each other. + if (resolvedParts.length === 0) { + useImageUrlsFallback = true; + } + } else { + // No file IDs at all — try image URLs directly. + useImageUrlsFallback = true; + } + + if (useImageUrlsFallback) { + for (const imageUrl of extractImageUrls(context.structuredContent)) { + parts.push({ + type: "file", + mediaType: guessImageMediaType(imageUrl), + url: imageUrl, + }); + } + } + + return parts; + } + + // Last resort: if still nothing, serialize the entire context. + if (parts.length === 0) { + parts.push({ + type: "text", + text: `Widget ${toolCallId} context: ${safeStringify(context)}`, + }); + } + return parts; +} + +export async function buildWidgetModelContextMessages( + queue: WidgetModelContextQueueItem[], +): Promise { + const now = Date.now(); + const messages = await Promise.all( + queue.map(async ({ toolCallId, context }, index) => ({ + id: `model-context-${toolCallId}-${now}-${index}`, + role: "user" as const, + parts: await contextToParts(toolCallId, context), + metadata: { + source: "widget-model-context", + toolCallId, + }, + })), + ); + // Skip queue items whose resolved parts are empty (e.g. structuredContent + // with only privateContent) to avoid emitting invalid user messages. + return messages.filter((msg) => msg.parts.length > 0); +} diff --git a/mcpjam-inspector/client/src/lib/mcp-ui/openai-widget-state-messages.ts b/mcpjam-inspector/client/src/lib/mcp-ui/openai-widget-state-messages.ts new file mode 100644 index 000000000..09073ed55 --- /dev/null +++ b/mcpjam-inspector/client/src/lib/mcp-ui/openai-widget-state-messages.ts @@ -0,0 +1,211 @@ +import type { UIMessage } from "ai"; +import { authFetch } from "@/lib/session-token"; +import { HOSTED_MODE } from "@/lib/config"; + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +export function safeStringify(value: unknown): string { + try { + return JSON.stringify(value) ?? String(value); + } catch { + return String(value); + } +} + +function isValidUploadedFileId(value: unknown): value is string { + return typeof value === "string" && value.length > 0; +} + +function toStringArray(value: unknown): string[] { + return Array.isArray(value) + ? value.filter((item): item is string => typeof item === "string") + : []; +} + +function isHttpUrl(value: unknown): value is string { + if (typeof value !== "string") return false; + return value.startsWith("http://") || value.startsWith("https://"); +} + +export function extractImageUrls(state: unknown): string[] { + if (!isRecord(state)) return []; + const urls = new Set(); + + const addFromSelectedImages = (value: unknown) => { + if (!Array.isArray(value)) return; + for (const item of value) { + if (!isRecord(item)) continue; + if (isHttpUrl(item.imageUrl)) { + urls.add(item.imageUrl); + } + } + }; + + addFromSelectedImages(state.selectedImages); + + const modelContent = isRecord(state.modelContent) ? state.modelContent : null; + if (modelContent) { + addFromSelectedImages(modelContent.selectedImages); + } + + // Search-result widgets store selected image metadata under privateContent + const privateContent = isRecord(state.privateContent) + ? state.privateContent + : null; + if (privateContent) { + addFromSelectedImages(privateContent.selectedImages); + } + + return [...urls]; +} + +export function guessImageMediaType(url: string): string { + // Extract file extension from the end of the URL pathname to avoid false + // matches from extensions that appear mid-path (e.g. /icons.png-archive/cat.webp). + let pathname = url; + try { + pathname = new URL(url).pathname; + } catch { + // Not a valid URL — fall through and match against the raw string. + } + + const extMatch = pathname.match(/\.([a-z0-9]+)$/i); + if (extMatch) { + const ext = extMatch[1].toLowerCase(); + const mimeMap: Record = { + png: "image/png", + webp: "image/webp", + gif: "image/gif", + avif: "image/avif", + bmp: "image/bmp", + tiff: "image/tiff", + tif: "image/tiff", + jpg: "image/jpeg", + jpeg: "image/jpeg", + }; + if (mimeMap[ext]) return mimeMap[ext]; + } + + return "image/*"; +} + +export function extractUploadedFileIds(state: unknown): string[] { + if (!isRecord(state)) return []; + + const ids = new Set(); + const addId = (candidate: unknown) => { + if (isValidUploadedFileId(candidate)) ids.add(candidate); + }; + + // state.imageIds is the canonical source for uploaded file IDs per the Apps SDK spec. + // privateContent is UI-only state the model must not see, so we never read from it. + // See: https://developers.openai.com/apps-sdk/build/state-management#image-ids-in-widget-state-model-visible-images-chatgpt-extension + for (const item of toStringArray(state.imageIds)) addId(item); + + return [...ids]; +} + +function getFileEndpoints(fileId: string): string[] { + const encoded = encodeURIComponent(fileId); + + // ChatGPT widget uploads are handled by the local app runtime endpoint + // even when running in hosted mode. Keep web endpoint as a fallback. + const endpoints = [`/api/apps/chatgpt-apps/file/${encoded}`]; + if (HOSTED_MODE) { + endpoints.push(`/api/web/apps/chatgpt-apps/file/${encoded}`); + } + return endpoints; +} + +function blobToDataUrl(blob: Blob): Promise { + return new Promise((resolve, reject) => { + const reader = new FileReader(); + reader.onload = () => { + if (typeof reader.result === "string") { + resolve(reader.result); + } else { + reject(new Error("Failed to convert blob to data URL")); + } + }; + reader.onerror = () => reject(reader.error); + reader.readAsDataURL(blob); + }); +} + +export async function resolveFilePart( + fileId: string, +): Promise | null> { + for (const endpoint of getFileEndpoints(fileId)) { + try { + const response = await authFetch(endpoint); + if (!response.ok) continue; + + const blob = await response.blob(); + const dataUrl = await blobToDataUrl(blob); + + return { + type: "file", + mediaType: blob.type || "application/octet-stream", + url: dataUrl, + }; + } catch { + continue; + } + } + + return null; +} + +export function buildWidgetStateText( + toolCallId: string, + state: unknown, +): string { + // Per the Apps SDK structured widget state shape, modelContent is the + // model-visible portion; privateContent is UI-only and must not be exposed. + // See: https://developers.openai.com/apps-sdk/build/state-management#image-ids-in-widget-state-model-visible-images-chatgpt-extension + if (isRecord(state) && "modelContent" in state) { + const { modelContent } = state; + if ( + typeof modelContent === "string" && + modelContent !== "undefined" && + modelContent !== "null" + ) { + return modelContent; + } + } + + if (isRecord(state)) { + // Even without modelContent, privateContent remains UI-only and must not + // be serialized into model-visible text. + const { privateContent: _, ...modelVisible } = state; + const payload = Object.keys(modelVisible).length > 0 ? modelVisible : {}; + return `The state of widget ${toolCallId} is: ${safeStringify(payload)}`; + } + + return `The state of widget ${toolCallId} is: ${safeStringify(state)}`; +} + +export async function buildWidgetStateParts( + toolCallId: string, + state: unknown, +): Promise { + const parts: UIMessage["parts"] = [ + { type: "text", text: buildWidgetStateText(toolCallId, state) }, + ]; + + const fileParts: Array | null> = []; + for (const fileId of extractUploadedFileIds(state)) { + fileParts.push(await resolveFilePart(fileId)); + } + + for (const filePart of fileParts) { + if (filePart) parts.push(filePart); + } + + return parts; +} diff --git a/mcpjam-inspector/client/src/test/mocks/client-runtime.ts b/mcpjam-inspector/client/src/test/mocks/client-runtime.ts new file mode 100644 index 000000000..61f619620 --- /dev/null +++ b/mcpjam-inspector/client/src/test/mocks/client-runtime.ts @@ -0,0 +1,50 @@ +import type { UIMessage } from "ai"; +import { vi } from "vitest"; +import { mcpApiPresets } from "./mcp-api"; +import { storePresets } from "./stores"; + +type WidgetStatePartsBuilder = ( + toolCallId: string, + state: unknown, +) => Promise; + +export const clientRuntimeMocks = { + authFetchMock: vi.fn(), + useAppStateMock: vi.fn(), + mcpApiMock: {} as Record, + buildWidgetStatePartsMock: vi.fn(), + hostedMode: false, +}; + +interface ApplyClientRuntimePresetsOptions { + mcpApi?: Record; + appState?: ReturnType; + hostedMode?: boolean; + buildWidgetStateParts?: WidgetStatePartsBuilder; +} + +export function applyClientRuntimePresets( + options: ApplyClientRuntimePresetsOptions = {}, +): void { + const { + mcpApi = mcpApiPresets.allSuccess(), + appState = storePresets.empty(), + hostedMode = false, + buildWidgetStateParts, + } = options; + + for (const key of Object.keys(clientRuntimeMocks.mcpApiMock)) { + delete clientRuntimeMocks.mcpApiMock[key]; + } + Object.assign(clientRuntimeMocks.mcpApiMock, mcpApi); + clientRuntimeMocks.useAppStateMock.mockReturnValue(appState); + clientRuntimeMocks.hostedMode = hostedMode; + clientRuntimeMocks.authFetchMock.mockReset(); + clientRuntimeMocks.buildWidgetStatePartsMock.mockReset(); + + if (buildWidgetStateParts) { + clientRuntimeMocks.buildWidgetStatePartsMock.mockImplementation( + buildWidgetStateParts, + ); + } +} diff --git a/mcpjam-inspector/client/src/test/mocks/widget-state-sync.ts b/mcpjam-inspector/client/src/test/mocks/widget-state-sync.ts new file mode 100644 index 000000000..fc6e4329a --- /dev/null +++ b/mcpjam-inspector/client/src/test/mocks/widget-state-sync.ts @@ -0,0 +1,11 @@ +import { vi } from "vitest"; +import { clientRuntimeMocks } from "./client-runtime"; + +vi.mock("@/lib/mcp-ui/openai-widget-state-messages", () => ({ + buildWidgetStateParts: clientRuntimeMocks.buildWidgetStatePartsMock, +})); + +export { + applyClientRuntimePresets, + clientRuntimeMocks, +} from "./client-runtime";