feat: Add support for multi-modal messages

Author: MikeRyanDev

CHANGELOG.md

@@ -0,0 +1,43 @@
+# Changelog
+
+## v0.0.40 (Unreleased)
+
+### Highlights
+- Added end-to-end multimodal user message support across TypeScript and Python SDKs, LangGraph integrations, and docs, including new `InputContent` schemas and example agents.
+- Introduced runtime improvements such as `connectAgent`, safer cloning, and run lifecycle fixes, plus expanded event metadata (`parentRunId`, embedded inputs).
+- Delivered a new `compactEvents` utility for consolidating streaming deltas and hardened backwards compatibility layers for legacy clients.
+- Updated integrations (LangGraph, Vercel AI SDK, Mastra, Google ADK) to translate multimodal content and align with the latest protocol expectations.
+- Bumped package versions to the `0.0.40` pre-release series (TypeScript) and `0.2.0a0` (Python) to ship these capabilities.
+
+### TypeScript SDK
+- Extended `UserMessage` to accept multimodal `InputContent[]`, added `TextInputContent`/`BinaryInputContent` schemas, and exported the associated types for consumers.
+- Added optional `parentRunId` and embedded `input` payloads to `RunAgentInput` and `RunStartedEvent` schemas, plus surfaced a dedicated `AGUIConnectNotImplementedError`.
+- Refined `AbstractAgent` by making `run` protected, introducing `connectAgent`, tracking `isRunning`, and ensuring `clone` copies agent state; adjusted `HttpAgent`, Mastra, and Vercel agents to clone safely.
+- Updated `defaultApplyEvents` to handle non-string message content, merge `runStarted.input.messages` into local state, and improved event processing tests (`run-started-input`, cloning, multimodal, backwards compatibility).
+- Added and exported a `compactEvents` helper (with comprehensive tests) for consolidating streaming text/tool call deltas before replaying them to subscribers.
+- Improved legacy converters and integrations to flatten multimodal content when talking to text-only surfaces, guarding the behavior with new backwards compatibility tests.
+- Updated the React SDK chat surface to render multimodal user messages, including inline attachment previews and safer clipboard handling when no text is present.
+- Incremented package versions for `@ag-ui/core`, `@ag-ui/client`, CLI, encoder, and proto packages to `0.0.40-alpha.6`.
+
+### Python SDK
+- Introduced `TextInputContent` and `BinaryInputContent` models, allowing `UserMessage` instances to carry ordered multimodal content alongside traditional strings.
+- Added optional `parent_run_id` and `input` fields to `RunStartedEvent` plus `parent_run_id` to `RunAgentInput`, mirroring TypeScript schema changes.
+- Relaxed the base model configuration to allow extra fields for backwards compatibility and added validation to ensure binary payloads provide an ID, URL, or data source.
+- Expanded the test suite to cover multimodal serialization, binary payload validation, run input parsing, and extra-field tolerance.
+- Documented multimodal usage in the Python README and bumped the `ag-ui-protocol` package to `0.2.0a0`.
+
+### Integrations
+- **LangGraph (Python & TypeScript):** Added bidirectional converters for multimodal content, new vision-friendly example agents, updated tests, and bumped the package to `0.0.18a0` with matching dependency pins.
+- **Google ADK Middleware:** Flattened multimodal message content into text parts when translating to ADK events and updated helper utilities accordingly.
+- **Mastra:** Preserved constructor config when cloning agents and flattened AG-UI messages into the formats expected by Mastra clients.
+- **Vercel AI SDK:** Added safe cloning, converted user content into SDK-compatible parts, and ensured multimodal inputs degrade gracefully to text.
+- **General:** Updated converters and utilities across integrations to handle `InputContent` arrays without breaking existing text-only flows.
+
+### Documentation
+- Updated core concepts and SDK reference docs to describe multimodal user messages, the new input content schemas, and extended `RunStartedEvent` properties.
+- Documented `connectAgent`, `connect`, and the replayable `events$` stream on `AbstractAgent`, clarifying how persistent connections are implemented.
+- Marked the multimodal messages specification as implemented (October 16, 2025) and added README snippets showing multimodal message creation in both SDKs.
+
+### Miscellaneous
+- Added an explicit `@ts-expect-error` in the A2A middleware noting the intentional call to a protected method until a public API exists.
+- Updated Poetry lockfiles and dependency pins to align with the new Python package versions.

apps/angular/demo-server/package.json

@@ -8,7 +8,7 @@
     "start": "node --env-file=.env --loader tsx src/index.ts"
   },
   "dependencies": {
-    "@ag-ui/client": "0.0.40-alpha.6",
+    "@ag-ui/client": "0.0.40-alpha.7",
     "@ag-ui/langgraph": "^0.0.11",
     "@copilotkitnext/demo-agents": "workspace:^",
     "@copilotkitnext/runtime": "workspace:^",

apps/angular/storybook/package.json

@@ -9,7 +9,7 @@
     "storybook:build": "ng run storybook-angular:build-storybook"
   },
   "dependencies": {
-    "@ag-ui/client": "0.0.40-alpha.6",
+    "@ag-ui/client": "0.0.40-alpha.7",
     "@angular/animations": "^18.2.0",
     "@angular/common": "^18.2.0",
     "@angular/compiler": "^18.2.0",

apps/react/demo/package.json

@@ -9,7 +9,7 @@
     "lint": "next lint"
   },
   "dependencies": {
-    "@ag-ui/client": "0.0.40-alpha.6",
+    "@ag-ui/client": "0.0.40-alpha.7",
     "@copilotkitnext/agent": "workspace:*",
     "@copilotkitnext/core": "workspace:*",
     "@copilotkitnext/react": "workspace:*",

packages/agent/package.json

@@ -36,7 +36,7 @@
     "vitest": "^3.0.5"
   },
   "dependencies": {
-    "@ag-ui/client": "0.0.40-alpha.6",
+    "@ag-ui/client": "0.0.40-alpha.7",
     "@ai-sdk/anthropic": "^2.0.22",
     "@ai-sdk/google": "^2.0.17",
     "@ai-sdk/openai": "^2.0.42",

packages/agent/src/__tests__/utils.test.ts

@@ -90,6 +90,58 @@ describe("convertMessagesToVercelAISDKMessages", () => {
     ]);
   });
 
+  it("should convert user messages with binary content", () => {
+    const messages: Message[] = [
+      {
+        id: "1",
+        role: "user",
+        content: [
+          { type: "text", text: "Here is the design" },
+          {
+            type: "binary",
+            mimeType: "image/png",
+            url: "https://example.com/image.png",
+            filename: "image.png",
+          },
+        ],
+      },
+    ];
+
+    const result = convertMessagesToVercelAISDKMessages(messages);
+    const content = result[0].content;
+
+    expect(Array.isArray(content)).toBe(true);
+    if (Array.isArray(content)) {
+      expect(content[0]).toEqual({ type: "text", text: "Here is the design" });
+      expect(content[1]).toMatchObject({
+        type: "file",
+        mediaType: "image/png",
+        filename: "image.png",
+      });
+    }
+  });
+
+  it("should fall back to placeholders when binary content has no data", () => {
+    const messages: Message[] = [
+      {
+        id: "1",
+        role: "user",
+        content: [
+          {
+            type: "binary",
+            mimeType: "application/octet-stream",
+            id: "file-1",
+          },
+        ],
+      },
+    ];
+
+    const result = convertMessagesToVercelAISDKMessages(messages);
+    expect(result[0].content).toEqual([
+      { type: "text", text: "[Attachment: file-1]" },
+    ]);
+  });
+
   it("should convert assistant messages with text content", () => {
     const messages: Message[] = [
       {

packages/agent/src/index.ts

@@ -1,6 +1,7 @@
 import {
   AbstractAgent,
   BaseEvent,
+  BinaryInputContent,
   RunAgentInput,
   EventType,
   Message,
@@ -22,9 +23,11 @@ import {
   AssistantModelMessage,
   UserModelMessage,
   ToolModelMessage,
+  FilePart,
   ToolCallPart,
   ToolResultPart,
   TextPart,
+  UserContent,
   tool as createVercelAISDKTool,
   ToolChoice,
   ToolSet,
@@ -232,6 +235,79 @@ export function defineTool<TParameters extends z.ZodTypeAny>(config: {
   };
 }
 
+function convertBinaryInputContentToFilePart(content: BinaryInputContent): FilePart | null {
+  if (content.url) {
+    try {
+      return {
+        type: "file",
+        data: new URL(content.url),
+        mediaType: content.mimeType,
+        filename: content.filename,
+      } satisfies FilePart;
+    } catch {
+      return {
+        type: "file",
+        data: content.url,
+        mediaType: content.mimeType,
+        filename: content.filename,
+      } satisfies FilePart;
+    }
+  }
+
+  if (content.data) {
+    return {
+      type: "file",
+      data: content.data,
+      mediaType: content.mimeType,
+      filename: content.filename,
+    } satisfies FilePart;
+  }
+
+  return null;
+}
+
+function convertUserMessageContent(content: Message["content"]): UserContent {
+  if (!content) {
+    return "";
+  }
+
+  if (typeof content === "string") {
+    return content;
+  }
+
+  if (content.every((part) => part.type === "text")) {
+    return content.map((part) => part.text).join("\n\n");
+  }
+
+  const parts: Array<TextPart | FilePart> = [];
+
+  for (const part of content) {
+    if (part.type === "text") {
+      if (part.text.length > 0) {
+        parts.push({ type: "text", text: part.text });
+      }
+      continue;
+    }
+
+    const filePart = convertBinaryInputContentToFilePart(part);
+    if (filePart) {
+      parts.push(filePart);
+    } else {
+      const label = part.filename ?? part.id ?? part.mimeType;
+      parts.push({
+        type: "text",
+        text: `[Attachment: ${label}]`,
+      });
+    }
+  }
+
+  if (parts.length === 0) {
+    return "";
+  }
+
+  return parts;
+}
+
 /**
  * Converts AG-UI messages to Vercel AI SDK ModelMessage format
  */
@@ -260,7 +336,7 @@ export function convertMessagesToVercelAISDKMessages(messages: Message[]): Model
     } else if (message.role === "user") {
       const userMsg: UserModelMessage = {
         role: "user",
-        content: message.content || "",
+        content: convertUserMessageContent(message.content),
       };
       result.push(userMsg);
     } else if (message.role === "tool") {

packages/angular/package.json

@@ -31,8 +31,8 @@
     "test:watch": "vitest --watch"
   },
   "dependencies": {
-    "@ag-ui/client": "0.0.40-alpha.6",
-    "@ag-ui/core": "0.0.40-alpha.6",
+    "@ag-ui/client": "0.0.40-alpha.7",
+    "@ag-ui/core": "0.0.40-alpha.7",
     "@copilotkitnext/core": "workspace:*",
     "@copilotkitnext/shared": "workspace:*",
     "clsx": "^2.1.1",

packages/angular/src/lib/components/chat/copilot-chat-user-message-renderer.ts

@@ -7,6 +7,11 @@ import {
 } from "@angular/core";
 import { CommonModule } from "@angular/common";
 import { cn } from "../../utils";
+import type { BinaryInputContent, InputContent } from "@ag-ui/client";
+import {
+  getUserMessageBinaryContents,
+  getUserMessageTextContent,
+} from "@copilotkitnext/shared";
 
 @Component({
   selector: "copilot-chat-user-message-renderer",
@@ -17,10 +22,54 @@ import { cn } from "../../utils";
   host: {
     "[class]": "computedClass()",
   },
-  template: `{{ content() }}`,
+  template: `
+    @if (textContent()) {
+      <span>{{ textContent() }}</span>
+    }
+    @if (attachments().length) {
+      <div [class]="attachmentsClass()">
+        @for (attachment of attachments(); track trackAttachment(attachment, $index)) {
+          <ng-container *ngIf="isImage(attachment); else fileTemplate">
+            <figure class="flex flex-col gap-1">
+              <img
+                [src]="resolveSource(attachment)"
+                [alt]="attachment.filename || attachment.id || attachment.mimeType"
+                class="max-h-64 rounded-lg border border-border object-contain"
+              />
+              @if (attachment.filename || attachment.id) {
+                <figcaption class="text-xs text-muted-foreground">
+                  {{ attachment.filename || attachment.id }}
+                </figcaption>
+              }
+            </figure>
+          </ng-container>
+          <ng-template #fileTemplate>
+            <div class="rounded-md border border-dashed border-border bg-muted/70 px-3 py-2 text-xs text-muted-foreground">
+              {{ attachment.filename || attachment.id || 'Attachment' }}
+              <span class="block text-[10px] uppercase tracking-wide text-muted-foreground/70">
+                {{ attachment.mimeType }}
+              </span>
+              @if (resolveSource(attachment) && !isImage(attachment)) {
+                <a
+                  [href]="resolveSource(attachment)"
+                  target="_blank"
+                  rel="noreferrer"
+                  class="mt-1 block text-xs text-primary underline"
+                >
+                  Open
+                </a>
+              }
+            </div>
+          </ng-template>
+        }
+      </div>
+    }
+  `,
 })
 export class CopilotChatUserMessageRenderer {
   readonly content = input<string>("");
+  readonly contents = input<InputContent[]>([]);
+  readonly attachments = input<BinaryInputContent[] | undefined>(undefined);
   readonly inputClass = input<string | undefined>();
 
   readonly computedClass = computed(() => {
@@ -29,4 +78,44 @@ export class CopilotChatUserMessageRenderer {
       this.inputClass()
     );
   });
+
+  readonly textContent = computed(() => {
+    const explicit = this.content();
+    if (explicit && explicit.length > 0) {
+      return explicit;
+    }
+    return getUserMessageTextContent(this.contents());
+  });
+
+  readonly attachments = computed(() => {
+    const provided = this.attachments() ?? [];
+    if (provided.length > 0) {
+      return provided;
+    }
+    return getUserMessageBinaryContents(this.contents());
+  });
+
+  readonly attachmentsClass = computed(() =>
+    this.textContent().trim().length > 0
+      ? "mt-3 flex flex-col gap-2"
+      : "flex flex-col gap-2",
+  );
+
+  resolveSource(attachment: BinaryInputContent): string | null {
+    if (attachment.url) {
+      return attachment.url;
+    }
+    if (attachment.data) {
+      return `data:${attachment.mimeType};base64,${attachment.data}`;
+    }
+    return null;
+  }
+
+  isImage(attachment: BinaryInputContent): boolean {
+    return attachment.mimeType.startsWith("image/") && !!this.resolveSource(attachment);
+  }
+
+  trackAttachment(attachment: BinaryInputContent, index: number): string {
+    return attachment.id ?? attachment.url ?? attachment.filename ?? index.toString();
+  }
 }