feat: Add support for multi-modal messages

Author: MikeRyanDev

apps/angular/demo-server/package.json

@@ -8,7 +8,7 @@
     "start": "node --env-file=.env --loader tsx src/index.ts"
   },
   "dependencies": {
-    "@ag-ui/client": "0.0.40-alpha.6",
+    "@ag-ui/client": "0.0.40-alpha.7",
     "@ag-ui/langgraph": "^0.0.11",
     "@copilotkitnext/demo-agents": "workspace:^",
     "@copilotkitnext/runtime": "workspace:^",

apps/angular/storybook/package.json

@@ -9,7 +9,7 @@
     "storybook:build": "ng run storybook-angular:build-storybook"
   },
   "dependencies": {
-    "@ag-ui/client": "0.0.40-alpha.6",
+    "@ag-ui/client": "0.0.40-alpha.7",
     "@angular/animations": "^18.2.0",
     "@angular/common": "^18.2.0",
     "@angular/compiler": "^18.2.0",

apps/react/demo/package.json

@@ -9,7 +9,7 @@
     "lint": "next lint"
   },
   "dependencies": {
-    "@ag-ui/client": "0.0.40-alpha.6",
+    "@ag-ui/client": "0.0.40-alpha.7",
     "@copilotkitnext/agent": "workspace:*",
     "@copilotkitnext/core": "workspace:*",
     "@copilotkitnext/react": "workspace:*",

packages/agent/package.json

@@ -36,7 +36,7 @@
     "vitest": "^3.0.5"
   },
   "dependencies": {
-    "@ag-ui/client": "0.0.40-alpha.6",
+    "@ag-ui/client": "0.0.40-alpha.7",
     "@ai-sdk/anthropic": "^2.0.22",
     "@ai-sdk/google": "^2.0.17",
     "@ai-sdk/openai": "^2.0.42",

packages/agent/src/__tests__/utils.test.ts

@@ -90,6 +90,58 @@ describe("convertMessagesToVercelAISDKMessages", () => {
     ]);
   });
 
+  it("should convert user messages with binary content", () => {
+    const messages: Message[] = [
+      {
+        id: "1",
+        role: "user",
+        content: [
+          { type: "text", text: "Here is the design" },
+          {
+            type: "binary",
+            mimeType: "image/png",
+            url: "https://example.com/image.png",
+            filename: "image.png",
+          },
+        ],
+      },
+    ];
+
+    const result = convertMessagesToVercelAISDKMessages(messages);
+    const content = result[0].content;
+
+    expect(Array.isArray(content)).toBe(true);
+    if (Array.isArray(content)) {
+      expect(content[0]).toEqual({ type: "text", text: "Here is the design" });
+      expect(content[1]).toMatchObject({
+        type: "file",
+        mediaType: "image/png",
+        filename: "image.png",
+      });
+    }
+  });
+
+  it("should fall back to placeholders when binary content has no data", () => {
+    const messages: Message[] = [
+      {
+        id: "1",
+        role: "user",
+        content: [
+          {
+            type: "binary",
+            mimeType: "application/octet-stream",
+            id: "file-1",
+          },
+        ],
+      },
+    ];
+
+    const result = convertMessagesToVercelAISDKMessages(messages);
+    expect(result[0].content).toEqual([
+      { type: "text", text: "[Attachment: file-1]" },
+    ]);
+  });
+
   it("should convert assistant messages with text content", () => {
     const messages: Message[] = [
       {

packages/agent/src/index.ts

@@ -1,6 +1,7 @@
 import {
   AbstractAgent,
   BaseEvent,
+  BinaryInputContent,
   RunAgentInput,
   EventType,
   Message,
@@ -22,9 +23,11 @@ import {
   AssistantModelMessage,
   UserModelMessage,
   ToolModelMessage,
+  FilePart,
   ToolCallPart,
   ToolResultPart,
   TextPart,
+  UserContent,
   tool as createVercelAISDKTool,
   ToolChoice,
   ToolSet,
@@ -232,6 +235,79 @@ export function defineTool<TParameters extends z.ZodTypeAny>(config: {
   };
 }
 
+function convertBinaryInputContentToFilePart(content: BinaryInputContent): FilePart | null {
+  if (content.url) {
+    try {
+      return {
+        type: "file",
+        data: new URL(content.url),
+        mediaType: content.mimeType,
+        filename: content.filename,
+      } satisfies FilePart;
+    } catch {
+      return {
+        type: "file",
+        data: content.url,
+        mediaType: content.mimeType,
+        filename: content.filename,
+      } satisfies FilePart;
+    }
+  }
+
+  if (content.data) {
+    return {
+      type: "file",
+      data: content.data,
+      mediaType: content.mimeType,
+      filename: content.filename,
+    } satisfies FilePart;
+  }
+
+  return null;
+}
+
+function convertUserMessageContent(content: Message["content"]): UserContent {
+  if (!content) {
+    return "";
+  }
+
+  if (typeof content === "string") {
+    return content;
+  }
+
+  if (content.every((part) => part.type === "text")) {
+    return content.map((part) => part.text).join("\n\n");
+  }
+
+  const parts: Array<TextPart | FilePart> = [];
+
+  for (const part of content) {
+    if (part.type === "text") {
+      if (part.text.length > 0) {
+        parts.push({ type: "text", text: part.text });
+      }
+      continue;
+    }
+
+    const filePart = convertBinaryInputContentToFilePart(part);
+    if (filePart) {
+      parts.push(filePart);
+    } else {
+      const label = part.filename ?? part.id ?? part.mimeType;
+      parts.push({
+        type: "text",
+        text: `[Attachment: ${label}]`,
+      });
+    }
+  }
+
+  if (parts.length === 0) {
+    return "";
+  }
+
+  return parts;
+}
+
 /**
  * Converts AG-UI messages to Vercel AI SDK ModelMessage format
  */
@@ -260,7 +336,7 @@ export function convertMessagesToVercelAISDKMessages(messages: Message[]): Model
     } else if (message.role === "user") {
       const userMsg: UserModelMessage = {
         role: "user",
-        content: message.content || "",
+        content: convertUserMessageContent(message.content),
       };
       result.push(userMsg);
     } else if (message.role === "tool") {

packages/angular/package.json

@@ -31,8 +31,8 @@
     "test:watch": "vitest --watch"
   },
   "dependencies": {
-    "@ag-ui/client": "0.0.40-alpha.6",
-    "@ag-ui/core": "0.0.40-alpha.6",
+    "@ag-ui/client": "0.0.40-alpha.7",
+    "@ag-ui/core": "0.0.40-alpha.7",
     "@copilotkitnext/core": "workspace:*",
     "@copilotkitnext/shared": "workspace:*",
     "clsx": "^2.1.1",

packages/angular/src/lib/components/chat/copilot-chat-user-message-renderer.ts

@@ -7,6 +7,11 @@ import {
 } from "@angular/core";
 import { CommonModule } from "@angular/common";
 import { cn } from "../../utils";
+import type { BinaryInputContent, InputContent } from "@ag-ui/client";
+import {
+  getUserMessageBinaryContents,
+  getUserMessageTextContent,
+} from "@copilotkitnext/shared";
 
 @Component({
   selector: "copilot-chat-user-message-renderer",
@@ -17,10 +22,54 @@ import { cn } from "../../utils";
   host: {
     "[class]": "computedClass()",
   },
-  template: `{{ content() }}`,
+  template: `
+    @if (textContent()) {
+      <span>{{ textContent() }}</span>
+    }
+    @if (attachments().length) {
+      <div [class]="attachmentsClass()">
+        @for (attachment of attachments(); track trackAttachment(attachment, $index)) {
+          <ng-container *ngIf="isImage(attachment); else fileTemplate">
+            <figure class="flex flex-col gap-1">
+              <img
+                [src]="resolveSource(attachment)"
+                [alt]="attachment.filename || attachment.id || attachment.mimeType"
+                class="max-h-64 rounded-lg border border-border object-contain"
+              />
+              @if (attachment.filename || attachment.id) {
+                <figcaption class="text-xs text-muted-foreground">
+                  {{ attachment.filename || attachment.id }}
+                </figcaption>
+              }
+            </figure>
+          </ng-container>
+          <ng-template #fileTemplate>
+            <div class="rounded-md border border-dashed border-border bg-muted/70 px-3 py-2 text-xs text-muted-foreground">
+              {{ attachment.filename || attachment.id || 'Attachment' }}
+              <span class="block text-[10px] uppercase tracking-wide text-muted-foreground/70">
+                {{ attachment.mimeType }}
+              </span>
+              @if (resolveSource(attachment) && !isImage(attachment)) {
+                <a
+                  [href]="resolveSource(attachment)"
+                  target="_blank"
+                  rel="noreferrer"
+                  class="mt-1 block text-xs text-primary underline"
+                >
+                  Open
+                </a>
+              }
+            </div>
+          </ng-template>
+        }
+      </div>
+    }
+  `,
 })
 export class CopilotChatUserMessageRenderer {
   readonly content = input<string>("");
+  readonly contents = input<InputContent[]>([]);
+  readonly attachments = input<BinaryInputContent[] | undefined>(undefined);
   readonly inputClass = input<string | undefined>();
 
   readonly computedClass = computed(() => {
@@ -29,4 +78,44 @@ export class CopilotChatUserMessageRenderer {
       this.inputClass()
     );
   });
+
+  readonly textContent = computed(() => {
+    const explicit = this.content();
+    if (explicit && explicit.length > 0) {
+      return explicit;
+    }
+    return getUserMessageTextContent(this.contents());
+  });
+
+  readonly attachments = computed(() => {
+    const provided = this.attachments() ?? [];
+    if (provided.length > 0) {
+      return provided;
+    }
+    return getUserMessageBinaryContents(this.contents());
+  });
+
+  readonly attachmentsClass = computed(() =>
+    this.textContent().trim().length > 0
+      ? "mt-3 flex flex-col gap-2"
+      : "flex flex-col gap-2",
+  );
+
+  resolveSource(attachment: BinaryInputContent): string | null {
+    if (attachment.url) {
+      return attachment.url;
+    }
+    if (attachment.data) {
+      return `data:${attachment.mimeType};base64,${attachment.data}`;
+    }
+    return null;
+  }
+
+  isImage(attachment: BinaryInputContent): boolean {
+    return attachment.mimeType.startsWith("image/") && !!this.resolveSource(attachment);
+  }
+
+  trackAttachment(attachment: BinaryInputContent, index: number): string {
+    return attachment.id ?? attachment.url ?? attachment.filename ?? index.toString();
+  }
 }

packages/angular/src/lib/components/chat/copilot-chat-user-message.ts

@@ -29,6 +29,11 @@ import {
 import { CopilotChatUserMessageToolbar } from "./copilot-chat-user-message-toolbar";
 import { CopilotChatUserMessageBranchNavigation } from "./copilot-chat-user-message-branch-navigation";
 import { cn } from "../../utils";
+import {
+  getUserMessageBinaryContents,
+  getUserMessageTextContent,
+  normalizeUserMessageContents,
+} from "@copilotkitnext/shared";
 
 @Component({
   standalone: true,
@@ -46,17 +51,20 @@ import { cn } from "../../utils";
   encapsulation: ViewEncapsulation.None,
   template: `
     <div [class]="computedClass()" [attr.data-message-id]="message()?.id">
+      @let messageCtx = messageRendererContext();
       <!-- Message Renderer -->
       @if (messageRendererTemplate || messageRendererComponent()) {
         <copilot-slot
           [slot]="messageRendererTemplate || messageRendererComponent()"
-          [context]="messageRendererContext()"
+          [context]="messageCtx"
           [defaultComponent]="CopilotChatUserMessageRenderer"
         >
         </copilot-slot>
       } @else {
         <copilot-chat-user-message-renderer
-          [content]="message()?.content || ''"
+          [content]="messageCtx.content"
+          [contents]="messageCtx.contents"
+          [attachments]="messageCtx.attachments"
           [inputClass]="messageRendererClass()"
         >
         </copilot-chat-user-message-renderer>
@@ -84,14 +92,14 @@ import { cn } from "../../utils";
             @if (copyButtonTemplate || copyButtonComponent()) {
               <copilot-slot
                 [slot]="copyButtonTemplate || copyButtonComponent()"
-                [context]="{ content: message()?.content || '' }"
+                [context]="{ content: messageCtx.content }"
                 [outputs]="copyButtonOutputs"
                 [defaultComponent]="CopilotChatUserMessageCopyButton"
               >
               </copilot-slot>
             } @else {
               <copilot-chat-user-message-copy-button
-                [content]="message()?.content"
+                [content]="messageCtx.content"
                 [inputClass]="copyButtonClass()"
                 (clicked)="handleCopy()"
               >
@@ -211,9 +219,15 @@ export class CopilotChatUserMessage {
   );
 
   // Context for slots (reactive via signals)
-  messageRendererContext = computed<MessageRendererContext>(() => ({
-    content: this.message()?.content || "",
-  }));
+  messageRendererContext = computed<MessageRendererContext>(() => {
+    const message = this.message();
+    const contents = normalizeUserMessageContents(message?.content);
+    return {
+      content: getUserMessageTextContent(contents),
+      contents,
+      attachments: getUserMessageBinaryContents(contents),
+    };
+  });
 
   // Output maps for slots
   copyButtonOutputs = { clicked: () => this.handleCopy() };