feat(webui): upload images/PDFs from the chat composer

examples/a2ui_agent/agent.py

@@ -22,6 +22,7 @@
 """
 
 from veadk import Agent
+from veadk.utils.pdf_to_images import pdf_to_images_before_model_callback
 
 INSTRUCTION = """You are a helpful assistant that can render rich UI.
 
@@ -38,6 +39,9 @@
     description="Demo agent that replies with A2UI rich UI.",
     instruction=INSTRUCTION,
     enable_a2ui=True,
+    # Uploaded PDFs are rendered to page images so the vision model can read
+    # them. The default model (doubao-seed-1.6) is vision-capable.
+    before_model_callback=pdf_to_images_before_model_callback,
 )
 
 # Required by the Google ADK agent loader.

examples/basic-app/README.md

@@ -14,7 +14,7 @@ basic-app/
 ├── agents/
 │   └── basic_app_agent/         # backend agent (A2UI on), exposes root_agent
 ├── agentkit.yaml                # AgentKit deployment config (build_script wired)
-├── scripts/install_veadk.sh     # installs veadk[a2ui] from feat/a2ui (build time)
+├── scripts/install_veadk.sh     # installs veadk[a2ui,pdf] (build time)
 ├── requirements.txt             # empty (veadk is installed by the build script)
 ├── .dockerignore
 └── .env.example
@@ -50,7 +50,7 @@ cp .env.example .env
 ## 2. Run locally (optional)
 
 ```bash
-pip install "veadk-python[a2ui]"
+pip install "veadk-python[a2ui,pdf]"
 python app.py            # or: python -m app
 # open http://127.0.0.1:8000
 ```
@@ -59,6 +59,14 @@ You should see the web UI; ask e.g. "show me a flight status card" and the agent
 replies with rich A2UI. `/list-apps` returns `["basic_app_agent"]` and `/ping`
 returns `{"status": "ok"}`.
 
+### Attachments (image / PDF)
+
+The composer's **+** button uploads **images** and **PDFs**. Images are sent to
+the (vision-capable) model directly; PDFs are rendered to page images by a
+`before_model_callback` (`veadk.utils.pdf_to_images`) so the model can read them
+— this needs the `pdf` extra (included in `[a2ui,pdf]` above) and a
+vision-capable model (the default `doubao-seed-1.6` is).
+
 ## 3. Deploy to AgentKit
 
 ```bash

examples/basic-app/README.zh.md

@@ -13,7 +13,7 @@ basic-app/
 ├── agents/
 │   └── basic_app_agent/         # 后端 Agent（已开启 A2UI），暴露 root_agent
 ├── agentkit.yaml                # AgentKit 部署配置（已接入 build_script）
-├── scripts/install_veadk.sh     # 构建时从 feat/a2ui 安装 veadk[a2ui]
+├── scripts/install_veadk.sh     # 构建时从 feat/a2ui 安装 veadk[a2ui,pdf]
 ├── requirements.txt             # 留空（veadk 由构建脚本安装）
 ├── .dockerignore
 └── .env.example
@@ -47,14 +47,21 @@ cp .env.example .env
 ## 2. 本地运行（可选）
 
 ```bash
-pip install "veadk-python[a2ui]"
+pip install "veadk-python[a2ui,pdf]"
 python app.py            # 或：python -m app
 # 打开 http://127.0.0.1:8000
 ```
 
 你应当能看到 Web UI；试着问“给我一张航班状态卡片”，Agent 会用富 A2UI 作答。
 `/list-apps` 返回 `["basic_app_agent"]`，`/ping` 返回 `{"status": "ok"}`。
 
+### 附件（图片 / PDF）
+
+输入框的 **+** 按钮可上传**图片**与 **PDF**。图片会直接发送给（具备视觉能力的）
+模型；PDF 则由 `before_model_callback`（`veadk.utils.pdf_to_images`）渲染为逐页
+图片后交给模型识别——这需要 `pdf` 额外依赖（已包含在上面的 `[a2ui,pdf]` 中），
+并使用具备视觉能力的模型（默认的 `doubao-seed-1.6` 即可）。
+
 ## 3. 部署到 AgentKit
 
 ```bash

examples/basic-app/agents/basic_app_agent/agent.py

@@ -20,6 +20,7 @@
 """
 
 from veadk import Agent
+from veadk.utils.pdf_to_images import pdf_to_images_before_model_callback
 
 INSTRUCTION = """You are a helpful assistant that can render rich UI.
 
@@ -36,6 +37,9 @@
     description="Basic front+back demo agent that can reply with A2UI rich UI.",
     instruction=INSTRUCTION,
     enable_a2ui=True,
+    # Uploaded PDFs are rendered to page images so the vision model can read
+    # them. The default model (doubao-seed-1.6) is vision-capable.
+    before_model_callback=pdf_to_images_before_model_callback,
 )
 
 # Required by the Google ADK agent loader.

examples/basic-app/scripts/install_veadk.sh

@@ -34,4 +34,4 @@ done
 
 cd "$SRC"
 git sparse-checkout set veadk
-uv pip install ".[a2ui]"
+uv pip install ".[a2ui,pdf]"

frontend/src/App.tsx

@@ -1,5 +1,5 @@
 import { useEffect, useState } from "react";
-import { Check, Copy, Loader2 } from "lucide-react";
+import { Check, Copy, FileText, Loader2 } from "lucide-react";
 import { motion } from "motion/react";
 import {
   createSession,
@@ -9,6 +9,7 @@ import {
   listSessions,
   runSSE,
   type AdkSession,
+  type Attachment,
 } from "./adk/client";
 import { applyEvent, emptyAcc, eventsToTurns, type Turn } from "./blocks";
 import { Sidebar } from "./ui/Sidebar";
@@ -104,13 +105,30 @@ const GREETINGS = [
 ];
 const pickGreeting = () => GREETINGS[Math.floor(Math.random() * GREETINGS.length)];
 
+const MAX_FILE_BYTES = 20 * 1024 * 1024; // 20 MB/file (base64 inflates ~33%)
+
+/** Read a File as base64 (without the `data:...;base64,` prefix). */
+function fileToBase64(file: File): Promise<string> {
+  return new Promise((resolve, reject) => {
+    const reader = new FileReader();
+    reader.onload = () => {
+      const res = String(reader.result);
+      const comma = res.indexOf(",");
+      resolve(comma >= 0 ? res.slice(comma + 1) : res);
+    };
+    reader.onerror = () => reject(reader.error);
+    reader.readAsDataURL(file);
+  });
+}
+
 export default function App() {
   const [apps, setApps] = useState<string[]>([]);
   const [appName, setAppName] = useState("");
   const [sessions, setSessions] = useState<AdkSession[]>([]);
   const [sessionId, setSessionId] = useState("");
   const [turns, setTurns] = useState<Turn[]>([]);
   const [input, setInput] = useState("");
+  const [attachments, setAttachments] = useState<Attachment[]>([]);
   const [busy, setBusy] = useState(false);
   const [error, setError] = useState("");
   const [traceOpen, setTraceOpen] = useState(false);
@@ -220,8 +238,25 @@ export default function App() {
     }
   }
 
-  async function send(text: string) {
-    if (!text.trim() || busy || !appName || !userId) return;
+  async function addFiles(files: FileList | File[]) {
+    const picked: Attachment[] = [];
+    for (const f of Array.from(files)) {
+      if (f.size > MAX_FILE_BYTES) {
+        setError(`文件过大（>20MB）：${f.name}`);
+        continue;
+      }
+      const data = await fileToBase64(f);
+      picked.push({
+        mimeType: f.type || "application/octet-stream",
+        data,
+        name: f.name,
+      });
+    }
+    if (picked.length) setAttachments((a) => [...a, ...picked]);
+  }
+
+  async function send(text: string, atts: Attachment[] = []) {
+    if ((!text.trim() && atts.length === 0) || busy || !appName || !userId) return;
     setError("");
     setBusy(true);
 
@@ -238,17 +273,30 @@ export default function App() {
       }
     }
 
+    const userBlocks: Turn["blocks"] = [];
+    if (atts.length)
+      userBlocks.push({
+        kind: "attachment",
+        files: atts.map((a) => ({ mimeType: a.mimeType, data: a.data, name: a.name })),
+      });
+    if (text.trim()) userBlocks.push({ kind: "text", text });
     setTurns((t) => [
       ...t,
-      { role: "user", blocks: [{ kind: "text", text }], meta: { ts: Date.now() / 1000 } },
+      { role: "user", blocks: userBlocks, meta: { ts: Date.now() / 1000 } },
       { role: "assistant", blocks: [] },
     ]);
 
     try {
       let acc = emptyAcc();
       let tokens = 0;
       let ts = Date.now() / 1000;
-      for await (const event of runSSE({ appName, userId, sessionId: sid, text })) {
+      for await (const event of runSSE({
+        appName,
+        userId,
+        sessionId: sid,
+        text,
+        attachments: atts,
+      })) {
         acc = applyEvent(acc, event);
         const usage = event.usageMetadata ?? event.usage_metadata;
         if (usage?.totalTokenCount) tokens = usage.totalTokenCount;
@@ -305,11 +353,18 @@ export default function App() {
             onChange={setInput}
             onSubmit={() => {
               const text = input;
+              const atts = attachments;
               setInput("");
-              send(text);
+              setAttachments([]);
+              send(text, atts);
             }}
             disabled={!appName || !userId}
             busy={busy}
+            attachments={attachments}
+            onAddFiles={addFiles}
+            onRemoveAttachment={(i) =>
+              setAttachments((a) => a.filter((_, j) => j !== i))
+            }
           />
         );
         return (
@@ -332,7 +387,10 @@ export default function App() {
                   {turns.map((turn, i) => {
             const isLast = i === turns.length - 1;
             if (turn.role === "user") {
-              const text = turn.blocks.map((b) => ("text" in b ? b.text : "")).join("");
+              const text = turn.blocks.map((b) => (b.kind === "text" ? b.text : "")).join("");
+              const atts = turn.blocks.flatMap((b) =>
+                b.kind === "attachment" ? b.files : [],
+              );
               return (
                 <motion.div
                   key={i}
@@ -341,9 +399,30 @@ export default function App() {
                   animate={{ opacity: 1, y: 0 }}
                   transition={{ duration: 0.2, ease: "easeOut" }}
                 >
-                  <div className="bubble">
-                    <Markdown text={text} />
-                  </div>
+                  {atts.length > 0 && (
+                    <div className="msg-attachments">
+                      {atts.map((f, j) =>
+                        f.mimeType?.startsWith("image/") && f.data ? (
+                          <img
+                            key={j}
+                            className="attachment-thumb"
+                            src={`data:${f.mimeType};base64,${f.data}`}
+                            alt={f.name ?? "image"}
+                          />
+                        ) : (
+                          <div key={j} className="attachment-file">
+                            <FileText className="icon" />
+                            <span className="attachment-file-name">{f.name ?? "文件"}</span>
+                          </div>
+                        ),
+                      )}
+                    </div>
+                  )}
+                  {text && (
+                    <div className="bubble">
+                      <Markdown text={text} />
+                    </div>
+                  )}
                   <div className="turn-actions turn-actions--right">
                     {turn.meta?.ts && <span className="meta-text">{fmtTime(turn.meta.ts)}</span>}
                     <CopyButton text={text} />

frontend/src/adk/client.ts

@@ -45,16 +45,34 @@ export interface AdkSession {
   [k: string]: unknown;
 }
 
+export interface AdkInlineData {
+  mimeType?: string;
+  data?: string; // base64 (no data: prefix)
+  displayName?: string;
+  // snake_case fallback (defensive, in case the server echoes snake_case)
+  mime_type?: string;
+  display_name?: string;
+}
+
 export interface AdkPart {
   text?: string;
   thought?: boolean;
+  inlineData?: AdkInlineData;
+  inline_data?: AdkInlineData; // snake_case fallback (defensive)
   functionCall?: { name?: string; args?: Record<string, unknown> };
   functionResponse?: { name?: string; response?: Record<string, unknown> };
   // snake_case fallbacks (defensive)
   function_call?: { name?: string; args?: Record<string, unknown> };
   function_response?: { name?: string; response?: Record<string, unknown> };
 }
 
+/** A file the user attached in the composer, encoded for `/run_sse`. */
+export interface Attachment {
+  mimeType: string;
+  data: string; // base64 (no data: prefix)
+  name?: string;
+}
+
 const API_BASE = ""; // same origin (prod) / proxied (dev)
 
 /** fetch wrapper that forwards the gateway auth querystring on every request. */
@@ -123,6 +141,7 @@ export interface RunArgs {
   userId: string;
   sessionId: string;
   text: string;
+  attachments?: Attachment[];
 }
 
 /** Stream agent events for one user turn. */
@@ -131,15 +150,22 @@ export async function* runSSE({
   userId,
   sessionId,
   text,
+  attachments = [],
 }: RunArgs): AsyncGenerator<AdkEvent, void, unknown> {
+  const parts: AdkPart[] = [
+    ...attachments.map((a) => ({
+      inlineData: { mimeType: a.mimeType, data: a.data, displayName: a.name },
+    })),
+    ...(text.trim() ? [{ text }] : []),
+  ];
   const res = await apiFetch(`/run_sse`, {
     method: "POST",
     headers: { "Content-Type": "application/json" },
     body: JSON.stringify({
       app_name: appName,
       user_id: userId,
       session_id: sessionId,
-      new_message: { role: "user", parts: [{ text }] },
+      new_message: { role: "user", parts },
       streaming: true,
     }),
   });