Skip to content

feat: add image support for MCP tool responses #6165

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
roomote wants to merge 1 commit into from
Closed

feat: add image support for MCP tool responses #6165

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
114 changes: 112 additions & 2 deletions src/core/tools/__tests__/useMcpToolTool.spec.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,11 @@ import { ToolUse } from "../../../shared/tools"
// Mock dependencies
vi.mock("../../prompts/responses", () => ({
formatResponse: {
toolResult: vi.fn((result: string) => `Tool result: ${result}`),
toolResult: vi.fn((result: string, images?: string[]) =>
images && images.length > 0
? `Tool result: ${result} [with ${images.length} image(s)]`
: `Tool result: ${result}`,
),
toolError: vi.fn((error: string) => `Tool error: ${error}`),
invalidMcpToolArgumentError: vi.fn((server: string, tool: string) => `Invalid args for ${server}:${tool}`),
},
Expand Down Expand Up @@ -208,10 +212,116 @@ describe("useMcpToolTool", () => {
expect(mockTask.consecutiveMistakeCount).toBe(0)
expect(mockAskApproval).toHaveBeenCalled()
expect(mockTask.say).toHaveBeenCalledWith("mcp_server_request_started")
expect(mockTask.say).toHaveBeenCalledWith("mcp_server_response", "Tool executed successfully")
expect(mockTask.say).toHaveBeenCalledWith("mcp_server_response", "Tool executed successfully", [])
expect(mockPushToolResult).toHaveBeenCalledWith("Tool result: Tool executed successfully")
})

it("should handle tool response with images", async () => {
const block: ToolUse = {
type: "tool_use",
name: "use_mcp_tool",
params: {
server_name: "screenshot_server",
tool_name: "capture_screenshot",
arguments: '{"url": "https://example.com"}',
},
partial: false,
}

mockAskApproval.mockResolvedValue(true)

const mockToolResult = {
content: [
{ type: "text", text: "Screenshot captured successfully" },
{
type: "image",
data: "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg==",
mimeType: "image/png",
},
],
isError: false,
}

mockProviderRef.deref.mockReturnValue({
getMcpHub: () => ({
callTool: vi.fn().mockResolvedValue(mockToolResult),
}),
postMessageToWebview: vi.fn(),
})

await useMcpToolTool(
mockTask as Task,
block,
mockAskApproval,
mockHandleError,
mockPushToolResult,
mockRemoveClosingTag,
)

expect(mockTask.consecutiveMistakeCount).toBe(0)
expect(mockAskApproval).toHaveBeenCalled()
expect(mockTask.say).toHaveBeenCalledWith("mcp_server_request_started")
expect(mockTask.say).toHaveBeenCalledWith(
"mcp_server_response",
"Screenshot captured successfully\n\n[Image: image/png]",
[
"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg==",
],
)
expect(mockPushToolResult).toHaveBeenCalledWith(
"Tool result: Screenshot captured successfully\n\n[Image: image/png] [with 1 image(s)]",
)
})

it("should handle tool response with multiple images", async () => {
const block: ToolUse = {
type: "tool_use",
name: "use_mcp_tool",
params: {
server_name: "image_processor",
tool_name: "process_images",
arguments: "{}",
},
partial: false,
}

mockAskApproval.mockResolvedValue(true)

const mockToolResult = {
content: [
{ type: "text", text: "Processed 2 images" },
{ type: "image", data: "data:image/png;base64,ABC123", mimeType: "image/png" },
{ type: "image", data: "XYZ789", mimeType: "image/jpeg" },
],
isError: false,
}

mockProviderRef.deref.mockReturnValue({
getMcpHub: () => ({
callTool: vi.fn().mockResolvedValue(mockToolResult),
}),
postMessageToWebview: vi.fn(),
})

await useMcpToolTool(
mockTask as Task,
block,
mockAskApproval,
mockHandleError,
mockPushToolResult,
mockRemoveClosingTag,
)

expect(mockTask.say).toHaveBeenCalledWith(
"mcp_server_response",
"Processed 2 images\n\n[Image: image/png]\n\n[Image: image/jpeg]",
["data:image/png;base64,ABC123", "data:image/jpeg;base64,XYZ789"],
)
expect(mockPushToolResult).toHaveBeenCalledWith(
"Tool result: Processed 2 images\n\n[Image: image/png]\n\n[Image: image/jpeg] [with 2 image(s)]",
)
})

it("should handle user rejection", async () => {
const block: ToolUse = {
type: "tool_use",
Expand Down
28 changes: 22 additions & 6 deletions src/core/tools/useMcpToolTool.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -89,16 +89,27 @@ async function sendExecutionStatus(cline: Task, status: McpExecutionStatus): Pro
})
}

function processToolContent(toolResult: any): string {
function processToolContent(toolResult: any): { text: string; images: string[] } {
const images: string[] = []

if (!toolResult?.content || toolResult.content.length === 0) {
return ""
return { text: "", images }
}

return toolResult.content
const text = toolResult.content
.map((item: any) => {
if (item.type === "text") {
return item.text
}
if (item.type === "image" && item.data && item.mimeType) {
// Handle base64 image data
if (item.data.startsWith("data:")) {
images.push(item.data)
} else {
images.push(`data:${item.mimeType};base64,${item.data}`)
}
return `[Image: ${item.mimeType}]`
}
if (item.type === "resource") {
const { blob: _, ...rest } = item.resource
return JSON.stringify(rest, null, 2)
Expand All @@ -107,6 +118,8 @@ function processToolContent(toolResult: any): string {
})
.filter(Boolean)
.join("\n\n")

return { text, images }
}

async function executeToolAndProcessResult(
Expand All @@ -130,9 +143,12 @@ async function executeToolAndProcessResult(
const toolResult = await cline.providerRef.deref()?.getMcpHub()?.callTool(serverName, toolName, parsedArguments)

let toolResultPretty = "(No response)"
let images: string[] = []

if (toolResult) {
const outputText = processToolContent(toolResult)
const processedContent = processToolContent(toolResult)
const outputText = processedContent.text
images = processedContent.images

if (outputText) {
await sendExecutionStatus(cline, {
Expand Down Expand Up @@ -160,8 +176,8 @@ async function executeToolAndProcessResult(
})
}

await cline.say("mcp_server_response", toolResultPretty)
pushToolResult(formatResponse.toolResult(toolResultPretty))
await cline.say("mcp_server_response", toolResultPretty, images)
pushToolResult(formatResponse.toolResult(toolResultPretty, images))
}

export async function useMcpToolTool(
Expand Down