MCP Image: Support image type messages / base64 text (RooCodeInc#2962)

* feat: MCP ImageContent support

* feat: MCP ImageContent support

* feat: MCP ImageContent support changeset

* Update src/core/prompts/responses.ts

Co-authored-by: Dennis Bartlett <[email protected]>

* Update src/core/Cline.ts

Co-authored-by: Dennis Bartlett <[email protected]>

* feat: MCP ImageContent support run format:fix

* Remove mcpToolResult

* Fix: Display original data:image URLs in rich display mode to maintain transparency

---------

Co-authored-by: rikaaa0928 <[email protected]>
Co-authored-by: rikaaa0928 <[email protected]>
Co-authored-by: Dennis Bartlett <[email protected]>
Co-authored-by: Saoud Rizwan <[email protected]>
Co-authored-by: Andrei Edell <[email protected]>

Author: 5 people

.changeset/slimy-jeans-dance.md

@@ -0,0 +1,5 @@
+---
+"claude-dev": patch
+---
+
+MCP ImageContent as tool result

src/core/prompts/responses.ts

@@ -2,6 +2,7 @@ import { Anthropic } from "@anthropic-ai/sdk"
 import * as diff from "diff"
 import * as path from "path"
 import { ClineIgnoreController, LOCK_TEXT_SYMBOL } from "../ignore/ClineIgnoreController"
+import { McpToolCallResponse } from "../../shared/mcp"
 
 export const formatResponse = {
 	duplicateFileReadNotice: () =>

src/core/task/index.ts

@@ -2690,8 +2690,13 @@ export class Task {
 								await this.say("mcp_server_request_started") // same as browser_action_result
 								const toolResult = await this.mcpHub.callTool(server_name, tool_name, parsedArguments)
 
-								// TODO: add progress indicator and ability to parse images and non-text responses
-								const toolResultPretty =
+								// TODO: add progress indicator
+
+								const toolResultImages =
+									toolResult?.content
+										.filter((item) => item.type === "image")
+										.map((item) => `data:${item.mimeType};base64,${item.data}`) || []
+								let toolResultText =
 									(toolResult?.isError ? "Error:\n" : "") +
 										toolResult?.content
 											.map((item) => {
@@ -2706,8 +2711,21 @@ export class Task {
 											})
 											.filter(Boolean)
 											.join("\n\n") || "(No response)"
-								await this.say("mcp_server_response", toolResultPretty)
-								pushToolResult(formatResponse.toolResult(toolResultPretty))
+								// webview extracts images from the text response to display in the UI
+								const toolResultToDisplay =
+									toolResultText + toolResultImages?.map((image) => `\n\n${image}`).join("")
+								await this.say("mcp_server_response", toolResultToDisplay)
+
+								// MCP's might return images to display to the user, but the model may not support them
+								const supportsImages = this.api.getModel().info.supportsImages ?? false
+								if (toolResultImages.length > 0 && !supportsImages) {
+									toolResultText += `\n\n[${toolResultImages.length} images were provided in the response, and while they are displayed to the user, you do not have the ability to view them.]`
+								}
+
+								// only passes in images if model supports them
+								pushToolResult(
+									formatResponse.toolResult(toolResultText, supportsImages ? toolResultImages : undefined),
+								)
 
 								await this.saveCheckpoint()
 

webview-ui/src/components/mcp/chat-display/McpResponseDisplay.tsx

@@ -162,7 +162,7 @@ const McpResponseDisplay: React.FC<McpResponseDisplayProps> = ({ responseText })
 			try {
 				const text = responseText || ""
 				const matches: UrlMatch[] = []
-				const urlRegex = /https?:\/\/[^\s<>"']+/g
+				const urlRegex = /(?:https?:\/\/|data:image)[^\s<>"']+/g
 				let urlMatch: RegExpExecArray | null
 				let urlCount = 0