fix: complete PR #8225 - add missing webview URI to base64 conversion

- Add normalizeImageRefsToDataUrls() function to convert webview URIs to base64 data URLs
- Add formatImagesIntoBlocksAsync() for async image processing in backend
- Update Task.ts to use async conversion when storing images in backend messages
- Backend now stores base64 (for API calls), frontend displays webview URIs (memory efficient)
- Fixes OpenRouter and other providers not being able to see attached images
- Maintains PR goals: webview memory efficiency + working image functionality

Author: daniel-lxs

src/core/prompts/responses.ts

@@ -3,6 +3,7 @@ import * as path from "path"
 import * as diff from "diff"
 import { RooIgnoreController, LOCK_TEXT_SYMBOL } from "../ignore/RooIgnoreController"
 import { RooProtectedController } from "../protect/RooProtectedController"
+import { normalizeImageRefsToDataUrls } from "../../integrations/misc/imageDataUrl"
 
 export const formatResponse = {
 	toolDenied: () => `The user denied this operation.`,
@@ -200,6 +201,23 @@ const formatImagesIntoBlocks = (images?: string[]): Anthropic.ImageBlockParam[]
 		: []
 }
 
+/**
+ * Async version that converts webview URIs to base64 data URLs before creating image blocks
+ * This is the missing piece from PR #8225 - allows frontend to use webview URIs while
+ * backend stores base64 for API calls.
+ */
+export const formatImagesIntoBlocksAsync = async (images?: string[]): Promise<Anthropic.ImageBlockParam[]> => {
+	if (!images || images.length === 0) {
+		return []
+	}
+
+	// Convert any webview URIs to base64 data URLs
+	const dataUrls = await normalizeImageRefsToDataUrls(images)
+
+	// Now use the regular function to create image blocks
+	return formatImagesIntoBlocks(dataUrls)
+}
+
 const toolUseInstructionsReminder = `# Reminder: Instructions for Tool Use
 
 Tool uses are formatted using XML-style tags. The tool name itself becomes the XML tag name. Each parameter is enclosed within its own set of tags. Here's the structure:

src/core/task/Task.ts

@@ -1215,7 +1215,9 @@ export class Task extends EventEmitter<TaskEvents> implements TaskLike {
 		await this.say("text", task, images)
 		this.isInitialized = true
 
-		let imageBlocks: Anthropic.ImageBlockParam[] = formatResponse.imageBlocks(images)
+		// Convert webview URIs to base64 for backend storage
+		const { formatImagesIntoBlocksAsync } = await import("../prompts/responses")
+		let imageBlocks: Anthropic.ImageBlockParam[] = await formatImagesIntoBlocksAsync(images)
 
 		// Task starting
 
@@ -1480,7 +1482,10 @@ export class Task extends EventEmitter<TaskEvents> implements TaskLike {
 		}
 
 		if (responseImages && responseImages.length > 0) {
-			newUserContent.push(...formatResponse.imageBlocks(responseImages))
+			// Convert webview URIs to base64 for backend storage
+			const { formatImagesIntoBlocksAsync } = await import("../prompts/responses")
+			const responseImageBlocks = await formatImagesIntoBlocksAsync(responseImages)
+			newUserContent.push(...responseImageBlocks)
 		}
 
 		// Ensure we have at least some content to send to the API.

src/integrations/misc/__tests__/imageDataUrl.spec.ts

@@ -0,0 +1,63 @@
+import { describe, it, expect, vi, beforeEach } from "vitest"
+import { normalizeImageRefsToDataUrls } from "../imageDataUrl"
+import * as fs from "fs/promises"
+
+// Mock fs module
+vi.mock("fs/promises")
+
+describe("normalizeImageRefsToDataUrls", () => {
+	beforeEach(() => {
+		vi.clearAllMocks()
+	})
+
+	it("should pass through data URLs unchanged", async () => {
+		const dataUrl =
+			"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8/5+hHgAHggJ/PchI7wAAAABJRU5ErkJggg=="
+		const result = await normalizeImageRefsToDataUrls([dataUrl])
+
+		expect(result).toEqual([dataUrl])
+	})
+
+	it("should convert webview URIs to data URLs", async () => {
+		const webviewUri = "file:///path/to/test.png"
+		const mockBuffer = Buffer.from("test image data")
+
+		vi.mocked(fs.readFile).mockResolvedValue(mockBuffer)
+
+		const result = await normalizeImageRefsToDataUrls([webviewUri])
+
+		expect(result).toHaveLength(1)
+		expect(result[0]).toMatch(/^data:image\/png;base64,/)
+		expect(fs.readFile).toHaveBeenCalledWith("/path/to/test.png")
+	})
+
+	it("should handle mixed arrays of data URLs and webview URIs", async () => {
+		const dataUrl = "data:image/jpeg;base64,test123"
+		const webviewUri = "file:///path/to/test.png"
+		const mockBuffer = Buffer.from("test image data")
+
+		vi.mocked(fs.readFile).mockResolvedValue(mockBuffer)
+
+		const result = await normalizeImageRefsToDataUrls([dataUrl, webviewUri])
+
+		expect(result).toHaveLength(2)
+		expect(result[0]).toBe(dataUrl) // Data URL unchanged
+		expect(result[1]).toMatch(/^data:image\/png;base64,/) // Webview URI converted
+	})
+
+	it("should handle errors gracefully by skipping problematic images", async () => {
+		const validDataUrl = "data:image/png;base64,valid"
+		const invalidWebviewUri = "file:///nonexistent/test.png"
+
+		vi.mocked(fs.readFile).mockRejectedValue(new Error("File not found"))
+
+		const result = await normalizeImageRefsToDataUrls([validDataUrl, invalidWebviewUri])
+
+		expect(result).toEqual([validDataUrl]) // Only valid ones returned
+	})
+
+	it("should handle empty arrays", async () => {
+		const result = await normalizeImageRefsToDataUrls([])
+		expect(result).toEqual([])
+	})
+})

src/integrations/misc/imageDataUrl.ts

@@ -0,0 +1,94 @@
+import * as fs from "fs/promises"
+import * as path from "path"
+
+/**
+ * Converts webview URIs to base64 data URLs for API calls.
+ * This is the missing piece from PR #8225 that allows webview URIs
+ * to be used in frontend while converting to base64 for API calls.
+ */
+export async function normalizeImageRefsToDataUrls(imageRefs: string[]): Promise<string[]> {
+	const results: string[] = []
+
+	for (const imageRef of imageRefs) {
+		// If it's already a data URL, keep it as is
+		if (imageRef.startsWith("data:image/")) {
+			results.push(imageRef)
+			continue
+		}
+
+		// Convert webview URI to file path and then to base64
+		try {
+			const filePath = webviewUriToFilePath(imageRef)
+			const buffer = await fs.readFile(filePath)
+			const base64 = buffer.toString("base64")
+			const mimeType = getMimeTypeFromPath(filePath)
+			const dataUrl = `data:${mimeType};base64,${base64}`
+			results.push(dataUrl)
+		} catch (error) {
+			console.error("Failed to convert webview URI to base64:", imageRef, error)
+			// Skip this image
+		}
+	}
+
+	return results
+}
+
+/**
+ * Converts a webview URI to a file system path
+ */
+function webviewUriToFilePath(webviewUri: string): string {
+	// Handle vscode-resource URIs like:
+	// vscode-resource://vscode-webview/path/to/file
+	if (webviewUri.includes("vscode-resource://")) {
+		// Extract the path portion after vscode-resource://vscode-webview/
+		const match = webviewUri.match(/vscode-resource:\/\/[^\/]+(.+)/)
+		if (match) {
+			return decodeURIComponent(match[1])
+		}
+	}
+
+	// Handle file:// URIs
+	if (webviewUri.startsWith("file://")) {
+		return decodeURIComponent(webviewUri.substring(7))
+	}
+
+	// Handle VS Code webview URIs that contain encoded paths
+	if (webviewUri.includes("vscode-userdata") || webviewUri.includes("vscode-cdn.net")) {
+		// Try to decode the URI and extract the file path
+		const decoded = decodeURIComponent(webviewUri)
+		// Look for a file path pattern in the decoded URI
+		const pathMatch = decoded.match(/(?:Users|C:)([^?#]+\.(?:png|jpg|jpeg|gif|webp))/i)
+		if (pathMatch) {
+			const extractedPath = pathMatch[0]
+			return extractedPath
+		}
+	}
+
+	// As a last resort, try treating it as a file path
+	return webviewUri
+}
+
+/**
+ * Gets the MIME type from a file path
+ */
+function getMimeTypeFromPath(filePath: string): string {
+	const ext = path.extname(filePath).toLowerCase()
+
+	switch (ext) {
+		case ".png":
+			return "image/png"
+		case ".jpg":
+		case ".jpeg":
+			return "image/jpeg"
+		case ".gif":
+			return "image/gif"
+		case ".webp":
+			return "image/webp"
+		case ".svg":
+			return "image/svg+xml"
+		case ".bmp":
+			return "image/bmp"
+		default:
+			return "image/png"
+	}
+}