fix: optimize image handling with dual-storage approach

- Add imagesBase64 field to ClineMessage schema for efficient dual storage
- Store both webview URIs (for display) and base64 (for API) when creating messages
- Remove repeated base64→file→URI conversions on every render
- Update all tools to use base64 from stored messages for API calls
- Eliminates file I/O overhead during rendering and message updates
- Improves performance by storing each format once instead of converting repeatedly

Author: daniel-lxs

packages/types/src/message.ts

@@ -206,7 +206,8 @@ export const clineMessageSchema = z.object({
 	ask: clineAskSchema.optional(),
 	say: clineSaySchema.optional(),
 	text: z.string().optional(),
-	images: z.array(z.string()).optional(),
+	images: z.array(z.string()).optional(), // Webview URIs for frontend display
+	imagesBase64: z.array(z.string()).optional(), // Base64 data URLs for API calls
 	partial: z.boolean().optional(),
 	reasoning: z.string().optional(),
 	conversationHistoryIndex: z.number().optional(),

src/core/assistant-message/presentAssistantMessage.ts

@@ -291,7 +291,12 @@ export async function presentAssistantMessage(cline: Task) {
 					// Handle both messageResponse and noButtonClicked with text.
 					if (text) {
 						await cline.say("user_feedback", text, images)
-						pushToolResult(formatResponse.toolResult(formatResponse.toolDeniedWithFeedback(text), images))
+						// Get base64 from the just-stored message for API call
+						const lastMessage = cline.clineMessages.at(-1)
+						const base64Images = lastMessage?.imagesBase64
+						pushToolResult(
+							formatResponse.toolResult(formatResponse.toolDeniedWithFeedback(text), base64Images),
+						)
 					} else {
 						pushToolResult(formatResponse.toolDenied())
 					}
@@ -302,7 +307,12 @@ export async function presentAssistantMessage(cline: Task) {
 				// Handle yesButtonClicked with text.
 				if (text) {
 					await cline.say("user_feedback", text, images)
-					pushToolResult(formatResponse.toolResult(formatResponse.toolApprovedWithFeedback(text), images))
+					// Get base64 from the just-stored message for API call
+					const lastMessage = cline.clineMessages.at(-1)
+					const base64Images = lastMessage?.imagesBase64
+					pushToolResult(
+						formatResponse.toolResult(formatResponse.toolApprovedWithFeedback(text), base64Images),
+					)
 				}
 
 				return true
@@ -396,18 +406,22 @@ export async function presentAssistantMessage(cline: Task) {
 					)
 
 					if (response === "messageResponse") {
-						// Add user feedback to userContent.
+						// Add user feedback to chat (stores both formats)
+						await cline.say("user_feedback", text, images)
+
+						// Get base64 from the just-stored message for API call
+						const lastMessage = cline.clineMessages.at(-1)
+						const base64Images = lastMessage?.imagesBase64
+
+						// Add user feedback to userContent for API
 						cline.userMessageContent.push(
 							{
 								type: "text" as const,
 								text: `Tool repetition limit reached. User feedback: ${text}`,
 							},
-							...formatResponse.imageBlocks(images),
+							...formatResponse.imageBlocks(base64Images),
 						)
 
-						// Add user feedback to chat.
-						await cline.say("user_feedback", text, images)
-
 						// Track tool repetition in telemetry.
 						TelemetryService.instance.captureConsecutiveMistakeError(cline.taskId)
 					}

src/core/task/Task.ts

@@ -644,6 +644,9 @@ export class Task extends EventEmitter<TaskEvents> implements TaskLike {
 
 	private async updateClineMessage(message: ClineMessage) {
 		const provider = this.providerRef.deref()
+
+		// Messages now store both formats, so no conversion needed
+		// The 'images' field already contains webview URIs for display
 		await provider?.postMessageToWebview({ type: "messageUpdated", clineMessage: message })
 		this.emit(RooCodeEventName.Message, { action: "updated", message })
 
@@ -735,6 +738,7 @@ export class Task extends EventEmitter<TaskEvents> implements TaskLike {
 					lastMessage.partial = partial
 					lastMessage.progressStatus = progressStatus
 					lastMessage.isProtected = isProtected
+					// Note: ask messages don't typically have images, so we don't update them here
 					// TODO: Be more efficient about saving and posting only new
 					// data or one whole message at a time so ignore partial for
 					// saves, and only post parts of partial message instead of
@@ -877,16 +881,8 @@ export class Task extends EventEmitter<TaskEvents> implements TaskLike {
 			text: this.askResponseText,
 			images: this.askResponseImages,
 		}
-		// Normalize any image refs to base64 data URLs before handing back to callers
-		if (Array.isArray(result.images) && result.images.length > 0) {
-			try {
-				const { normalizeImageRefsToDataUrls } = await import("../../integrations/misc/imageDataUrl")
-				const normalized = await normalizeImageRefsToDataUrls(result.images)
-				result.images = normalized
-			} catch (e) {
-				console.error("[Task#ask] Failed to normalize image refs:", e)
-			}
-		}
+		// Images from askResponse are already webview URIs from the frontend,
+		// so no conversion needed here
 		this.askResponse = undefined
 		this.askResponseText = undefined
 		this.askResponseImages = undefined
@@ -1084,13 +1080,23 @@ export class Task extends EventEmitter<TaskEvents> implements TaskLike {
 			throw new Error(`[RooCode#say] task ${this.taskId}.${this.instanceId} aborted`)
 		}
 
-		// Ensure any image refs are normalized to base64 data URLs before persisting or sending to APIs
+		// Convert images to both formats for efficient dual storage
+		let webviewUris: string[] | undefined
+		let base64Images: string[] | undefined
+
 		if (Array.isArray(images) && images.length > 0) {
 			try {
 				const { normalizeImageRefsToDataUrls } = await import("../../integrations/misc/imageDataUrl")
-				images = await normalizeImageRefsToDataUrls(images)
+
+				// Store original webview URIs/file paths for frontend
+				webviewUris = images
+
+				// Convert to base64 for API calls
+				base64Images = await normalizeImageRefsToDataUrls(images)
 			} catch (e) {
 				console.error("[Task#say] Failed to normalize image refs:", e)
+				// Fall back to original images if conversion fails
+				webviewUris = images
 			}
 		}
 
@@ -1104,7 +1110,8 @@ export class Task extends EventEmitter<TaskEvents> implements TaskLike {
 				if (isUpdatingPreviousPartial) {
 					// Existing partial message, so update it.
 					lastMessage.text = text
-					lastMessage.images = images
+					lastMessage.images = webviewUris
+					lastMessage.imagesBase64 = base64Images
 					lastMessage.partial = partial
 					lastMessage.progressStatus = progressStatus
 					this.updateClineMessage(lastMessage)
@@ -1121,7 +1128,8 @@ export class Task extends EventEmitter<TaskEvents> implements TaskLike {
 						type: "say",
 						say: type,
 						text,
-						images,
+						images: webviewUris,
+						imagesBase64: base64Images,
 						partial,
 						contextCondense,
 						metadata: options.metadata,
@@ -1137,7 +1145,8 @@ export class Task extends EventEmitter<TaskEvents> implements TaskLike {
 					}
 
 					lastMessage.text = text
-					lastMessage.images = images
+					lastMessage.images = webviewUris
+					lastMessage.imagesBase64 = base64Images
 					lastMessage.partial = false
 					lastMessage.progressStatus = progressStatus
 					if (options.metadata) {
@@ -1168,7 +1177,8 @@ export class Task extends EventEmitter<TaskEvents> implements TaskLike {
 						type: "say",
 						say: type,
 						text,
-						images,
+						images: webviewUris,
+						imagesBase64: base64Images,
 						contextCondense,
 						metadata: options.metadata,
 					})
@@ -1191,7 +1201,8 @@ export class Task extends EventEmitter<TaskEvents> implements TaskLike {
 				type: "say",
 				say: type,
 				text,
-				images,
+				images: webviewUris,
+				imagesBase64: base64Images,
 				checkpoint,
 				contextCondense,
 			})
@@ -1236,14 +1247,16 @@ export class Task extends EventEmitter<TaskEvents> implements TaskLike {
 
 		await this.providerRef.deref()?.postStateToWebview()
 
-		// Convert webview URIs to base64 data URLs for backend storage (one-time conversion)
-		const { normalizeImageRefsToDataUrls } = await import("../../integrations/misc/imageDataUrl")
-		const base64Images = images ? await normalizeImageRefsToDataUrls(images) : undefined
-
-		await this.say("text", task, base64Images) // Store base64 in backend messages
+		// Store the task message with both webview URIs and base64
+		// This is now handled in say() method which stores both formats
+		await this.say("text", task, images)
 		this.isInitialized = true
 
-		// Convert base64 to image blocks for API (no conversion needed, already base64)
+		// Get base64 from the stored message for API call
+		const lastMessage = this.clineMessages.at(-1)
+		const base64Images = lastMessage?.imagesBase64
+
+		// Convert base64 to image blocks for API
 		const { formatResponse } = await import("../prompts/responses")
 		let imageBlocks: Anthropic.ImageBlockParam[] = formatResponse.imageBlocks(base64Images)
 
@@ -1509,11 +1522,11 @@ export class Task extends EventEmitter<TaskEvents> implements TaskLike {
 		}
 
 		if (responseImages && responseImages.length > 0) {
-			// Convert webview URIs to base64 data URLs for backend storage (one-time conversion)
+			// Images from user response are webview URIs, convert to base64 for API
 			const { normalizeImageRefsToDataUrls } = await import("../../integrations/misc/imageDataUrl")
 			const base64ResponseImages = await normalizeImageRefsToDataUrls(responseImages)
 
-			// Convert base64 to image blocks for API (no conversion needed, already base64)
+			// Convert base64 to image blocks for API
 			const { formatResponse } = await import("../prompts/responses")
 			const responseImageBlocks = formatResponse.imageBlocks(base64ResponseImages)
 			newUserContent.push(...responseImageBlocks)
@@ -1778,15 +1791,19 @@ export class Task extends EventEmitter<TaskEvents> implements TaskLike {
 				)
 
 				if (response === "messageResponse") {
+					await this.say("user_feedback", text, images)
+
+					// Get base64 from the just-stored message for API call
+					const lastMessage = this.clineMessages.at(-1)
+					const base64Images = lastMessage?.imagesBase64
+
 					currentUserContent.push(
 						...[
 							{ type: "text" as const, text: formatResponse.tooManyMistakes(text) },
-							...formatResponse.imageBlocks(images),
+							...formatResponse.imageBlocks(base64Images),
 						],
 					)
 
-					await this.say("user_feedback", text, images)
-
 					// Track consecutive mistake errors in telemetry.
 					TelemetryService.instance.captureConsecutiveMistakeError(this.taskId)
 				}

src/core/tools/accessMcpResourceTool.ts

@@ -82,7 +82,12 @@ export async function accessMcpResourceTool(
 			})
 
 			await cline.say("mcp_server_response", resourceResultPretty, images)
-			pushToolResult(formatResponse.toolResult(resourceResultPretty, images))
+
+			// Get base64 from the just-stored message for API call
+			// Note: MCP images are already base64, but say() will store them in both formats
+			const lastMessage = cline.clineMessages.at(-1)
+			const base64Images = lastMessage?.imagesBase64
+			pushToolResult(formatResponse.toolResult(resourceResultPretty, base64Images))
 
 			return
 		}

src/core/tools/askFollowupQuestionTool.ts

@@ -78,7 +78,11 @@ export async function askFollowupQuestionTool(
 			cline.consecutiveMistakeCount = 0
 			const { text, images } = await cline.ask("followup", JSON.stringify(follow_up_json), false)
 			await cline.say("user_feedback", text ?? "", images)
-			pushToolResult(formatResponse.toolResult(`<answer>\n${text}\n</answer>`, images))
+
+			// Get base64 from the just-stored message for API call
+			const lastMessage = cline.clineMessages.at(-1)
+			const base64Images = lastMessage?.imagesBase64
+			pushToolResult(formatResponse.toolResult(`<answer>\n${text}\n</answer>`, base64Images))
 
 			return
 		}

src/core/tools/attemptCompletionTool.ts

@@ -121,14 +121,19 @@ export async function attemptCompletionTool(
 			}
 
 			await cline.say("user_feedback", text ?? "", images)
+
+			// Get base64 from the just-stored message for API call
+			const lastMessage = cline.clineMessages.at(-1)
+			const base64Images = lastMessage?.imagesBase64
+
 			const toolResults: (Anthropic.TextBlockParam | Anthropic.ImageBlockParam)[] = []
 
 			toolResults.push({
 				type: "text",
 				text: `The user has provided feedback on the results. Consider their input to continue the task, and then attempt completion again.\n<feedback>\n${text}\n</feedback>`,
 			})
 
-			toolResults.push(...formatResponse.imageBlocks(images))
+			toolResults.push(...formatResponse.imageBlocks(base64Images))
 			cline.userMessageContent.push({ type: "text", text: `${toolDescription()} Result:` })
 			cline.userMessageContent.push(...toolResults)
 

src/core/tools/executeCommandTool.ts

@@ -311,6 +311,10 @@ export async function executeCommand(
 		const { text, images } = message
 		await task.say("user_feedback", text, images)
 
+		// Get base64 from the just-stored message for API call
+		const lastMessage = task.clineMessages.at(-1)
+		const base64Images = lastMessage?.imagesBase64
+
 		return [
 			true,
 			formatResponse.toolResult(
@@ -320,7 +324,7 @@ export async function executeCommand(
 					`The user provided the following feedback:`,
 					`<feedback>\n${text}\n</feedback>`,
 				].join("\n"),
-				images,
+				base64Images,
 			),
 		]
 	} else if (completed || exitDetails) {

src/core/tools/readFileTool.ts

@@ -406,29 +406,36 @@ export async function readFileTool(
 
 			const { response, text, images } = await cline.ask("tool", completeMessage, false)
 
+			let feedbackBase64Images: string[] | undefined
 			if (response !== "yesButtonClicked") {
 				// Handle both messageResponse and noButtonClicked with text
 				if (text) {
 					await cline.say("user_feedback", text, images)
+					// Get base64 from the just-stored message
+					const lastMessage = cline.clineMessages.at(-1)
+					feedbackBase64Images = lastMessage?.imagesBase64
 				}
 				cline.didRejectTool = true
 
 				updateFileResult(relPath, {
 					status: "denied",
 					xmlContent: `<file><path>${relPath}</path><status>Denied by user</status></file>`,
 					feedbackText: text,
-					feedbackImages: images,
+					feedbackImages: feedbackBase64Images,
 				})
 			} else {
 				// Handle yesButtonClicked with text
 				if (text) {
 					await cline.say("user_feedback", text, images)
+					// Get base64 from the just-stored message
+					const lastMessage = cline.clineMessages.at(-1)
+					feedbackBase64Images = lastMessage?.imagesBase64
 				}
 
 				updateFileResult(relPath, {
 					status: "approved",
 					feedbackText: text,
-					feedbackImages: images,
+					feedbackImages: feedbackBase64Images,
 				})
 			}
 		}