add image handling code

Author: canrobins13

src/api/transform/image-cleaning.ts

@@ -0,0 +1,28 @@
+import { ApiHandler } from ".."
+import { ApiMessage } from "../../core/task-persistence/apiMessages"
+
+/* Removes image blocks from messages if they are not supported by the Api Handler */
+export function maybeRemoveImageBlocks(messages: ApiMessage[], apiHandler: ApiHandler): ApiMessage[] {
+	return messages.map((message) => {
+		// Handle array content (could contain image blocks).
+		let { content } = message
+		if (Array.isArray(content)) {
+			if (!apiHandler.getModel().info.supportsImages) {
+				// Convert image blocks to text descriptions.
+				content = content.map((block) => {
+					if (block.type === "image") {
+						// Convert image blocks to text descriptions.
+						// Note: We can't access the actual image content/url due to API limitations,
+						// but we can indicate that an image was present in the conversation.
+						return {
+							type: "text",
+							text: "[Referenced image in conversation]",
+						}
+					}
+					return block
+				})
+			}
+		}
+		return { ...message, content }
+	})
+}

src/core/condense/index.ts

@@ -1,6 +1,7 @@
 import Anthropic from "@anthropic-ai/sdk"
 import { ApiHandler } from "../../api"
 import { ApiMessage } from "../task-persistence/apiMessages"
+import { maybeRemoveImageBlocks } from "../../api/transform/image-cleaning"
 
 const CONTEXT_FRAC_FOR_SUMMARY = 0.5 // TODO(canyon): make this configurable
 const N_MESSAGES_TO_KEEP = 3
@@ -82,7 +83,10 @@ async function summarizeConversation(messages: ApiMessage[], apiHandler: ApiHand
 		role: "user",
 		content: "Summarize the conversation so far, as described in the prompt instructions.",
 	}
-	const stream = apiHandler.createMessage(SUMMARY_PROMPT, [...messagesToSummarize, finalRequestMessage])
+	const requestMessages = maybeRemoveImageBlocks([...messagesToSummarize, finalRequestMessage], apiHandler).map(
+		({ role, content }) => ({ role, content }),
+	)
+	const stream = apiHandler.createMessage(SUMMARY_PROMPT, requestMessages)
 	let summary = ""
 	for await (const chunk of stream) {
 		if (chunk.type === "text") {

src/core/task/Task.ts

@@ -80,6 +80,7 @@ import {
 import { processUserContentMentions } from "../mentions/processUserContentMentions"
 import { ApiMessage } from "../task-persistence/apiMessages"
 import { getMessagesSinceLastSummary, summarizeConversationIfNeeded } from "../condense"
+import { maybeRemoveImageBlocks } from "../../api/transform/image-cleaning"
 
 export type ClineEvents = {
 	message: [{ action: "created" | "updated"; message: ClineMessage }]
@@ -1484,32 +1485,9 @@ export class Task extends EventEmitter<ClineEvents> {
 		}
 
 		const messagesSinceLastSummary = getMessagesSinceLastSummary(this.apiConversationHistory)
-
-		// Clean conversation history by:
-		// 1. Converting to Anthropic.MessageParam by spreading only the API-required properties.
-		// 2. Converting image blocks to text descriptions if model doesn't support images.
-		const cleanConversationHistory = messagesSinceLastSummary.map(({ role, content }) => {
-			// Handle array content (could contain image blocks).
-			if (Array.isArray(content)) {
-				if (!this.api.getModel().info.supportsImages) {
-					// Convert image blocks to text descriptions.
-					content = content.map((block) => {
-						if (block.type === "image") {
-							// Convert image blocks to text descriptions.
-							// Note: We can't access the actual image content/url due to API limitations,
-							// but we can indicate that an image was present in the conversation.
-							return {
-								type: "text",
-								text: "[Referenced image in conversation]",
-							}
-						}
-						return block
-					})
-				}
-			}
-
-			return { role, content }
-		})
+		const cleanConversationHistory = maybeRemoveImageBlocks(messagesSinceLastSummary, this.api).map(
+			({ role, content }) => ({ role, content }),
+		)
 
 		const stream = this.api.createMessage(systemPrompt, cleanConversationHistory)
 		const iterator = stream[Symbol.asyncIterator]()