removed all images except the latest two in AnthropicCuaClient (#905)

# why
part of STG-586

Currently, we leave all of the images for anthropic cua client within
the LLM's context as the task progresses

# what changed

We now remove all screenshots aside from the last two when experimental
flag is set to true within stagehand config

# test plan

tested locally

Author: tkattkat

.changeset/fresh-papayas-beam.md

@@ -0,0 +1,5 @@
+---
+"@browserbasehq/stagehand": patch
+---
+
+Delete old images from anthropic cua client

lib/agent/AgentProvider.ts

@@ -35,6 +35,7 @@ export class AgentProvider {
     modelName: string,
     clientOptions?: Record<string, unknown>,
     userProvidedInstructions?: string,
+    experimental?: boolean,
   ): AgentClient {
     const type = AgentProvider.getAgentProvider(modelName);
     this.logger({
@@ -58,6 +59,7 @@ export class AgentProvider {
             modelName,
             userProvidedInstructions,
             clientOptions,
+            experimental,
           );
         default:
           throw new UnsupportedModelProviderError(

lib/agent/AnthropicCUAClient.ts

@@ -13,6 +13,7 @@ import {
 } from "@/types/agent";
 import { AgentClient } from "./AgentClient";
 import { AgentScreenshotProviderError } from "@/types/stagehandErrors";
+import { compressConversationImages } from "./imageCompressionUtils";
 
 export type ResponseInputItem = AnthropicMessage | AnthropicToolResult;
 
@@ -30,12 +31,14 @@ export class AnthropicCUAClient extends AgentClient {
   private screenshotProvider?: () => Promise<string>;
   private actionHandler?: (action: AgentAction) => Promise<void>;
   private thinkingBudget: number | null = null;
+  private experimental: boolean = false;
 
   constructor(
     type: AgentType,
     modelName: string,
     userProvidedInstructions?: string,
     clientOptions?: Record<string, unknown>,
+    experimental?: boolean,
   ) {
     super(type, modelName, userProvidedInstructions);
 
@@ -51,6 +54,7 @@ export class AnthropicCUAClient extends AgentClient {
     ) {
       this.thinkingBudget = clientOptions.thinkingBudget;
     }
+    this.experimental = experimental || false;
 
     // Store client options for reference
     this.clientOptions = {
@@ -321,6 +325,9 @@ export class AnthropicCUAClient extends AgentClient {
       const nextInputItems: ResponseInputItem[] = [...inputItems];
 
       // Add the assistant message with tool_use blocks to the history
+      if (this.experimental) {
+        compressConversationImages(nextInputItems);
+      }
       nextInputItems.push(assistantMessage);
 
       // Generate tool results and add them as a user message

lib/agent/imageCompressionUtils.ts

@@ -0,0 +1,87 @@
+import {
+  AnthropicMessage,
+  AnthropicContentBlock,
+  AnthropicToolResult,
+} from "@/types/agent";
+
+export type ResponseInputItem = AnthropicMessage | AnthropicToolResult;
+
+/**
+ * Finds all items in the conversation history that contain images
+ * @param items - Array of conversation items to check
+ * @returns Array of indices where images were found
+ */
+export function findItemsWithImages(items: ResponseInputItem[]): number[] {
+  const itemsWithImages: number[] = [];
+
+  items.forEach((item, index) => {
+    let hasImage = false;
+
+    if (Array.isArray(item.content)) {
+      hasImage = item.content.some(
+        (contentItem: AnthropicContentBlock) =>
+          contentItem.type === "tool_result" &&
+          "content" in contentItem &&
+          Array.isArray(contentItem.content) &&
+          (contentItem.content as AnthropicContentBlock[]).some(
+            (nestedItem: AnthropicContentBlock) => nestedItem.type === "image",
+          ),
+      );
+    }
+
+    if (hasImage) {
+      itemsWithImages.push(index);
+    }
+  });
+
+  return itemsWithImages;
+}
+
+/**
+ * Compresses conversation history by removing images from older items
+ * while keeping the most recent images intact
+ * @param items - Array of conversation items to process
+ * @param keepMostRecentCount - Number of most recent image-containing items to preserve (default: 2)
+ * @returns Object with processed items
+ */
+export function compressConversationImages(
+  items: ResponseInputItem[],
+  keepMostRecentCount: number = 2,
+): { items: ResponseInputItem[] } {
+  const itemsWithImages = findItemsWithImages(items);
+
+  items.forEach((item, index) => {
+    const imageIndex = itemsWithImages.indexOf(index);
+    const shouldCompress =
+      imageIndex >= 0 &&
+      imageIndex < itemsWithImages.length - keepMostRecentCount;
+
+    if (shouldCompress) {
+      if (Array.isArray(item.content)) {
+        item.content = item.content.map(
+          (contentItem: AnthropicContentBlock) => {
+            if (
+              contentItem.type === "tool_result" &&
+              "content" in contentItem &&
+              Array.isArray(contentItem.content) &&
+              (contentItem.content as AnthropicContentBlock[]).some(
+                (nestedItem: AnthropicContentBlock) =>
+                  nestedItem.type === "image",
+              )
+            ) {
+              return {
+                ...contentItem,
+                content: "screenshot taken",
+              } as AnthropicContentBlock;
+            }
+            return contentItem;
+          },
+        );
+      }
+    }
+  });
+
+  return {
+    items,
+  };
+}

lib/handlers/agentHandler.ts

@@ -41,6 +41,7 @@ export class StagehandAgentHandler {
       options.modelName,
       options.clientOptions || {},
       options.userProvidedInstructions,
+      options.experimental,
     );
 
     // Store the client

lib/index.ts

@@ -931,6 +931,7 @@ export class Stagehand {
       You are currently on the following page: ${this.stagehandPage.page.url()}.
       Do not ask follow up questions, the user will trust your judgement.`,
         agentType: options.provider,
+        experimental: this.experimental,
       },
     );
 

types/agent.ts

@@ -52,6 +52,7 @@ export interface AgentHandlerOptions {
   clientOptions?: Record<string, unknown>;
   userProvidedInstructions?: string;
   agentType: AgentType;
+  experimental?: boolean;
 }
 
 export interface ActionExecutionResult {