replace operator agent with base of new agent (#1014)

# Why

Replace operator agent with new agent handler

The operator agent was an older implementation that did not use tool
calling and used a single model for both high-level reasoning and
low-level action execution.

# What Changed

- **Removed operator agent** (`StagehandOperatorHandler`)

- **Added new agent handler** (`StagehandAgentHandler`) 
  - Leverages AI SDK for proper tool call handling
  - **New `executionModel` option** for dual-model architecture
  - Better error handling and retry mechanisms
  - Structured tool system with Zod schema validation

- **ExecutionModel feature:**
- Use a powerful model (like claude 4 sonnet) for reasoning and planning
- Use a faster model (like gemini 2.0 flash) for Stagehand operations
like `act()` and `extract()`
  - Enables cost and performance optimization


# Test Plan

- Tested locally with various agent tasks
- Verified backward compatibility 
- Tested dual-model execution with different model combinations
- Installed package from branch, for additional local testing to catch
any additional edge cases

---------

Co-authored-by: Sameel <[email protected]>
Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com>
Co-authored-by: miguel <[email protected]>

Author: 4 people

.changeset/pink-snakes-sneeze.md

@@ -0,0 +1,5 @@
+---
+"@browserbasehq/stagehand": patch
+---
+
+Replace operator handler with base of new agent

.changeset/tired-cats-repeat.md

@@ -0,0 +1,5 @@
+---
+"@browserbasehq/stagehand": patch
+---
+
+replace operator agent with scaffold for new stagehand agent

docs/basics/agent.mdx

@@ -26,7 +26,11 @@ agent.execute("apply for a job at browserbase")
 
 ## Using `agent()`
 
-Here is how you can use `agent()` to create an agent.
+There are two ways to create agents in Stagehand:
+
+### Computer Use Agents
+
+Use computer use agents with specialized models from OpenAI or Anthropic: 
 
 <CodeGroup>
 ```typescript TypeScript
@@ -54,6 +58,18 @@ await agent.execute("apply for a job at Browserbase")
 ```
 </CodeGroup>
 
+### Use Stagehand Agent with Any LLM
+
+Use the agent without specifying a provider to utilize any model or LLM provider:
+
+<Note>Non CUA agents are currently only supported in TypeScript</Note>
+
+```typescript TypeScript
+const agent = stagehand.agent();
+await agent.execute("apply for a job at Browserbase")
+```
+
+
 ## MCP Integrations
 
 Agents can be enhanced with external tools and services through MCP (Model Context Protocol) integrations. This allows your agent to access external APIs and data sources beyond just browser interactions.

evals/index.eval.ts

@@ -33,7 +33,7 @@ import { CustomOpenAIClient } from "@/examples/external_clients/customOpenAI";
 import OpenAI from "openai";
 import { initStagehand } from "./initStagehand";
 import { AgentProvider } from "@/lib/agent/AgentProvider";
-import { AISdkClient } from "@/examples/external_clients/aisdk";
+import { AISdkClient } from "@/lib/llm/aisdk";
 import { getAISDKLanguageModel } from "@/lib/llm/LLMProvider";
 import { loadApiKeyFromEnv } from "@/lib/utils";
 import { LogLine } from "@/types/log";

evals/initStagehand.ts

@@ -114,6 +114,11 @@ export const initStagehand = async ({
       model: modelName,
       provider: modelName.startsWith("claude") ? "anthropic" : "openai",
     } as AgentConfig;
+  } else {
+    agentConfig = {
+      model: modelName,
+      executionModel: "google/gemini-2.5-flash",
+    } as AgentConfig;
   }
 
   const agent = stagehand.agent(agentConfig);

evals/taskConfig.ts

@@ -106,7 +106,11 @@ const DEFAULT_EVAL_MODELS = process.env.EVAL_MODELS
 
 const DEFAULT_AGENT_MODELS = process.env.EVAL_AGENT_MODELS
   ? process.env.EVAL_AGENT_MODELS.split(",")
-  : ["computer-use-preview-2025-03-11", "claude-sonnet-4-20250514"];
+  : [
+      "computer-use-preview-2025-03-11",
+      "claude-sonnet-4-20250514",
+      "anthropic/claude-sonnet-4-20250514",
+    ];
 
 /**
  * getModelList:

evals/tasks/agent/sf_library_card.ts

@@ -10,19 +10,15 @@ export const sf_library_card: EvalFunction = async ({
 }) => {
   try {
     await stagehand.page.goto("https://sflib1.sfpl.org/selfreg");
-
     const agentResult = await agent.execute({
-      instruction:
-        "Fill in the 'Residential Address' field with '166 Geary St'",
+      instruction: "Fill in the 'street Address' field with '166 Geary St'",
       maxSteps: Number(process.env.AGENT_EVAL_MAX_STEPS) || 3,
     });
     logger.log(agentResult);
-
-    await stagehand.page.mouse.wheel(0, -1000);
     const evaluator = new Evaluator(stagehand);
     const result = await evaluator.ask({
       question:
-        "Does the page show the 'Residential Address' field filled with '166 Geary St'?",
+        "Does the page show the 'street Address' field filled with '166 Geary St'?",
     });
 
     if (result.evaluation !== "YES" && result.evaluation !== "NO") {

lib/agent/tools/act.ts

@@ -0,0 +1,55 @@
+import { tool } from "ai";
+import { z } from "zod/v3";
+import { StagehandPage } from "../../StagehandPage";
+
+export const createActTool = (
+  stagehandPage: StagehandPage,
+  executionModel?: string,
+) =>
+  tool({
+    description: "Perform an action on the page (click, type)",
+    parameters: z.object({
+      action: z.string()
+        .describe(`Describe what to click, or type within in a short, specific phrase that mentions the element type. 
+          Examples:
+          - "click the Login button"
+          - "click the language dropdown"
+          - type "John" into the first name input
+          - type "Doe" into the last name input`),
+    }),
+    execute: async ({ action }) => {
+      try {
+        let result;
+        if (executionModel) {
+          result = await stagehandPage.page.act({
+            action,
+            modelName: executionModel,
+          });
+        } else {
+          result = await stagehandPage.page.act(action);
+        }
+        const isIframeAction = result.action === "an iframe";
+
+        if (isIframeAction) {
+          const fallback = await stagehandPage.page.act(
+            executionModel
+              ? { action, modelName: executionModel, iframes: true }
+              : { action, iframes: true },
+          );
+          return {
+            success: fallback.success,
+            action: fallback.action,
+            isIframe: true,
+          };
+        }
+
+        return {
+          success: result.success,
+          action: result.action,
+          isIframe: false,
+        };
+      } catch (error) {
+        return { success: false, error: error.message };
+      }
+    },
+  });

lib/agent/tools/ariaTree.ts

@@ -0,0 +1,35 @@
+import { tool } from "ai";
+import { z } from "zod/v3";
+import { StagehandPage } from "../../StagehandPage";
+
+export const createAriaTreeTool = (stagehandPage: StagehandPage) =>
+  tool({
+    description:
+      "gets the accessibility (ARIA) tree from the current page. this is useful for understanding the page structure and accessibility features. it should provide full context of what is on the page",
+    parameters: z.object({}),
+    execute: async () => {
+      const { page_text } = await stagehandPage.page.extract();
+      const pageUrl = stagehandPage.page.url();
+
+      let content = page_text;
+      const MAX_CHARACTERS = 70000;
+
+      const estimatedTokens = Math.ceil(content.length / 4);
+
+      if (estimatedTokens > MAX_CHARACTERS) {
+        const maxCharacters = MAX_CHARACTERS * 4;
+        content =
+          content.substring(0, maxCharacters) +
+          "\n\n[CONTENT TRUNCATED: Exceeded 70,000 token limit]";
+      }
+
+      return {
+        content,
+        pageUrl,
+      };
+    },
+    experimental_toToolResultContent: (result) => {
+      const content = typeof result === "string" ? result : result.content;
+      return [{ type: "text", text: `Accessibility Tree:\n${content}` }];
+    },
+  });

lib/agent/tools/close.ts

@@ -0,0 +1,16 @@
+import { tool } from "ai";
+import { z } from "zod/v3";
+
+export const createCloseTool = () =>
+  tool({
+    description: "Complete the task and close",
+    parameters: z.object({
+      reasoning: z.string().describe("Summary of what was accomplished"),
+      taskComplete: z
+        .boolean()
+        .describe("Whether the task was completed successfully"),
+    }),
+    execute: async ({ reasoning, taskComplete }) => {
+      return { success: true, reasoning, taskComplete };
+    },
+  });