offload initialization of agent to the runner

tkattkat · tkattkat · commit 81f0e1ab4c6f · 2025-08-13T09:41:11.000-07:00
diff --git a/evals/initStagehand.ts b/evals/initStagehand.ts
@@ -18,6 +18,7 @@ import {
 } from "@browserbasehq/stagehand";
 import { EvalLogger } from "./logger";
 import type { StagehandInitResult } from "@/types/evals";
+import type { AgentConfig } from "@/dist";
 import { AvailableModel } from "@browserbasehq/stagehand";
 
 /**
@@ -104,12 +105,26 @@ export const initStagehand = async ({
   // Set navigation timeout to 60 seconds for evaluations
   stagehand.context.setDefaultNavigationTimeout(60_000);
 
+  const isCUAModel = (model: string): boolean =>
+    model.includes("computer-use-preview") || model.startsWith("claude");
+
+  let agentConfig: AgentConfig | undefined;
+  if (isCUAModel(modelName)) {
+    agentConfig = {
+      model: modelName,
+      provider: modelName.startsWith("claude") ? "anthropic" : "openai",
+    } as AgentConfig;
+  }
+
+  const agent = stagehand.agent(agentConfig);
+
   return {
     stagehand,
     stagehandConfig: config,
     logger,
     debugUrl,
     sessionUrl,
     modelName,
+    agent,
   };
 };
diff --git a/evals/tasks/agent/all_recipes.ts b/evals/tasks/agent/all_recipes.ts
@@ -5,17 +5,11 @@ export const all_recipes: EvalFunction = async ({
   sessionUrl,
   stagehand,
   logger,
-  modelName,
+  agent,
 }) => {
   try {
     await stagehand.page.goto("https://www.allrecipes.com/");
 
-    const agent = stagehand.agent({
-      model: modelName,
-      provider: modelName.startsWith("claude") ? "anthropic" : "openai",
-      instructions: `You are a helpful assistant that can help me with my tasks. You are given a task and you need to complete it without asking follow up questions. The current page is ${await stagehand.page.title()}`,
-    });
-
     const agentResult = await agent.execute({
       instruction:
         "Search for a recipe for Beef Wellington on Allrecipes that has at least 200 reviews and an average rating of 4.5 stars or higher. List the main ingredients required for the dish.",
diff --git a/evals/tasks/agent/amazon_shoes.ts b/evals/tasks/agent/amazon_shoes.ts
@@ -5,15 +5,10 @@ export const amazon_shoes: EvalFunction = async ({
   sessionUrl,
   stagehand,
   logger,
-  modelName,
+  agent,
 }) => {
   try {
     await stagehand.page.goto("https://www.amazon.com/");
-    const agent = stagehand.agent({
-      model: modelName,
-      provider: modelName.startsWith("claude") ? "anthropic" : "openai",
-      instructions: `You are a helpful web automation assistant. DON'T ASK FOLLOW UP QUESTIONS UNTIL YOU HAVE FULFILLED THE USER'S REQUEST. Today is ${new Date().toLocaleDateString()}.`,
-    });
 
     const agentResult = await agent.execute({
       instruction:
diff --git a/evals/tasks/agent/apple_trade_in.ts b/evals/tasks/agent/apple_trade_in.ts
@@ -5,15 +5,10 @@ export const apple_trade_in: EvalFunction = async ({
   sessionUrl,
   stagehand,
   logger,
-  modelName,
+  agent,
 }) => {
   try {
     await stagehand.page.goto("https://www.apple.com/");
-    const agent = stagehand.agent({
-      model: modelName,
-      provider: modelName.startsWith("claude") ? "anthropic" : "openai",
-      instructions: `You are a helpful web automation assistant. DON'T ASK FOLLOW UP QUESTIONS UNTIL YOU HAVE FULFILLED THE USER'S REQUEST. Today is ${new Date().toLocaleDateString()}.`,
-    });
 
     const agentResult = await agent.execute({
       instruction:
diff --git a/evals/tasks/agent/apple_tv.ts b/evals/tasks/agent/apple_tv.ts
@@ -5,15 +5,10 @@ export const apple_tv: EvalFunction = async ({
   sessionUrl,
   stagehand,
   logger,
-  modelName,
+  agent,
 }) => {
   try {
     await stagehand.page.goto("https://www.apple.com/");
-    const agent = stagehand.agent({
-      model: modelName,
-      provider: modelName.startsWith("claude") ? "anthropic" : "openai",
-      instructions: `You are a helpful web automation assistant. DON'T ASK FOLLOW UP QUESTIONS UNTIL YOU HAVE FULFILLED THE USER'S REQUEST. Today is ${new Date().toLocaleDateString()}.`,
-    });
 
     const agentResult = await agent.execute({
       instruction:
diff --git a/evals/tasks/agent/arxiv_gpt_report.ts b/evals/tasks/agent/arxiv_gpt_report.ts
@@ -5,15 +5,10 @@ export const arxiv_gpt_report: EvalFunction = async ({
   sessionUrl,
   stagehand,
   logger,
-  modelName,
+  agent,
 }) => {
   try {
     await stagehand.page.goto("https://arxiv.org/");
-    const agent = stagehand.agent({
-      model: modelName,
-      provider: modelName.startsWith("claude") ? "anthropic" : "openai",
-      instructions: `You are a helpful web automation assistant. DON'T ASK FOLLOW UP QUESTIONS UNTIL YOU HAVE FULFILLED THE USER'S REQUEST. Today is ${new Date().toLocaleDateString()}.`,
-    });
 
     const agentResult = await agent.execute({
       instruction:
diff --git a/evals/tasks/agent/github.ts b/evals/tasks/agent/github.ts
@@ -5,17 +5,11 @@ export const github: EvalFunction = async ({
   sessionUrl,
   stagehand,
   logger,
-  modelName,
+  agent,
 }) => {
   try {
     await stagehand.page.goto("https://github.com/");
 
-    const agent = stagehand.agent({
-      model: modelName,
-      provider: modelName.startsWith("claude") ? "anthropic" : "openai",
-      instructions: `You are a helpful assistant that can help me with my tasks. You are given a task and you need to complete it without asking follow up questions. The current page is ${await stagehand.page.title()}`,
-    });
-
     const agentResult = await agent.execute({
       instruction:
         "Find a Ruby repository on GitHub that has been updated in the past 3 days and has at least 1000 stars.",
diff --git a/evals/tasks/agent/github_react_version.ts b/evals/tasks/agent/github_react_version.ts
@@ -5,15 +5,10 @@ export const github_react_version: EvalFunction = async ({
   sessionUrl,
   stagehand,
   logger,
-  modelName,
+  agent,
 }) => {
   try {
     await stagehand.page.goto("https://github.com/");
-    const agent = stagehand.agent({
-      model: modelName,
-      provider: modelName.startsWith("claude") ? "anthropic" : "openai",
-      instructions: `You are a helpful web automation assistant. DON'T ASK FOLLOW UP QUESTIONS UNTIL YOU HAVE FULFILLED THE USER'S REQUEST. Today is ${new Date().toLocaleDateString()}.`,
-    });
 
     const agentResult = await agent.execute({
       instruction:
diff --git a/evals/tasks/agent/google_flights.ts b/evals/tasks/agent/google_flights.ts
@@ -6,17 +6,11 @@ export const google_flights: EvalFunction = async ({
   sessionUrl,
   stagehand,
   logger,
-  modelName,
+  agent,
 }) => {
   try {
     await stagehand.page.goto("https://google.com/travel/flights");
 
-    const agent = stagehand.agent({
-      model: modelName,
-      provider: modelName.startsWith("claude") ? "anthropic" : "openai",
-      instructions: `You are a helpful assistant that can help me with my tasks. You are given a task and you need to complete it without asking follow up questions. Today is ${new Date().toISOString().slice(0, 10)}. The current page is ${await stagehand.page.title()}`,
-    });
-
     const agentResult = await agent.execute({
       instruction:
         "Search for flights from San Francisco to New York for next weekend",
diff --git a/evals/tasks/agent/google_maps.ts b/evals/tasks/agent/google_maps.ts
@@ -6,17 +6,11 @@ export const google_maps: EvalFunction = async ({
   sessionUrl,
   stagehand,
   logger,
-  modelName,
+  agent,
 }) => {
   try {
     await stagehand.page.goto("https://maps.google.com");
 
-    const agent = stagehand.agent({
-      model: modelName,
-      provider: modelName.startsWith("claude") ? "anthropic" : "openai",
-      instructions: `You are a helpful assistant that can help me with my tasks. You are given a task and you need to complete it without asking follow up questions. The current page is ${await stagehand.page.title()}`,
-    });
-
     const agentResult = await agent.execute({
       instruction:
         "How long does it take to get from San Francisco to New York driving?",
diff --git a/evals/tasks/agent/google_maps_2.ts b/evals/tasks/agent/google_maps_2.ts
@@ -7,17 +7,11 @@ export const google_maps_2: EvalFunction = async ({
   sessionUrl,
   stagehand,
   logger,
-  modelName,
+  agent,
 }) => {
   try {
     await stagehand.page.goto("https://maps.google.com");
 
-    const agent = stagehand.agent({
-      model: modelName,
-      provider: modelName.startsWith("claude") ? "anthropic" : "openai",
-      instructions: `You are a helpful assistant that can help me with my tasks. You are given a task and you need to complete it without asking follow up questions. The current page is ${await stagehand.page.title()}`,
-    });
-
     const agentResult = await agent.execute({
       instruction:
         "Search for the fastest walking route from La Puerta de Alcalá to La Puerta del Sol",
diff --git a/evals/tasks/agent/google_maps_3.ts b/evals/tasks/agent/google_maps_3.ts
@@ -5,15 +5,10 @@ export const google_maps_3: EvalFunction = async ({
   sessionUrl,
   stagehand,
   logger,
-  modelName,
+  agent,
 }) => {
   try {
     await stagehand.page.goto("https://maps.google.com/");
-    const agent = stagehand.agent({
-      model: modelName,
-      provider: modelName.startsWith("claude") ? "anthropic" : "openai",
-      instructions: `You are a helpful web automation assistant. DON'T ASK FOLLOW UP QUESTIONS UNTIL YOU HAVE FULFILLED THE USER'S REQUEST. Today is ${new Date().toLocaleDateString()}.`,
-    });
 
     const agentResult = await agent.execute({
       instruction:
diff --git a/evals/tasks/agent/google_shopping.ts b/evals/tasks/agent/google_shopping.ts
@@ -5,17 +5,11 @@ export const google_shopping: EvalFunction = async ({
   sessionUrl,
   stagehand,
   logger,
-  modelName,
+  agent,
 }) => {
   try {
     await stagehand.page.goto("https://www.google.com/shopping");
 
-    const agent = stagehand.agent({
-      model: modelName,
-      provider: modelName.startsWith("claude") ? "anthropic" : "openai",
-      instructions: `You are a helpful assistant that can help me with my tasks. You are given a task and you need to complete it without asking follow up questions. The current page is ${await stagehand.page.title()}`,
-    });
-
     const agentResult = await agent.execute({
       instruction:
         "Create a list of drip coffee makers that are on sale and within $25-60 and have a black finish.",
diff --git a/evals/tasks/agent/hotel_booking.ts b/evals/tasks/agent/hotel_booking.ts
@@ -5,17 +5,11 @@ export const hotel_booking: EvalFunction = async ({
   sessionUrl,
   stagehand,
   logger,
-  modelName,
+  agent,
 }) => {
   try {
     await stagehand.page.goto("https://www.booking.com/");
 
-    const agent = stagehand.agent({
-      model: modelName,
-      provider: modelName.startsWith("claude") ? "anthropic" : "openai",
-      instructions: `You are a helpful assistant that can help me with my tasks. You are given a task and you need to complete it without asking follow up questions. The current page is ${await stagehand.page.title()}`,
-    });
-
     const agentResult = await agent.execute({
       instruction:
         "Find a hotel in Sydney with a rating of 8 or higher, providing free Wi-Fi and parking, available for a four-night stay starting on December 10, 2025.",
diff --git a/evals/tasks/agent/hugging_face.ts b/evals/tasks/agent/hugging_face.ts
@@ -5,15 +5,10 @@ export const hugging_face: EvalFunction = async ({
   sessionUrl,
   stagehand,
   logger,
-  modelName,
+  agent,
 }) => {
   try {
     await stagehand.page.goto("https://huggingface.co/");
-    const agent = stagehand.agent({
-      model: modelName,
-      provider: modelName.startsWith("claude") ? "anthropic" : "openai",
-      instructions: `You are a helpful web automation assistant. DON'T ASK FOLLOW UP QUESTIONS UNTIL YOU HAVE FULFILLED THE USER'S REQUEST. Today is ${new Date().toLocaleDateString()}.`,
-    });
 
     const agentResult = await agent.execute({
       instruction:
diff --git a/evals/tasks/agent/iframe_form.ts b/evals/tasks/agent/iframe_form.ts
@@ -6,16 +6,11 @@ export const iframe_form: EvalFunction = async ({
   sessionUrl,
   stagehand,
   logger,
-  modelName,
+  agent,
 }) => {
   try {
     await stagehand.page.goto("https://tucowsdomains.com/abuse-form/phishing/");
 
-    const agent = stagehand.agent({
-      provider: "anthropic",
-      model: modelName,
-    });
-
     const agentResult = await agent.execute({
       instruction: "Fill in the form name with 'John Smith'",
       maxSteps: 3,
diff --git a/evals/tasks/agent/iframe_form_multiple.ts b/evals/tasks/agent/iframe_form_multiple.ts
@@ -6,16 +6,11 @@ export const iframe_form_multiple: EvalFunction = async ({
   sessionUrl,
   stagehand,
   logger,
-  modelName,
+  agent,
 }) => {
   try {
     await stagehand.page.goto("https://tucowsdomains.com/abuse-form/phishing/");
 
-    const agent = stagehand.agent({
-      provider: modelName.startsWith("claude") ? "anthropic" : "openai",
-      model: modelName,
-    });
-
     const agentResult = await agent.execute({
       instruction:
         "Fill in the form name with 'John Smith', the email with 'john.smith@example.com', and select the 'Are you the domain owner?' option as 'No'",
diff --git a/evals/tasks/agent/kayak.ts b/evals/tasks/agent/kayak.ts
@@ -6,18 +6,12 @@ export const kayak: EvalFunction = async ({
   sessionUrl,
   stagehand,
   logger,
+  agent,
 }) => {
   try {
     const evaluator = new Evaluator(stagehand);
     await stagehand.page.goto("https://www.kayak.com");
-    const agent = stagehand.agent({
-      provider: "openai",
-      model: "computer-use-preview",
-      instructions: `You are a helpful assistant that can help me find flights. DON'T ASK FOLLOW UP QUESTIONS UNTIL YOU HAVE FULFILLED THE USER'S REQUEST. Today is ${new Date().toLocaleDateString()}.`,
-      options: {
-        apiKey: process.env.OPENAI_API_KEY,
-      },
-    });
+
     await agent.execute({
       instruction: "Find flights from San Francisco to Tokyo next week",
       maxSteps: 15,
diff --git a/evals/tasks/agent/kith.ts b/evals/tasks/agent/kith.ts
@@ -6,18 +6,13 @@ export const kith: EvalFunction = async ({
   sessionUrl,
   stagehand,
   logger,
-  modelName,
+  agent,
 }) => {
   try {
     const evaluator = new Evaluator(stagehand);
     await stagehand.page.goto(
       "https://kith.com/collections/nike-air-force-1/products/nkcw2288-111?variant=19439468707968",
     );
-    const agent = stagehand.agent({
-      model: modelName,
-      provider: modelName.startsWith("claude") ? "anthropic" : "openai",
-      instructions: `You are a helpful assistant that can help me order shoes from kith. DON'T ASK FOLLOW UP QUESTIONS UNTIL YOU HAVE FULFILLED THE USER'S REQUEST. Today is ${new Date().toLocaleDateString()}.`,
-    });
 
     await agent.execute({
       instruction:
diff --git a/evals/tasks/agent/nba_trades.ts b/evals/tasks/agent/nba_trades.ts
@@ -5,17 +5,11 @@ export const nba_trades: EvalFunction = async ({
   sessionUrl,
   stagehand,
   logger,
-  modelName,
+  agent,
 }) => {
   try {
     await stagehand.page.goto("https://www.espn.com/");
 
-    const agent = stagehand.agent({
-      model: modelName,
-      provider: modelName.startsWith("claude") ? "anthropic" : "openai",
-      instructions: `You are a helpful assistant that can help me with my tasks. You are given a task and you need to complete it without asking follow up questions. The current page is ${await stagehand.page.title()}`,
-    });
-
     const agentResult = await agent.execute({
       instruction:
         "Find the latest Team transactions in the NBA within the past week.",
diff --git a/evals/tasks/agent/sf_library_card.ts b/evals/tasks/agent/sf_library_card.ts
diff --git a/evals/tasks/agent/sf_library_card_multiple.ts b/evals/tasks/agent/sf_library_card_multiple.ts
diff --git a/evals/tasks/agent/sign_in.ts b/evals/tasks/agent/sign_in.ts
diff --git a/evals/tasks/agent/steam_games.ts b/evals/tasks/agent/steam_games.ts
diff --git a/evals/tasks/agent/trivago.ts b/evals/tasks/agent/trivago.ts
diff --git a/evals/tasks/agent/ubereats.ts b/evals/tasks/agent/ubereats.ts
diff --git a/evals/tasks/agent/wolframalpha_weight_loss.ts b/evals/tasks/agent/wolframalpha_weight_loss.ts
diff --git a/evals/tasks/agent/youtube.ts b/evals/tasks/agent/youtube.ts
