Patch a few agent evals (#1055)

miguelg719 · tkattkat · web-flow · commit 6ae809d921fd · 2025-09-05T15:17:01.000-07:00
# why

# what changed

# test plan

---------

Co-authored-by: tkattkat &lt;48974763+tkattkat@users.noreply.github.com&gt;
Co-authored-by: tkattkat &lt;tkat@tkat.net&gt;
diff --git a/evals/tasks/agent/all_recipes.ts b/evals/tasks/agent/all_recipes.ts
@@ -24,7 +24,6 @@ export const all_recipes: EvalFunction = async ({
     logger.log(agentResult);
 
     const success =
-      agentResult.success &&
       evaluation === "YES" &&
       stagehand.page.url() ===
         "https://www.allrecipes.com/recipe/16899/beef-wellington/";
diff --git a/evals/tasks/agent/apple_trade_in.ts b/evals/tasks/agent/apple_trade_in.ts
@@ -12,7 +12,7 @@ export const apple_trade_in: EvalFunction = async ({
   try {
     await stagehand.page.goto("https://www.apple.com/shop/trade-in");
     const evaluator = new Evaluator(stagehand);
-    const agentResult = await agent.execute({
+    await agent.execute({
       instruction:
         "Find out the trade-in value for an iPhone 13 Pro Max in good condition on the Apple website.",
       maxSteps: Number(process.env.AGENT_EVAL_MAX_STEPS) || 30,
@@ -25,7 +25,7 @@ export const apple_trade_in: EvalFunction = async ({
       answer: "360",
     });
 
-    const success = agentResult.success && evaluation === "YES";
+    const success = evaluation === "YES";
 
     if (!success) {
       return {
diff --git a/evals/tasks/agent/apple_tv.ts b/evals/tasks/agent/apple_tv.ts
@@ -1,6 +1,6 @@
 import { EvalFunction } from "@/types/evals";
-import { z } from "zod";
 
+import { Evaluator } from "../../evaluator";
 export const apple_tv: EvalFunction = async ({
   debugUrl,
   sessionUrl,
@@ -14,24 +14,19 @@ export const apple_tv: EvalFunction = async ({
     const agentResult = await agent.execute({
       instruction:
         "Identify the size and weight for the Apple TV 4K and list the Siri Remote features introduced.",
-      maxSteps: Number(process.env.AGENT_EVAL_MAX_STEPS) || 30,
+      maxSteps: Number(process.env.AGENT_EVAL_MAX_STEPS) || 50,
     });
 
-    const { height, width } = await stagehand.page.extract({
-      modelName: "google/gemini-2.5-flash",
-      instruction: "Extract the size and weight of the Apple TV 4K",
-      schema: z.object({
-        height: z.number().describe("The height of the Apple TV 4K in inches"),
-        width: z.number().describe("The width of the Apple TV 4K in inches"),
-      }),
+    const evaluator = new Evaluator(stagehand);
+    const result = await evaluator.ask({
+      question:
+        "did the agent find the height and width of the Apple TV 4K in its reasoning which is 1.2 and 3.66?",
+      answer: agentResult.message,
     });
 
     const success =
-      agentResult.success &&
-      height === 1.2 &&
-      width === 3.66 &&
+      result.evaluation === "YES" &&
       stagehand.page.url().includes("https://www.apple.com/apple-tv-4k/specs/");
-
     if (!success) {
       return {
         _success: false,
diff --git a/evals/tasks/agent/arxiv_gpt_report.ts b/evals/tasks/agent/arxiv_gpt_report.ts
@@ -12,7 +12,7 @@ export const arxiv_gpt_report: EvalFunction = async ({
     const evaluator = new Evaluator(stagehand);
     await stagehand.page.goto("https://arxiv.org/");
 
-    const agentResult = await agent.execute({
+    await agent.execute({
       instruction:
         "Find the paper 'GPT-4 Technical Report', when was v3 submitted?",
       maxSteps: Number(process.env.AGENT_EVAL_MAX_STEPS) || 25,
@@ -29,7 +29,7 @@ export const arxiv_gpt_report: EvalFunction = async ({
 
     console.log(`reasoning: ${reasoning}`);
 
-    const success = agentResult.success && evaluation === "YES";
+    const success = evaluation === "YES";
 
     if (!success) {
       return {
diff --git a/evals/tasks/agent/github.ts b/evals/tasks/agent/github.ts
@@ -22,7 +22,7 @@ export const github: EvalFunction = async ({
         "Ruby repository on GitHub that has been updated in the past 3 days and has at least 1000 stars.",
     });
 
-    const success = agentResult.success && evaluation === "YES";
+    const success = evaluation === "YES";
 
     if (!success) {
       return {
diff --git a/evals/tasks/agent/google_maps_2.ts b/evals/tasks/agent/google_maps_2.ts
@@ -46,7 +46,7 @@ export const google_maps_2: EvalFunction = async ({
     }
 
     if (result.evaluation === "YES") {
-      if (distance !== 1.5) {
+      if (distance <= 1.3 || distance >= 1.6) {
         return {
           _success: false,
           observations: "Distance is not 1.5 km",
diff --git a/evals/tasks/agent/google_maps_3.ts b/evals/tasks/agent/google_maps_3.ts
@@ -10,7 +10,7 @@ export const google_maps_3: EvalFunction = async ({
   try {
     await stagehand.page.goto("https://maps.google.com/");
     const evaluator = new Evaluator(stagehand);
-    const agentResult = await agent.execute({
+    await agent.execute({
       instruction:
         "Search for locksmiths open now but not open 24 hours in Texas City.",
       maxSteps: Number(process.env.AGENT_EVAL_MAX_STEPS) || 35,
@@ -21,7 +21,7 @@ export const google_maps_3: EvalFunction = async ({
         "Does the page show a locksmiths open now but not open 24 hours in Texas City?",
     });
 
-    const success = agentResult.success && evaluation === "YES";
+    const success = evaluation === "YES";
 
     if (!success) {
       return {
diff --git a/evals/tasks/agent/google_shopping.ts b/evals/tasks/agent/google_shopping.ts
@@ -24,7 +24,7 @@ export const google_shopping: EvalFunction = async ({
         "Does the page show a drip coffee maker that is on sale and within $25-60 and has a black finish?",
     });
 
-    const success = agentResult.success && evaluation === "YES";
+    const success = evaluation === "YES";
 
     if (!success) {
       return {
diff --git a/evals/tasks/agent/hotel_booking.ts b/evals/tasks/agent/hotel_booking.ts
@@ -24,7 +24,7 @@ export const hotel_booking: EvalFunction = async ({
         "Does the page show a hotel in Sydney with a rating of 8 or higher, providing free Wi-Fi and parking, available for a four-night stay starting on December 10, 2025?",
     });
 
-    const success = agentResult.success && evaluation === "YES";
+    const success = evaluation === "YES";
 
     if (!success) {
       return {
diff --git a/evals/tasks/agent/hugging_face.ts b/evals/tasks/agent/hugging_face.ts
@@ -24,7 +24,7 @@ export const hugging_face: EvalFunction = async ({
       screenshot: false,
     });
 
-    const success = agentResult.success && evaluation === "YES";
+    const success = evaluation === "YES";
 
     console.log(`reasoning: ${reasoning}`);
     if (!success) {
diff --git a/evals/tasks/agent/iframe_form.ts b/evals/tasks/agent/iframe_form.ts
@@ -9,15 +9,16 @@ export const iframe_form: EvalFunction = async ({
   agent,
 }) => {
   try {
-    await stagehand.page.goto("https://tucowsdomains.com/abuse-form/phishing/");
+    await stagehand.page.goto(
+      "https://browserbase.github.io/stagehand-eval-sites/sites/iframe-form-filling/",
+    );
 
     const agentResult = await agent.execute({
       instruction: "Fill in the form name with 'John Smith'",
-      maxSteps: Number(process.env.AGENT_EVAL_MAX_STEPS) || 3,
+      maxSteps: Number(process.env.AGENT_EVAL_MAX_STEPS) || 5,
     });
     logger.log(agentResult);
 
-    await stagehand.page.mouse.wheel(0, -1000);
     const evaluator = new Evaluator(stagehand);
     const result = await evaluator.ask({
       question: "Is the form name input filled with 'John Smith'?",
diff --git a/evals/tasks/agent/iframe_form_multiple.ts b/evals/tasks/agent/iframe_form_multiple.ts
@@ -9,7 +9,9 @@ export const iframe_form_multiple: EvalFunction = async ({
   agent,
 }) => {
   try {
-    await stagehand.page.goto("https://tucowsdomains.com/abuse-form/phishing/");
+    await stagehand.page.goto(
+      "https://browserbase.github.io/stagehand-eval-sites/sites/iframe-form-filling/",
+    );
 
     const agentResult = await agent.execute({
       instruction:
@@ -27,10 +29,6 @@ export const iframe_form_multiple: EvalFunction = async ({
           question:
             "Is the form email input filled with 'john.smith@example.com'?",
         },
-        {
-          question:
-            "Is the 'Are you the domain owner?' option selected as 'No'?",
-        },
       ],
     });
 
diff --git a/evals/tasks/agent/kayak.ts b/evals/tasks/agent/kayak.ts
@@ -32,7 +32,7 @@ export const kayak: EvalFunction = async ({
     }
     const { evaluation, reasoning } = await evaluator.ask({
       question:
-        "Are the flights shown sorted by price? Check the sort button in the top left corner of the page",
+        "Are the flights shown sorted by price? Check the sort button in the top left corner of the page. It should show cheapest first; use this as the success criteria since the page might promote other flights and not show the list in order.",
     });
 
     const success = evaluation === "YES";
diff --git a/evals/tasks/agent/kith.ts b/evals/tasks/agent/kith.ts
@@ -16,7 +16,7 @@ export const kith: EvalFunction = async ({
 
     await agent.execute({
       instruction:
-        "add the shoes to cart, go to checkout, and fill the delivery information",
+        "add the shoes to cart, go to checkout, and fill the delivery information. Don't fill the payment information",
       maxSteps: Number(process.env.AGENT_EVAL_MAX_STEPS) || 25,
     });
 
@@ -28,7 +28,8 @@ export const kith: EvalFunction = async ({
 
     if (success) {
       await agent.execute({
-        instruction: "fill the credit card information",
+        instruction:
+          "fill the credit card information, do not submit the order just add placeholders",
         maxSteps: Number(process.env.AGENT_EVAL_MAX_STEPS) || 10,
       });
 
diff --git a/evals/tasks/agent/nba_trades.ts b/evals/tasks/agent/nba_trades.ts
@@ -23,7 +23,6 @@ export const nba_trades: EvalFunction = async ({
     });
 
     const success =
-      agentResult.success &&
       stagehand.page.url() === "https://www.espn.com/nba/transactions" &&
       evaluation === "YES";
 
diff --git a/evals/tasks/agent/steam_games.ts b/evals/tasks/agent/steam_games.ts
@@ -17,9 +17,9 @@ export const steam_games: EvalFunction = async ({
     });
 
     //strictly used url check and no extract as the top games / players can vary
-    const success =
-      agentResult.success &&
-      stagehand.page.url().includes("https://store.steampowered.com/");
+    const success = stagehand.page
+      .url()
+      .includes("https://store.steampowered.com/");
 
     if (!success) {
       return {
diff --git a/evals/tasks/agent/youtube.ts b/evals/tasks/agent/youtube.ts
@@ -12,7 +12,7 @@ export const youtube: EvalFunction = async ({
 
     const agentResult = await agent.execute({
       instruction:
-        "Search for Keinemusik's set under some very famous pointy landmarks",
+        "Search for Keinemusik's set under some very famous pointy landmarks. Make sure to click on the video",
       maxSteps: Number(process.env.AGENT_EVAL_MAX_STEPS) || 15,
     });
     logger.log(agentResult);
diff --git a/evals/tasks/observe_iframes1.ts b/evals/tasks/observe_iframes1.ts
@@ -7,7 +7,9 @@ export const observe_iframes1: EvalFunction = async ({
   logger,
 }) => {
   try {
-    await stagehand.page.goto("https://tucowsdomains.com/abuse-form/phishing/");
+    await stagehand.page.goto(
+      "https://browserbase.github.io/stagehand-eval-sites/sites/iframe-hn/",
+    );
 
     const observations = await stagehand.page.observe({
       instruction: "find the main header of the page",
@@ -24,8 +26,8 @@ export const observe_iframes1: EvalFunction = async ({
     }
 
     const possibleLocators = [
-      `#primary > div.singlePage > section > div > div > article > div > iframe`,
-      `#primary > div.heroBanner > section > div > h1`,
+      `body > main > section.iframe-wrapper > iframe`,
+      `body > header > h1`,
     ];
 
     const possibleHandles = [];
diff --git a/evals/tasks/observe_vantechjournal.ts b/evals/tasks/observe_vantechjournal.ts
@@ -24,7 +24,7 @@ export const observe_vantechjournal: EvalFunction = async ({
       };
     }
 
-    const expectedLocator = `xpath=/html/body/div[3]/section/div/div/div[3]/a`;
+    const expectedLocator = `xpath=/html/body/div[3]/div/section/div/div/div[3]/a`;
 
     const expectedResult = await stagehand.page.locator(expectedLocator);
 

Original file line number	Diff line number	Diff line change
`@@ -46,7 +46,7 @@ export const google_maps_2: EvalFunction = async ({`
`46`	`46`	`}`
`47`	`47`
`48`	`48`	`if (result.evaluation === "YES") {`
`49`		`- if (distance !== 1.5) {`
	`49`	`+ if (distance <= 1.3 \|\| distance >= 1.6) {`
`50`	`50`	`return {`
`51`	`51`	`_success: false,`
`52`	`52`	`observations: "Distance is not 1.5 km",`