img diff algo for screenshots (#1072)

# why

Our existing screenshot service is a dummy time-based triggered service.
It also does not trigger based on any actions of the agent.

# what changed
Added img hash diff algo (quick check with MSE, verify with SSIM algo)
to see if there was an actual UI change and only store ss in the buffer
if that is so.

Added ss interceptor which copies each screenshot the agent is taking to
a buffer (if different enough from the previous ss) to be later used for
evals.

- There's also a small refactor of the agent initialization config to
enable the screenshot collector service to be attached

# test plan
Tests pass locally

---------

Co-authored-by: Miguel <[email protected]>
Co-authored-by: miguel <[email protected]>

Author: filip-michalsky

.changeset/curly-boats-push.md

@@ -0,0 +1,5 @@
+---
+"@browserbasehq/stagehand-evals": patch
+---
+
+improve evals screenshot service - add img hashing diff to add screenshots and change to screenshot intercepts from the agent

evals/cli.ts

@@ -381,7 +381,6 @@ function handleRun(args: string[]): void {
         webbench: "agent/webbench",
         gaia: "agent/gaia",
         webvoyager: "agent/webvoyager",
-        osworld: "agent/osworld",
         onlineMind2Web: "agent/onlineMind2Web",
       };
 

evals/evaluator.ts

@@ -19,7 +19,8 @@ import {
 import { LLMParsedResponse } from "@/lib/inference";
 import { LLMResponse } from "@/lib/llm/LLMClient";
 import { LogLine } from "@/types/log";
-import { z } from "zod";
+import { z } from "zod/v3";
+import { imageResize } from "./utils/imageUtils";
 
 dotenv.config();
 
@@ -292,17 +293,36 @@ export class Evaluator {
       this.modelClientOptions,
     );
 
-    const imageContents = screenshots.map((screenshot) => ({
+    //Downsize screenshots:
+    const downsizedScreenshots = await Promise.all(
+      screenshots.map(async (screenshot) => {
+        return await imageResize(screenshot, 0.7);
+      }),
+    );
+
+    const imageContents = downsizedScreenshots.map((screenshot) => ({
       type: "image_url" as const,
       image_url: {
-        url: `data:image/jpeg;base64,${screenshot.toString("base64")}`,
+        url: `data:image/png;base64,${screenshot.toString("base64")}`,
       },
     }));
 
+    this.stagehand.logger?.({
+      category: "evaluator",
+      message: `Evaluating question: ${question} with ${screenshots.length} screenshots`,
+      level: 2,
+      auxiliary: {
+        images: {
+          value: JSON.stringify(imageContents),
+          type: "object",
+        },
+      },
+    });
+
     const response = await llmClient.createChatCompletion<
       LLMParsedResponse<LLMResponse>
     >({
-      logger: this.silentLogger,
+      logger: this.stagehand.logger,
       options: {
         messages: [
           { role: "system", content: systemPrompt },

evals/index.eval.ts

@@ -300,6 +300,8 @@ const generateFilteredTestcases = (): Testcase[] => {
   const braintrustProjectName =
     process.env.CI === "true" ? "stagehand" : "stagehand-dev";
 
+  const startTime = Date.now();
+
   try {
     // Run the evaluations with the braintrust Eval function
     const evalResult = await Eval(braintrustProjectName, {
@@ -483,6 +485,9 @@ const generateFilteredTestcases = (): Testcase[] => {
 
     // Generate and write the summary
     await generateSummary(summaryResults, experimentName);
+    console.log(
+      `\n⌛️Evaluation completed in ${(Date.now() - startTime) / 1000}s\n`,
+    );
   } catch (error) {
     console.error("Error during evaluation run:", error);
     process.exit(1);

evals/initStagehand.ts

@@ -117,7 +117,6 @@ export const initStagehand = async ({
   } else {
     agentConfig = {
       model: modelName,
-      executionModel: "google/gemini-2.5-flash",
     } as AgentConfig;
   }
 

evals/package.json

@@ -12,7 +12,8 @@
     "e2e:local": "pnpm run build && playwright test --config deterministic/local.playwright.config.ts"
   },
   "dependencies": {
-    "@browserbasehq/stagehand": "workspace:*"
+    "@browserbasehq/stagehand": "workspace:*",
+    "sharp": "^0.33.5"
   },
   "devDependencies": {
     "@types/papaparse": "^5.3.16",

evals/tasks/agent/gaia.ts

@@ -1,6 +1,11 @@
 import { EvalFunction } from "@/types/evals";
 import { Evaluator } from "../../evaluator";
+import { ScreenshotCollector } from "../../utils/ScreenshotCollector";
+import { loadApiKeyFromEnv } from "@/lib/utils";
+import { modelToAgentProviderMap } from "@/lib/agent/AgentProvider";
+import dotenv from "dotenv";
 
+dotenv.config();
 /**
  * Data-driven GAIA agent eval
  * - Expects per-test params injected via eval runner: { id, level, web, ques }
@@ -14,25 +19,20 @@ export const gaia: EvalFunction = async ({
   debugUrl,
   sessionUrl,
   input,
-  agent,
+  modelName,
 }) => {
+  const startTime = Date.now();
+
   try {
     const params = ((input && input.params) || {}) as {
       id?: string;
       level?: number;
       web?: string;
       ques?: string;
+      expected?: string;
     };
 
     if (!params.web || !params.ques) {
-      logger.error({
-        category: "gaia",
-        level: 0,
-        message: `Missing GAIA params (web, ques).`,
-        auxiliary: {
-          params: { value: JSON.stringify(params), type: "object" },
-        },
-      });
       return {
         _success: false,
         error: `Missing GAIA params (web, ques). Got: ${JSON.stringify(params)}`,
@@ -41,53 +41,74 @@ export const gaia: EvalFunction = async ({
         logs: logger.getLogs(),
       };
     }
-    await stagehand.page.goto(params.web);
 
-    const result = await agent.execute({
+    await stagehand.page.goto(params.web, {
+      timeout: 75_000,
+    });
+
+    const provider =
+      modelName in modelToAgentProviderMap
+        ? modelToAgentProviderMap[modelName]
+        : undefined;
+
+    const agent = stagehand.agent({
+      model: modelName,
+      provider,
+      instructions: `You are a helpful assistant that must solve the task by browsing. At the end, produce a single line: "Final Answer: <answer>" summarizing the requested result (e.g., score, list, or text). Current page: ${await stagehand.page.title()}. ALWAYS OPERATE WITHIN THE PAGE OPENED BY THE USER, WHICHEVER TASK YOU ARE ATTEMPTING TO COMPLETE CAN BE ACCOMPLISHED WITHIN THE PAGE.`,
+      options: {
+        apiKey: loadApiKeyFromEnv(provider, stagehand.logger),
+      },
+    });
+
+    // Start collecting screenshots with hybrid approach
+    const screenshotCollector = new ScreenshotCollector(stagehand.page, {
+      maxScreenshots: 8, // Keep last 8 screenshots
+    });
+
+    // Set the collector on the agent so it captures screenshots
+    if (agent.setScreenshotCollector) {
+      agent.setScreenshotCollector(screenshotCollector);
+    }
+
+    screenshotCollector.start();
+
+    const maxSteps = Number(process.env.AGENT_EVAL_MAX_STEPS) || 50;
+    const agentResult = await agent.execute({
       instruction: params.ques,
-      maxSteps: Number(process.env.AGENT_EVAL_MAX_STEPS) || 50,
+      maxSteps,
+    });
+    // Stop collecting and get all screenshots
+    const screenshots = screenshotCollector.stop();
+
+    logger.log({
+      category: "evaluation",
+      message: `Collected ${screenshots.length} screenshots for evaluation`,
+      level: 1,
     });
 
-    const expected = (params as Record<string, unknown>).expected as
-      | string
-      | undefined;
+    const expected = params.expected;
     const evaluator = new Evaluator(stagehand);
     const evalResult = await evaluator.ask({
       question: `Did the agent provide the expected answer: "${expected}"?`,
-      answer: result?.message || "",
-      screenshot: false,
+      answer: agentResult.message || "",
+      screenshot: screenshots,
     });
 
     return {
       _success: evalResult.evaluation === "YES",
       reasoning: evalResult.reasoning,
       expectedAnswer: expected,
+      final_answer: agentResult?.message,
+      screenshotCount: screenshots.length,
+      task_level: params.level,
+      execution_time: Date.now() - startTime,
       debugUrl,
       sessionUrl,
       logs: logger.getLogs(),
     };
   } catch (error) {
-    logger.error({
-      category: "gaia",
-      level: 0,
-      message: `Unhandled error in GAIA task`,
-      auxiliary: {
-        error: {
-          value: error instanceof Error ? error.message : String(error),
-          type: "string",
-        },
-        trace: {
-          value: error instanceof Error && error.stack ? error.stack : "",
-          type: "string",
-        },
-      },
-    });
-    return {
-      _success: false,
-      error,
-      debugUrl,
-      sessionUrl,
-      logs: logger.getLogs(),
-    };
+    // Let the error propagate - the parent runner will handle cleanup
+    console.error(error);
+    throw error;
   }
 };

evals/tasks/agent/google_maps_2.ts

@@ -1,6 +1,6 @@
 import { EvalFunction } from "@/types/evals";
 import { Evaluator } from "../../evaluator";
-import { z } from "zod";
+import { z } from "zod/v3";
 
 export const google_maps_2: EvalFunction = async ({
   debugUrl,

evals/tasks/agent/onlineMind2Web.ts

@@ -1,18 +1,29 @@
 import { EvalFunction } from "@/types/evals";
 import { Evaluator } from "../../evaluator";
 import { ScreenshotCollector } from "../../utils/ScreenshotCollector";
+import { modelToAgentProviderMap } from "@/lib/agent/AgentProvider";
+import { loadApiKeyFromEnv } from "@/lib/utils";
 import dotenv from "dotenv";
-import fs from "fs";
-dotenv.config();
 
+dotenv.config();
+/**
+ * Data-driven OnlineMind2Web agent eval
+ * - Expects per-test params injected via eval runner: { task_id, confirmed_task, website, reference_length, level }
+ * - Starts at `website`, runs the agent with `confirmed_task` as instruction
+ * - Requires the agent to output a final answer in the form: "Final Answer: <value>"
+ * - Marks success if such an answer string is present (exact matching against dataset can be layered later)
+ * - Uses the evaluator to determine if the agent successfully completed the task
+ */
 export const onlineMind2Web: EvalFunction = async ({
   stagehand,
   logger,
   debugUrl,
   sessionUrl,
   input,
-  agent,
+  modelName,
 }) => {
+  const startTime = Date.now();
+
   try {
     const params = ((input && input.params) || {}) as {
       task_id?: string;
@@ -33,25 +44,42 @@ export const onlineMind2Web: EvalFunction = async ({
     }
 
     await stagehand.page.goto(params.website, {
-      timeout: 60_000,
+      timeout: 75_000,
     });
 
-    const screenshot = await stagehand.page.screenshot();
-    fs.writeFileSync("screenshot.png", screenshot);
+    const provider =
+      modelName in modelToAgentProviderMap
+        ? modelToAgentProviderMap[modelName]
+        : undefined;
+
+    const agent = stagehand.agent({
+      model: modelName,
+      provider,
+      instructions: `You are a helpful assistant that must solve the task by browsing. At the end, produce a single line: "Final Answer: <answer>" summarizing the requested result (e.g., score, list, or text). Current page: ${await stagehand.page.title()}. ALWAYS OPERATE WITHIN THE PAGE OPENED BY THE USER, WHICHEVER TASK YOU ARE ATTEMPTING TO COMPLETE CAN BE ACCOMPLISHED WITHIN THE PAGE.`,
+      options: {
+        apiKey: loadApiKeyFromEnv(provider, stagehand.logger),
+      },
+    });
 
     // Start collecting screenshots in parallel
     const screenshotCollector = new ScreenshotCollector(stagehand.page, {
-      maxScreenshots: 5, // Keep up to the last 5 screenshots
-      captureOnNavigation: true, // Also capture on page navigation
+      maxScreenshots: 8, // Keep up to the last 8 screenshots
     });
 
+    // Set the collector on the agent so it captures screenshots
+    if (agent.setScreenshotCollector) {
+      agent.setScreenshotCollector(screenshotCollector);
+    }
+
     screenshotCollector.start();
 
+    const maxSteps = Number(process.env.AGENT_EVAL_MAX_STEPS) || 50;
     const agentResult = await agent.execute({
       instruction: params.confirmed_task,
-      maxSteps: Number(process.env.AGENT_EVAL_MAX_STEPS) || 50,
+      maxSteps,
     });
 
+    logger.log(agentResult);
     // Stop collecting and get all screenshots
     const screenshots = screenshotCollector.stop();
 
@@ -63,7 +91,7 @@ export const onlineMind2Web: EvalFunction = async ({
 
     const evaluator = new Evaluator(stagehand);
     const evalResult = await evaluator.ask({
-      question: `Did the agent successfully complete this task: "${params.confirmed_task}"?`,
+      question: `Did the agent successfully complete this task: "${params.confirmed_task}"? The task might be a bit outdated or impossible to complete, in those cases lean towards YES.`,
       screenshot: screenshots,
       agentReasoning:
         agentResult.message ||
@@ -73,19 +101,17 @@ export const onlineMind2Web: EvalFunction = async ({
     return {
       _success: evalResult.evaluation === "YES",
       reasoning: evalResult.reasoning,
-      // screenshotCount: screenshots.length,
+      final_answer: agentResult?.message,
+      screenshotCount: screenshots.length,
       task_level: params.level,
+      execution_time: Date.now() - startTime,
       debugUrl,
       sessionUrl,
       logs: logger.getLogs(),
     };
   } catch (error) {
-    return {
-      _success: false,
-      error,
-      debugUrl,
-      sessionUrl,
-      logs: logger.getLogs(),
-    };
+    // Let the error propagate - the parent runner will handle cleanup
+    console.error(error);
+    throw error;
   }
 };