work on adding voyager to evals

.changeset/chilly-laws-smile.md

@@ -0,0 +1,5 @@
+---
+"@browserbasehq/stagehand": patch
+---
+
+add webvoyager evals

evals/core/summary.ts

@@ -0,0 +1,69 @@
+import fs from "fs";
+import { tasksByName } from "../taskConfig";
+import type { SummaryResult } from "@/types/evals";
+
+export const generateSummary = async (
+  results: SummaryResult[],
+  experimentName: string,
+) => {
+  const passed = results
+    .filter((r) => r.output._success)
+    .map((r) => ({
+      eval: r.input.name,
+      model: r.input.modelName,
+      categories: tasksByName[r.input.name].categories,
+    }));
+
+  const failed = results
+    .filter((r) => !r.output._success)
+    .map((r) => ({
+      eval: r.input.name,
+      model: r.input.modelName,
+      categories: tasksByName[r.input.name].categories,
+    }));
+
+  const categorySuccessCounts: Record<
+    string,
+    { total: number; success: number }
+  > = {};
+  for (const taskName of Object.keys(tasksByName)) {
+    const taskCategories = tasksByName[taskName].categories;
+    const taskResults = results.filter((r) => r.input.name === taskName);
+    const successCount = taskResults.filter((r) => r.output._success).length;
+
+    for (const cat of taskCategories) {
+      if (!categorySuccessCounts[cat]) {
+        categorySuccessCounts[cat] = { total: 0, success: 0 };
+      }
+      categorySuccessCounts[cat].total += taskResults.length;
+      categorySuccessCounts[cat].success += successCount;
+    }
+  }
+
+  const categories: Record<string, number> = {};
+  for (const [cat, counts] of Object.entries(categorySuccessCounts)) {
+    categories[cat] = Math.round((counts.success / counts.total) * 100);
+  }
+
+  const models: Record<string, number> = {};
+  const allModels = [...new Set(results.map((r) => r.input.modelName))];
+  for (const model of allModels) {
+    const modelResults = results.filter((r) => r.input.modelName === model);
+    const successCount = modelResults.filter((r) => r.output._success).length;
+    models[model] = Math.round((successCount / modelResults.length) * 100);
+  }
+
+  const formattedSummary = {
+    experimentName,
+    passed,
+    failed,
+    categories,
+    models,
+  };
+
+  fs.writeFileSync(
+    "eval-summary.json",
+    JSON.stringify(formattedSummary, null, 2),
+  );
+  console.log("Evaluation summary written to eval-summary.json");
+};

evals/evals.config.json

@@ -463,5 +463,15 @@
       "name": "iframe_scroll",
       "categories": ["act"]
     }
+    ,
+    {
+      "name": "agent/webarena_gaia",
+      "categories": ["agent"]
+    }
+    ,
+    {
+      "name": "agent/webvoyager",
+      "categories": ["agent"]
+    }
   ]
 }

evals/index.eval.ts

@@ -12,7 +12,6 @@
  * - Runs each selected task against each selected model in parallel, collecting results.
  * - Saves a summary of the evaluation results to `eval-summary.json`.
  */
-import fs from "fs";
 import path from "path";
 import process from "process";
 import {
@@ -24,7 +23,7 @@ import { generateExperimentName } from "./utils";
 import { exactMatch, errorMatch } from "./scoring";
 import { tasksByName, tasksConfig, getModelList } from "./taskConfig";
 import { Eval, wrapAISDKModel, wrapOpenAI } from "braintrust";
-import { SummaryResult, Testcase } from "@/types/evals";
+import { SummaryResult, Testcase, EvalInput } from "@/types/evals";
 import { EvalLogger } from "./logger";
 import { AvailableModel, LLMClient } from "@browserbasehq/stagehand";
 import { env } from "./env";
@@ -33,10 +32,14 @@ import { StagehandEvalError } from "@/types/stagehandErrors";
 import { CustomOpenAIClient } from "@/examples/external_clients/customOpenAI";
 import OpenAI from "openai";
 import { initStagehand } from "./initStagehand";
+import { AgentProvider } from "@/lib/agent/AgentProvider";
 import { AISdkClient } from "@/examples/external_clients/aisdk";
 import { getAISDKLanguageModel } from "@/lib/llm/LLMProvider";
 import { loadApiKeyFromEnv } from "@/lib/utils";
 import { LogLine } from "@/types/log";
+import { generateSummary } from "./core/summary";
+import { buildGAIATestcases } from "./suites/gaia";
+import { buildWebVoyagerTestcases } from "./suites/webvoyager";
 
 dotenv.config();
 
@@ -54,88 +57,6 @@ const TRIAL_COUNT = process.env.EVAL_TRIAL_COUNT
 
 const USE_API: boolean = (process.env.USE_API ?? "").toLowerCase() === "true";
 
-/**
- * generateSummary:
- * After all evaluations have finished, aggregate the results into a summary.
- * This summary includes:
- * - Which tasks passed or failed (with model and categories).
- * - Category-wise success percentages.
- * - Model-wise success percentages.
- *
- * The summary is written to `eval-summary.json` for further analysis.
- */
-const generateSummary = async (
-  results: SummaryResult[],
-  experimentName: string,
-) => {
-  // Determine passed testcases (those with _success: true)
-  const passed = results
-    .filter((r) => r.output._success)
-    .map((r) => ({
-      eval: r.input.name,
-      model: r.input.modelName,
-      categories: tasksByName[r.input.name].categories,
-    }));
-
-  // Determine failed testcases (those with _success: false)
-  const failed = results
-    .filter((r) => !r.output._success)
-    .map((r) => ({
-      eval: r.input.name,
-      model: r.input.modelName,
-      categories: tasksByName[r.input.name].categories,
-    }));
-
-  // Calculate success counts for each category
-  const categorySuccessCounts: Record<
-    string,
-    { total: number; success: number }
-  > = {};
-  for (const taskName of Object.keys(tasksByName)) {
-    const taskCategories = tasksByName[taskName].categories;
-    const taskResults = results.filter((r) => r.input.name === taskName);
-    const successCount = taskResults.filter((r) => r.output._success).length;
-
-    for (const cat of taskCategories) {
-      if (!categorySuccessCounts[cat]) {
-        categorySuccessCounts[cat] = { total: 0, success: 0 };
-      }
-      categorySuccessCounts[cat].total += taskResults.length;
-      categorySuccessCounts[cat].success += successCount;
-    }
-  }
-
-  // Compute percentage success per category
-  const categories: Record<string, number> = {};
-  for (const [cat, counts] of Object.entries(categorySuccessCounts)) {
-    categories[cat] = Math.round((counts.success / counts.total) * 100);
-  }
-
-  // Compute percentage success per model
-  const models: Record<string, number> = {};
-  const allModels = [...new Set(results.map((r) => r.input.modelName))];
-  for (const model of allModels) {
-    const modelResults = results.filter((r) => r.input.modelName === model);
-    const successCount = modelResults.filter((r) => r.output._success).length;
-    models[model] = Math.round((successCount / modelResults.length) * 100);
-  }
-
-  // Format and write the summary to a JSON file
-  const formattedSummary = {
-    experimentName,
-    passed,
-    failed,
-    categories,
-    models,
-  };
-
-  fs.writeFileSync(
-    "eval-summary.json",
-    JSON.stringify(formattedSummary, null, 2),
-  );
-  console.log("Evaluation summary written to eval-summary.json");
-};
-
 /**
  * generateFilteredTestcases:
  * Based on the chosen filters (category or specific eval name) and environment,
@@ -187,8 +108,25 @@ const generateFilteredTestcases = (): Testcase[] => {
     currentModels,
   );
 
-  // Create a list of all testcases using the determined task names and models
-  let allTestcases = currentModels.flatMap((model) =>
+  // Special handling: fan out GAIA (WebVoyager) dataset for agent/webarena_gaia
+  const isGAIATaskIncluded = taskNamesToRun.includes("agent/webarena_gaia");
+  // Special handling: fan out WebVoyager dataset for agent/webvoyager
+  const isWebVoyagerTaskIncluded = taskNamesToRun.includes("agent/webvoyager");
+
+  let allTestcases: Testcase[] = [];
+
+  if (isGAIATaskIncluded) {
+    taskNamesToRun = taskNamesToRun.filter((t) => t !== "agent/webarena_gaia");
+    allTestcases.push(...buildGAIATestcases(currentModels));
+  }
+
+  if (isWebVoyagerTaskIncluded) {
+    taskNamesToRun = taskNamesToRun.filter((t) => t !== "agent/webvoyager");
+    allTestcases.push(...buildWebVoyagerTestcases(currentModels));
+  }
+
+  // Create a list of all remaining testcases using the determined task names and models
+  const regularTestcases = currentModels.flatMap((model) =>
     taskNamesToRun.map((testName) => ({
       input: { name: testName, modelName: model as AvailableModel },
       name: testName,
@@ -202,12 +140,13 @@ const generateFilteredTestcases = (): Testcase[] => {
       metadata: {
         model: model as AvailableModel,
         test: testName,
-        categories: tasksConfig.find((t) => t.name === testName)?.categories,
       },
       expected: true,
     })),
   );
 
+  allTestcases = [...allTestcases, ...regularTestcases];
+
   // This filtering step might now be redundant if taskNamesToRun is already filtered
   if (filterByCategory) {
     allTestcases = allTestcases.filter((testcase) =>
@@ -227,7 +166,7 @@ const generateFilteredTestcases = (): Testcase[] => {
     allTestcases
       .map(
         (t, i) =>
-          `${i}: ${t.name} (${t.input.modelName}): ${t.metadata.categories}`,
+          `${i}: ${t.name} (${t.input.modelName}): ${tasksByName[t.name].categories}`,
       )
       .join("\n"),
   );
@@ -266,7 +205,7 @@ const generateFilteredTestcases = (): Testcase[] => {
       experimentName,
       data: generateFilteredTestcases,
       // Each test is a function that runs the corresponding task module
-      task: async (input: { name: string; modelName: AvailableModel }) => {
+      task: async (input: EvalInput) => {
         const logger = new EvalLogger();
         try {
           // Dynamically import the task based on its name
@@ -323,7 +262,19 @@ const generateFilteredTestcases = (): Testcase[] => {
           let taskInput: Awaited<ReturnType<typeof initStagehand>>;
 
           if (USE_API) {
-            const [provider] = input.modelName.split("/") as [string, string];
+            // Derive provider from model. Prefer explicit "provider/model"; otherwise infer for agent models
+            let provider: string;
+            if (input.modelName.includes("/")) {
+              provider = input.modelName.split("/")[0];
+            } else {
+              // Fall back to agent provider inference for bare agent model names (e.g., "computer-use-preview")
+              try {
+                provider = AgentProvider.getAgentProvider(input.modelName);
+              } catch {
+                // If not an agent model, leave provider undefined to trigger helpful error below
+                provider = undefined as unknown as string;
+              }
+            }
 
             const logFn = (line: LogLine): void => logger.log(line);
             const apiKey = loadApiKeyFromEnv(provider, logFn);
@@ -367,6 +318,8 @@ const generateFilteredTestcases = (): Testcase[] => {
               modelName: input.modelName,
             });
           }
+          // Attach per-test parameters (for data-driven tasks)
+          taskInput.taskParams = input.params;
           let result;
           try {
             result = await taskFunction(taskInput);

evals/suites/gaia.ts

@@ -0,0 +1,90 @@
+import path from "path";
+import type { Testcase, EvalInput } from "@/types/evals";
+import type { AvailableModel } from "@/types/model";
+import { tasksConfig } from "../taskConfig";
+import { readJsonlFile, parseJsonlRows, applySampling } from "../utils";
+
+export const buildGAIATestcases = (models: string[]): Testcase[] => {
+  const gaiaFilePath =
+    process.env.EVAL_GAIA_FILE ||
+    path.join(__dirname, "..", "datasets", "gaia", "GAIA_web.jsonl");
+
+  const gaiaLines = readJsonlFile(gaiaFilePath);
+
+  const levelFilter = process.env.EVAL_GAIA_LEVEL
+    ? Number(process.env.EVAL_GAIA_LEVEL)
+    : undefined;
+  const maxCases = process.env.EVAL_GAIA_LIMIT
+    ? Number(process.env.EVAL_GAIA_LIMIT)
+    : 25;
+  const sampleCount = process.env.EVAL_GAIA_SAMPLE
+    ? Number(process.env.EVAL_GAIA_SAMPLE)
+    : undefined;
+
+  type GaiaRow = {
+    id: string;
+    Level?: number;
+    web: string;
+    ques: string;
+    [key: string]: unknown;
+  };
+
+  function isGaiaRow(parsed: unknown): parsed is GaiaRow {
+    if (parsed === null || typeof parsed !== "object") return false;
+    const obj = parsed as Record<string, unknown>;
+    return (
+      typeof obj.id === "string" &&
+      typeof obj.web === "string" &&
+      typeof obj.ques === "string"
+    );
+  }
+
+  const candidates = parseJsonlRows(gaiaLines, isGaiaRow);
+
+  // Filter by level if specified
+  const filteredCandidates = levelFilter
+    ? candidates.filter((row) => row.Level === levelFilter)
+    : candidates;
+
+  const gaiaRows = applySampling(filteredCandidates, sampleCount, maxCases);
+
+  const allTestcases: Testcase[] = [];
+  for (const model of models) {
+    for (const row of gaiaRows) {
+      const finalAnswer = (row as Record<string, unknown>)[
+        "Final answer"
+      ] as unknown;
+      const input: EvalInput = {
+        name: "agent/webarena_gaia",
+        modelName: model as AvailableModel,
+        params: {
+          id: row.id,
+          level: row.Level,
+          web: row.web,
+          ques: row.ques,
+          expected: typeof finalAnswer === "string" ? finalAnswer : undefined,
+        },
+      };
+      allTestcases.push({
+        input,
+        name: input.name,
+        tags: [
+          model,
+          input.name,
+          ...(
+            tasksConfig.find((t) => t.name === input.name)?.categories || []
+          ).map((x) => `category/${x}`),
+          `gaia/id/${row.id}`,
+          row.Level ? `gaia/level/${row.Level}` : "gaia/level/unknown",
+        ],
+        metadata: {
+          model: model as AvailableModel,
+          test: `${input.name}:${row.id}`,
+        },
+        expected: true,
+      });
+    }
+  }
+
+  return allTestcases;
+};