work on adding voyager to evals (#959)

⏺ ## Why
Add WebVoyager and GAIA evaluation suites to benchmark Stagehand's web
navigation and
   reasoning capabilities against industry-standard datasets.

  ## What Changed
- Added WebVoyager eval suite with 643 test cases for web navigation
tasks
- Added GAIA eval suite with 90 test cases for general AI assistant
tasks
  - Refactored eval infrastructure to support sampling and filtering
- Created reusable utilities for JSONL parsing and test case generation
  - Added configuration for new eval suites in `evals.config.json`

  ### Environment Variables
  - `EVAL_WEBVOYAGER_SAMPLE`: Random sample size from WebVoyager dataset
  - `EVAL_WEBVOYAGER_LIMIT`: Max cases to run (default: 25)
  - `EVAL_GAIA_SAMPLE`: Random sample size from GAIA dataset
  - `EVAL_GAIA_LIMIT`: Max cases to run (default: 25)
  - `EVAL_GAIA_LEVEL`: Filter GAIA by difficulty level (1, 2, or 3)

  ### Sampling Strategy
The sampling implementation uses Fisher-Yates shuffle for unbiased
random selection
when `SAMPLE` is specified, otherwise takes the first `LIMIT` cases.
This allows for
  both deterministic (first N) and randomized (sample N) test runs.

  ## Test Plan
  ```bash
  # Test WebVoyager with OpenAI
  EVAL_SUITE=webvoyager EVAL_WEBVOYAGER_SAMPLE=1
  EVAL_MODEL=openai/gpt-4o-computer-use-preview pnpm run evals

  # Test WebVoyager with Claude
  EVAL_SUITE=webvoyager EVAL_WEBVOYAGER_SAMPLE=1
  EVAL_MODEL=anthropic/claude-3-5-sonnet-20241022 pnpm run evals

  # Test GAIA with OpenAI
EVAL_SUITE=gaia EVAL_GAIA_SAMPLE=1
EVAL_MODEL=openai/gpt-4o-computer-use-preview pnpm
   run evals

  # Test GAIA with Claude
EVAL_SUITE=gaia EVAL_GAIA_SAMPLE=1
EVAL_MODEL=anthropic/claude-3-5-sonnet-20241022
  pnpm run evals

  # Verify existing evals still work
  pnpm run evals
  ```

---------

Co-authored-by: Claude <[email protected]>
Co-authored-by: Miguel <[email protected]>

Author: 3 people

.changeset/chilly-laws-smile.md

@@ -0,0 +1,5 @@
+---
+"@browserbasehq/stagehand": patch
+---
+
+add webvoyager evals

.github/workflows/ci.yml

@@ -12,7 +12,7 @@ on:
 
 env:
   EVAL_MODELS: "openai/gpt-4.1,google/gemini-2.0-flash,anthropic/claude-3-5-sonnet-latest"
-  EVAL_CATEGORIES: "observe,act,combination,extract,targeted_extract"
+  EVAL_CATEGORIES: "observe,act,combination,extract,targeted_extract,agent"
   EVAL_MAX_CONCURRENCY: 25
   EVAL_TRIAL_COUNT: 5
 
@@ -29,6 +29,7 @@ jobs:
       run-act: ${{ steps.check-labels.outputs.run-act }}
       run-observe: ${{ steps.check-labels.outputs.run-observe }}
       run-targeted-extract: ${{ steps.check-labels.outputs.run-targeted-extract }}
+      run-agent: ${{ steps.check-labels.outputs.run-agent }}
     steps:
       - id: check-labels
         run: |
@@ -40,6 +41,7 @@ jobs:
             echo "run-act=true" >> $GITHUB_OUTPUT
             echo "run-observe=true" >> $GITHUB_OUTPUT
             echo "run-targeted-extract=true" >> $GITHUB_OUTPUT
+            echo "run-agent=true" >> $GITHUB_OUTPUT
             exit 0
           fi
 
@@ -49,6 +51,7 @@ jobs:
           echo "run-act=${{ contains(github.event.pull_request.labels.*.name, 'act') }}" >> $GITHUB_OUTPUT
           echo "run-observe=${{ contains(github.event.pull_request.labels.*.name, 'observe') }}" >> $GITHUB_OUTPUT
           echo "run-targeted-extract=${{ contains(github.event.pull_request.labels.*.name, 'targeted-extract') }}" >> $GITHUB_OUTPUT
+          echo "run-agent=${{ contains(github.event.pull_request.labels.*.name, 'agent') }}" >> $GITHUB_OUTPUT
 
   run-lint:
     runs-on: ubuntu-latest
@@ -562,3 +565,73 @@ jobs:
             echo "Eval summary not found for targeted_extract category. Failing CI."
             exit 1
           fi
+
+  run-agent-evals:
+    needs: [run-targeted-extract-evals, determine-evals]
+    runs-on: ubuntu-latest
+    timeout-minutes: 90 # Agent evals can be long-running
+    env:
+      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+      ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+      GOOGLE_GENERATIVE_AI_API_KEY: ${{ secrets.GOOGLE_GENERATIVE_AI_API_KEY }}
+      BRAINTRUST_API_KEY: ${{ secrets.BRAINTRUST_API_KEY }}
+      BROWSERBASE_API_KEY: ${{ secrets.BROWSERBASE_API_KEY }}
+      BROWSERBASE_PROJECT_ID: ${{ secrets.BROWSERBASE_PROJECT_ID }}
+      HEADLESS: true
+      EVAL_ENV: browserbase
+      # Use agent models for agent evals in CI
+      EVAL_AGENT_MODELS: "computer-use-preview-2025-03-11,claude-3-7-sonnet-latest"
+      EVAL_TRIAL_COUNT: 2 # Reduce trials for agent evals
+      EVAL_MAX_CONCURRENCY: 10 # Lower concurrency for agent evals
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v4
+
+      - name: Check for 'agent' label
+        id: label-check
+        run: |
+          if [ "${{ needs.determine-evals.outputs.run-agent }}" != "true" ]; then
+            echo "has_label=false" >> $GITHUB_OUTPUT
+            echo "No label for AGENT. Exiting with success."
+          else
+            echo "has_label=true" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Set up Node.js
+        if: needs.determine-evals.outputs.run-agent == 'true'
+        uses: actions/setup-node@v4
+        with:
+          node-version: "20"
+
+      - name: Install dependencies
+        if: needs.determine-evals.outputs.run-agent == 'true'
+        run: |
+          rm -rf node_modules
+          npm i -g pnpm
+          pnpm install --no-frozen-lockfile
+
+      - name: Build Stagehand
+        if: needs.determine-evals.outputs.run-agent == 'true'
+        run: pnpm run build
+
+      - name: Run Agent Evals
+        if: needs.determine-evals.outputs.run-agent == 'true'
+        run: pnpm run evals category agent
+
+      - name: Log Agent Evals Performance
+        if: needs.determine-evals.outputs.run-agent == 'true'
+        run: |
+          experimentName=$(jq -r '.experimentName' eval-summary.json)
+          echo "View results at https://www.braintrust.dev/app/Browserbase/p/stagehand/experiments/${experimentName}"
+          if [ -f eval-summary.json ]; then
+            agent_score=$(jq '.categories.agent' eval-summary.json)
+            echo "Agent category score: $agent_score%"
+            # Lower threshold for agent evals since they're complex
+            if (( $(echo "$agent_score < 50" | bc -l) )); then
+              echo "Agent category score is below 50%. Failing CI."
+              exit 1
+            fi
+          else
+            echo "Eval summary not found for agent category. Failing CI."
+            exit 1
+          fi

README.md

@@ -125,6 +125,7 @@ pnpm playwright install
 pnpm run build
 pnpm run example # run the blank script at ./examples/example.ts
 pnpm run example 2048 # run the 2048 example at ./examples/2048.ts
+pnpm run evals -man # see evaluation suite options
 ```
 
 Stagehand is best when you have an API key for an LLM provider and Browserbase credentials. To add these to your project, run:

evals/args.ts

@@ -15,6 +15,8 @@ const parsedArgs: {
   trials?: number;
   concurrency?: number;
   provider?: string;
+  dataset?: string;
+  max_k?: number;
   leftover: string[];
 } = {
   leftover: [],
@@ -39,6 +41,13 @@ for (const arg of rawArgs) {
     }
   } else if (arg.startsWith("provider=")) {
     parsedArgs.provider = arg.split("=")[1]?.toLowerCase();
+  } else if (arg.startsWith("--dataset=")) {
+    parsedArgs.dataset = arg.split("=")[1]?.toLowerCase();
+  } else if (arg.startsWith("max_k=")) {
+    const val = parseInt(arg.split("=")[1], 10);
+    if (!isNaN(val)) {
+      parsedArgs.max_k = val;
+    }
   } else {
     parsedArgs.leftover.push(arg);
   }
@@ -63,6 +72,12 @@ if (parsedArgs.trials !== undefined) {
 if (parsedArgs.concurrency !== undefined) {
   process.env.EVAL_MAX_CONCURRENCY = String(parsedArgs.concurrency);
 }
+if (parsedArgs.max_k !== undefined) {
+  process.env.EVAL_MAX_K = String(parsedArgs.max_k);
+}
+if (parsedArgs.dataset !== undefined) {
+  process.env.EVAL_DATASET = parsedArgs.dataset;
+}
 
 const DEFAULT_EVAL_CATEGORIES = process.env.EVAL_CATEGORIES
   ? process.env.EVAL_CATEGORIES.split(",")
@@ -77,6 +92,7 @@ const DEFAULT_EVAL_CATEGORIES = process.env.EVAL_CATEGORIES
       "regression",
       "llm_clients",
       "agent",
+      "external_agent_benchmarks",
     ];
 
 const providerDefault = process.env.EVAL_PROVIDER ?? undefined;
@@ -104,6 +120,12 @@ function buildUsage(detailed = false): string {
   ${chalk.cyan("provider".padEnd(12))} ${"override LLM provider".padEnd(24)}
     (default ${chalk.dim(providerDefault)})        [${chalk.yellow("OPENAI")}, ${chalk.yellow("ANTHROPIC")}, ${chalk.yellow("GOOGLE")}, ${chalk.yellow("TOGETHER")}, ${chalk.yellow("GROQ")}, ${chalk.yellow("CEREBRAS")}]
 
+  ${chalk.cyan("max_k".padEnd(12))} ${"max test cases per dataset".padEnd(24)}
+    (default ${chalk.dim("25")})
+
+  ${chalk.cyan("--dataset".padEnd(12))} ${"filter dataset for benchmarks".padEnd(24)}
+    (optional)              [${chalk.yellow("gaia")}, ${chalk.yellow("webvoyager")}]
+
 
     ${chalk.magenta.underline("Positional filters\n")}
       category <category_name>   one of: ${DEFAULT_EVAL_CATEGORIES.map((c) =>

evals/core/summary.ts

@@ -0,0 +1,69 @@
+import fs from "fs";
+import { tasksByName } from "../taskConfig";
+import type { SummaryResult } from "@/types/evals";
+
+export const generateSummary = async (
+  results: SummaryResult[],
+  experimentName: string,
+) => {
+  const passed = results
+    .filter((r) => r.output._success)
+    .map((r) => ({
+      eval: r.input.name,
+      model: r.input.modelName,
+      categories: tasksByName[r.input.name].categories,
+    }));
+
+  const failed = results
+    .filter((r) => !r.output._success)
+    .map((r) => ({
+      eval: r.input.name,
+      model: r.input.modelName,
+      categories: tasksByName[r.input.name].categories,
+    }));
+
+  const categorySuccessCounts: Record<
+    string,
+    { total: number; success: number }
+  > = {};
+  for (const taskName of Object.keys(tasksByName)) {
+    const taskCategories = tasksByName[taskName].categories;
+    const taskResults = results.filter((r) => r.input.name === taskName);
+    const successCount = taskResults.filter((r) => r.output._success).length;
+
+    for (const cat of taskCategories) {
+      if (!categorySuccessCounts[cat]) {
+        categorySuccessCounts[cat] = { total: 0, success: 0 };
+      }
+      categorySuccessCounts[cat].total += taskResults.length;
+      categorySuccessCounts[cat].success += successCount;
+    }
+  }
+
+  const categories: Record<string, number> = {};
+  for (const [cat, counts] of Object.entries(categorySuccessCounts)) {
+    categories[cat] = Math.round((counts.success / counts.total) * 100);
+  }
+
+  const models: Record<string, number> = {};
+  const allModels = [...new Set(results.map((r) => r.input.modelName))];
+  for (const model of allModels) {
+    const modelResults = results.filter((r) => r.input.modelName === model);
+    const successCount = modelResults.filter((r) => r.output._success).length;
+    models[model] = Math.round((successCount / modelResults.length) * 100);
+  }
+
+  const formattedSummary = {
+    experimentName,
+    passed,
+    failed,
+    categories,
+    models,
+  };
+
+  fs.writeFileSync(
+    "eval-summary.json",
+    JSON.stringify(formattedSummary, null, 2),
+  );
+  console.log("Evaluation summary written to eval-summary.json");
+};