work on adding voyager to evals

.changeset/chilly-laws-smile.md

@@ -0,0 +1,5 @@
+---
+"@browserbasehq/stagehand": patch
+---
+
+add webvoyager evals

.github/workflows/ci.yml

@@ -12,9 +12,13 @@ on:
 
 env:
   EVAL_MODELS: "openai/gpt-4.1,google/gemini-2.0-flash,anthropic/claude-3-5-sonnet-latest"
-  EVAL_CATEGORIES: "observe,act,combination,extract,targeted_extract"
+  EVAL_CATEGORIES: "observe,act,combination,extract,targeted_extract,agent"
   EVAL_MAX_CONCURRENCY: 25
   EVAL_TRIAL_COUNT: 5
+  # Agent eval sampling configuration
+  EVAL_WEBVOYAGER_SAMPLE: 3  # Random sample of 3 cases from WebVoyager dataset
+  EVAL_GAIA_SAMPLE: 3        # Random sample of 3 cases from GAIA dataset  
+  EVAL_GAIA_LEVEL: 1          # GAIA difficulty level (1=easiest)
 
 concurrency:
   group: ${{ github.ref }}
@@ -29,6 +33,7 @@ jobs:
       run-act: ${{ steps.check-labels.outputs.run-act }}
       run-observe: ${{ steps.check-labels.outputs.run-observe }}
       run-targeted-extract: ${{ steps.check-labels.outputs.run-targeted-extract }}
+      run-agent: ${{ steps.check-labels.outputs.run-agent }}
     steps:
       - id: check-labels
         run: |
@@ -40,6 +45,7 @@ jobs:
             echo "run-act=true" >> $GITHUB_OUTPUT
             echo "run-observe=true" >> $GITHUB_OUTPUT
             echo "run-targeted-extract=true" >> $GITHUB_OUTPUT
+            echo "run-agent=true" >> $GITHUB_OUTPUT
             exit 0
           fi
 
@@ -49,6 +55,7 @@ jobs:
           echo "run-act=${{ contains(github.event.pull_request.labels.*.name, 'act') }}" >> $GITHUB_OUTPUT
           echo "run-observe=${{ contains(github.event.pull_request.labels.*.name, 'observe') }}" >> $GITHUB_OUTPUT
           echo "run-targeted-extract=${{ contains(github.event.pull_request.labels.*.name, 'targeted-extract') }}" >> $GITHUB_OUTPUT
+          echo "run-agent=${{ contains(github.event.pull_request.labels.*.name, 'agent') }}" >> $GITHUB_OUTPUT
 
   run-lint:
     runs-on: ubuntu-latest
@@ -562,3 +569,73 @@ jobs:
             echo "Eval summary not found for targeted_extract category. Failing CI."
             exit 1
           fi
+
+  run-agent-evals:
+    needs: [run-targeted-extract-evals, determine-evals]
+    runs-on: ubuntu-latest
+    timeout-minutes: 90  # Agent evals can be long-running
+    env:
+      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+      ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+      GOOGLE_GENERATIVE_AI_API_KEY: ${{ secrets.GOOGLE_GENERATIVE_AI_API_KEY }}
+      BRAINTRUST_API_KEY: ${{ secrets.BRAINTRUST_API_KEY }}
+      BROWSERBASE_API_KEY: ${{ secrets.BROWSERBASE_API_KEY }}
+      BROWSERBASE_PROJECT_ID: ${{ secrets.BROWSERBASE_PROJECT_ID }}
+      HEADLESS: true
+      EVAL_ENV: browserbase
+      # Use agent models for agent evals in CI
+      EVAL_AGENT_MODELS: "computer-use-preview-2025-03-11,claude-3-7-sonnet-latest"
+      EVAL_TRIAL_COUNT: 2  # Reduce trials for agent evals
+      EVAL_MAX_CONCURRENCY: 10  # Lower concurrency for agent evals
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v4
+
+      - name: Check for 'agent' label
+        id: label-check
+        run: |
+          if [ "${{ needs.determine-evals.outputs.run-agent }}" != "true" ]; then
+            echo "has_label=false" >> $GITHUB_OUTPUT
+            echo "No label for AGENT. Exiting with success."
+          else
+            echo "has_label=true" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Set up Node.js
+        if: needs.determine-evals.outputs.run-agent == 'true'
+        uses: actions/setup-node@v4
+        with:
+          node-version: "20"
+
+      - name: Install dependencies
+        if: needs.determine-evals.outputs.run-agent == 'true'
+        run: |
+          rm -rf node_modules
+          npm i -g pnpm
+          pnpm install --no-frozen-lockfile
+
+      - name: Build Stagehand
+        if: needs.determine-evals.outputs.run-agent == 'true'
+        run: pnpm run build
+
+      - name: Run Agent Evals
+        if: needs.determine-evals.outputs.run-agent == 'true'
+        run: pnpm run evals category agent api=true
+
+      - name: Log Agent Evals Performance
+        if: needs.determine-evals.outputs.run-agent == 'true'
+        run: |
+          experimentName=$(jq -r '.experimentName' eval-summary.json)
+          echo "View results at https://www.braintrust.dev/app/Browserbase/p/stagehand/experiments/${experimentName}"
+          if [ -f eval-summary.json ]; then
+            agent_score=$(jq '.categories.agent' eval-summary.json)
+            echo "Agent category score: $agent_score%"
+            # Lower threshold for agent evals since they're complex
+            if (( $(echo "$agent_score < 50" | bc -l) )); then
+              echo "Agent category score is below 50%. Failing CI."
+              exit 1
+            fi
+          else
+            echo "Eval summary not found for agent category. Failing CI."
+            exit 1
+          fi

evals/core/summary.ts

@@ -0,0 +1,69 @@
+import fs from "fs";
+import { tasksByName } from "../taskConfig";
+import type { SummaryResult } from "@/types/evals";
+
+export const generateSummary = async (
+  results: SummaryResult[],
+  experimentName: string,
+) => {
+  const passed = results
+    .filter((r) => r.output._success)
+    .map((r) => ({
+      eval: r.input.name,
+      model: r.input.modelName,
+      categories: tasksByName[r.input.name].categories,
+    }));
+
+  const failed = results
+    .filter((r) => !r.output._success)
+    .map((r) => ({
+      eval: r.input.name,
+      model: r.input.modelName,
+      categories: tasksByName[r.input.name].categories,
+    }));
+
+  const categorySuccessCounts: Record<
+    string,
+    { total: number; success: number }
+  > = {};
+  for (const taskName of Object.keys(tasksByName)) {
+    const taskCategories = tasksByName[taskName].categories;
+    const taskResults = results.filter((r) => r.input.name === taskName);
+    const successCount = taskResults.filter((r) => r.output._success).length;
+
+    for (const cat of taskCategories) {
+      if (!categorySuccessCounts[cat]) {
+        categorySuccessCounts[cat] = { total: 0, success: 0 };
+      }
+      categorySuccessCounts[cat].total += taskResults.length;
+      categorySuccessCounts[cat].success += successCount;
+    }
+  }
+
+  const categories: Record<string, number> = {};
+  for (const [cat, counts] of Object.entries(categorySuccessCounts)) {
+    categories[cat] = Math.round((counts.success / counts.total) * 100);
+  }
+
+  const models: Record<string, number> = {};
+  const allModels = [...new Set(results.map((r) => r.input.modelName))];
+  for (const model of allModels) {
+    const modelResults = results.filter((r) => r.input.modelName === model);
+    const successCount = modelResults.filter((r) => r.output._success).length;
+    models[model] = Math.round((successCount / modelResults.length) * 100);
+  }
+
+  const formattedSummary = {
+    experimentName,
+    passed,
+    failed,
+    categories,
+    models,
+  };
+
+  fs.writeFileSync(
+    "eval-summary.json",
+    JSON.stringify(formattedSummary, null, 2),
+  );
+  console.log("Evaluation summary written to eval-summary.json");
+};