tests: adding evals for agent tool

Kylejeong2 · Kylejeong2 · commit 81f508080134 · 2025-11-16T13:33:32.000-08:00
diff --git a/evals/mcp-eval-basic.config.json b/evals/mcp-eval-basic.config.json
@@ -121,6 +121,21 @@
         "browserbase_stagehand_get_url",
         "browserbase_session_close"
       ]
+    },
+    {
+      "name": "agent-basic-test",
+      "description": "Test autonomous agent functionality with simple task",
+      "steps": [
+        {
+          "user": "Create a browser session and use the agent to go to https://example.com and extract the page title, then close the session",
+          "expectedState": "Example Domain"
+        }
+      ],
+      "expectTools": [
+        "browserbase_session_create",
+        "browserbase_stagehand_agent",
+        "browserbase_session_close"
+      ]
     }
   ]
 }
diff --git a/evals/mcp-eval-minimal.config.json b/evals/mcp-eval-minimal.config.json
@@ -67,6 +67,21 @@
         "browserbase_stagehand_get_url",
         "browserbase_session_close"
       ]
+    },
+    {
+      "name": "smoke-test-agent",
+      "description": "Quick test to verify autonomous agent works",
+      "steps": [
+        {
+          "user": "Create a browser session and use the agent to visit example.org and tell me the page title, then close the session",
+          "expectedState": "Example Domain"
+        }
+      ],
+      "expectTools": [
+        "browserbase_session_create",
+        "browserbase_stagehand_agent",
+        "browserbase_session_close"
+      ]
     }
   ]
 }
diff --git a/evals/mcp-eval.config.json b/evals/mcp-eval.config.json
@@ -105,6 +105,21 @@
         "browserbase_session_create",
         "browserbase_stagehand_navigate"
       ]
+    },
+    {
+      "name": "agent-complex-task-test",
+      "description": "Test agent with multi-step task",
+      "steps": [
+        {
+          "user": "Create a browser session and use the agent to go to https://news.ycombinator.com, find the top post, and extract its title and score, then close the session",
+          "expectedState": "points"
+        }
+      ],
+      "expectTools": [
+        "browserbase_session_create",
+        "browserbase_stagehand_agent",
+        "browserbase_session_close"
+      ]
     }
   ]
 }
diff --git a/package.json b/package.json
@@ -57,6 +57,7 @@
     "@changesets/cli": "^2.29.6",
     "@eslint/js": "^9.29.0",
     "@smithery/cli": "^1.2.15",
+    "@types/node": "^24.10.1",
     "chalk": "^5.3.0",
     "eslint": "^9.29.0",
     "eslint-plugin-react": "^7.37.5",
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml

Original file line number	Diff line number	Diff line change
`@@ -121,6 +121,21 @@`
`121`	`121`	`"browserbase_stagehand_get_url",`
`122`	`122`	`"browserbase_session_close"`
`123`	`123`	`]`
	`124`	`+ },`
	`125`	`+ {`
	`126`	`+ "name": "agent-basic-test",`
	`127`	`+ "description": "Test autonomous agent functionality with simple task",`
	`128`	`+ "steps": [`
	`129`	`+ {`
	`130`	`+ "user": "Create a browser session and use the agent to go to https://example.com and extract the page title, then close the session",`
	`131`	`+ "expectedState": "Example Domain"`
	`132`	`+ }`
	`133`	`+ ],`
	`134`	`+ "expectTools": [`
	`135`	`+ "browserbase_session_create",`
	`136`	`+ "browserbase_stagehand_agent",`
	`137`	`+ "browserbase_session_close"`
	`138`	`+ ]`
`124`	`139`	`}`
`125`	`140`	`]`
`126`	`141`	`}`
Original file line number	Diff line number	Diff line change
`@@ -67,6 +67,21 @@`
`67`	`67`	`"browserbase_stagehand_get_url",`
`68`	`68`	`"browserbase_session_close"`
`69`	`69`	`]`
	`70`	`+ },`
	`71`	`+ {`
	`72`	`+ "name": "smoke-test-agent",`
	`73`	`+ "description": "Quick test to verify autonomous agent works",`
	`74`	`+ "steps": [`
	`75`	`+ {`
	`76`	`+ "user": "Create a browser session and use the agent to visit example.org and tell me the page title, then close the session",`
	`77`	`+ "expectedState": "Example Domain"`
	`78`	`+ }`
	`79`	`+ ],`
	`80`	`+ "expectTools": [`
	`81`	`+ "browserbase_session_create",`
	`82`	`+ "browserbase_stagehand_agent",`
	`83`	`+ "browserbase_session_close"`
	`84`	`+ ]`
`70`	`85`	`}`
`71`	`86`	`]`
`72`	`87`	`}`
Original file line number	Diff line number	Diff line change
`@@ -105,6 +105,21 @@`
`105`	`105`	`"browserbase_session_create",`
`106`	`106`	`"browserbase_stagehand_navigate"`
`107`	`107`	`]`
	`108`	`+ },`
	`109`	`+ {`
	`110`	`+ "name": "agent-complex-task-test",`
	`111`	`+ "description": "Test agent with multi-step task",`
	`112`	`+ "steps": [`
	`113`	`+ {`
	`114`	`+ "user": "Create a browser session and use the agent to go to https://news.ycombinator.com, find the top post, and extract its title and score, then close the session",`
	`115`	`+ "expectedState": "points"`
	`116`	`+ }`
	`117`	`+ ],`
	`118`	`+ "expectTools": [`
	`119`	`+ "browserbase_session_create",`
	`120`	`+ "browserbase_stagehand_agent",`
	`121`	`+ "browserbase_session_close"`
	`122`	`+ ]`
`108`	`123`	`}`
`109`	`124`	`]`
`110`	`125`	`}`