add evals

Author: seanmcguire12

evals/evals.config.json

@@ -418,6 +418,42 @@
     {
       "name": "agent/sign_in",
       "categories": ["agent"]
+    },
+    {
+      "name": "osr_in_oopif",
+      "categories": ["act"]
+    },
+    {
+      "name": "csr_in_oopif",
+      "categories": ["act"]
+    },
+    {
+      "name": "csr_in_spif",
+      "categories": ["act"]
+    },
+    {
+      "name": "csr_in_spif",
+      "categories": ["act"]
+    },
+    {
+      "name": "spif_in_osr",
+      "categories": ["act"]
+    },
+    {
+      "name": "oopif_in_osr",
+      "categories": ["act"]
+    },
+    {
+      "name": "spif_in_csr",
+      "categories": ["act"]
+    },
+    {
+      "name": "oopif_in_csr",
+      "categories": ["act"]
+    },
+    {
+      "name": "osr_in_spif",
+      "categories": ["act"]
     }
   ]
 }

evals/tasks/csr_in_oopif.ts

@@ -0,0 +1,47 @@
+import { EvalFunction } from "@/types/evals";
+
+export const csr_in_oopif: EvalFunction = async ({
+  debugUrl,
+  sessionUrl,
+  stagehand,
+  logger,
+}) => {
+  // this eval is designed to test whether stagehand can successfully
+  // click inside an CSR (closed mode shadow) root that is inside an
+  // OOPIF (out of process iframe)
+
+  const page = stagehand.page;
+  try {
+    await page.goto(
+      "https://browserbase.github.io/stagehand-eval-sites/sites/closed-shadow-root-in-oopif/",
+    );
+    await page.act({ action: "click the button", iframes: true });
+
+    const extraction = await page.extract({
+      instruction: "extract the entire page text",
+      iframes: true,
+    });
+
+    const pageText = extraction.extraction;
+
+    if (pageText.includes("button successfully clicked")) {
+      return {
+        _success: true,
+        message: `successfully clicked the button`,
+        debugUrl,
+        sessionUrl,
+        logs: logger.getLogs(),
+      };
+    }
+  } catch (error) {
+    return {
+      _success: false,
+      message: `error: ${error.message}`,
+      debugUrl,
+      sessionUrl,
+      logs: logger.getLogs(),
+    };
+  } finally {
+    await stagehand.close();
+  }
+};

evals/tasks/csr_in_spif.ts

@@ -0,0 +1,47 @@
+import { EvalFunction } from "@/types/evals";
+
+export const csr_in_spif: EvalFunction = async ({
+  debugUrl,
+  sessionUrl,
+  stagehand,
+  logger,
+}) => {
+  // this eval is designed to test whether stagehand can successfully
+  // click inside an CSR (closed mode shadow) root that is inside an
+  // SPIF (same process iframe)
+
+  const page = stagehand.page;
+  try {
+    await page.goto(
+      "https://browserbase.github.io/stagehand-eval-sites/sites/closed-shadow-dom-in-spif/",
+    );
+    await page.act({ action: "click the button", iframes: true });
+
+    const extraction = await page.extract({
+      instruction: "extract the entire page text",
+      iframes: true,
+    });
+
+    const pageText = extraction.extraction;
+
+    if (pageText.includes("button successfully clicked")) {
+      return {
+        _success: true,
+        message: `successfully clicked the button`,
+        debugUrl,
+        sessionUrl,
+        logs: logger.getLogs(),
+      };
+    }
+  } catch (error) {
+    return {
+      _success: false,
+      message: `error: ${error.message}`,
+      debugUrl,
+      sessionUrl,
+      logs: logger.getLogs(),
+    };
+  } finally {
+    await stagehand.close();
+  }
+};

evals/tasks/oopif_in_csr.ts

@@ -0,0 +1,50 @@
+import { EvalFunction } from "@/types/evals";
+
+export const oopif_in_csr: EvalFunction = async ({
+  debugUrl,
+  sessionUrl,
+  stagehand,
+  logger,
+}) => {
+  // this eval is designed to test whether stagehand can successfully
+  // fill a form inside a OOPIF (out of process iframe) that is inside an
+  // CSR (closed mode shadow) root
+
+  const page = stagehand.page;
+  try {
+    await page.goto(
+      "https://browserbase.github.io/stagehand-eval-sites/sites/oopif-in-open-shadow-dom/",
+    );
+    await page.act({
+      action: "fill 'nunya' into the first name field",
+      iframes: true,
+    });
+
+    const extraction = await page.extract({
+      instruction: "extract the entire page text",
+      iframes: true,
+    });
+
+    const pageText = extraction.extraction;
+
+    if (pageText.includes("nunya")) {
+      return {
+        _success: true,
+        message: `successfully clicked the button`,
+        debugUrl,
+        sessionUrl,
+        logs: logger.getLogs(),
+      };
+    }
+  } catch (error) {
+    return {
+      _success: false,
+      message: `error: ${error.message}`,
+      debugUrl,
+      sessionUrl,
+      logs: logger.getLogs(),
+    };
+  } finally {
+    await stagehand.close();
+  }
+};

evals/tasks/oopif_in_osr.ts

@@ -0,0 +1,50 @@
+import { EvalFunction } from "@/types/evals";
+
+export const oopif_in_osr: EvalFunction = async ({
+  debugUrl,
+  sessionUrl,
+  stagehand,
+  logger,
+}) => {
+  // this eval is designed to test whether stagehand can successfully
+  // fill a form inside a OOPIF (out of process iframe) that is inside an
+  // OSR (open mode shadow) root
+
+  const page = stagehand.page;
+  try {
+    await page.goto(
+      "https://browserbase.github.io/stagehand-eval-sites/sites/oopif-in-open-shadow-dom/",
+    );
+    await page.act({
+      action: "fill 'nunya' into the first name field",
+      iframes: true,
+    });
+
+    const extraction = await page.extract({
+      instruction: "extract the entire page text",
+      iframes: true,
+    });
+
+    const pageText = extraction.extraction;
+
+    if (pageText.includes("nunya")) {
+      return {
+        _success: true,
+        message: `successfully clicked the button`,
+        debugUrl,
+        sessionUrl,
+        logs: logger.getLogs(),
+      };
+    }
+  } catch (error) {
+    return {
+      _success: false,
+      message: `error: ${error.message}`,
+      debugUrl,
+      sessionUrl,
+      logs: logger.getLogs(),
+    };
+  } finally {
+    await stagehand.close();
+  }
+};

evals/tasks/osr_in_oopif.ts

@@ -0,0 +1,47 @@
+import { EvalFunction } from "@/types/evals";
+
+export const osr_in_oopif: EvalFunction = async ({
+  debugUrl,
+  sessionUrl,
+  stagehand,
+  logger,
+}) => {
+  // this eval is designed to test whether stagehand can successfully
+  // click inside an OSR (open mode shadow) root that is inside an
+  // OOPIF (out of process iframe)
+
+  const page = stagehand.page;
+  try {
+    await page.goto(
+      "https://browserbase.github.io/stagehand-eval-sites/sites/open-shadow-root-in-oopif/",
+    );
+    await page.act({ action: "click the button", iframes: true });
+
+    const extraction = await page.extract({
+      instruction: "extract the entire page text",
+      iframes: true,
+    });
+
+    const pageText = extraction.extraction;
+
+    if (pageText.includes("button successfully clicked")) {
+      return {
+        _success: true,
+        message: `successfully clicked the button`,
+        debugUrl,
+        sessionUrl,
+        logs: logger.getLogs(),
+      };
+    }
+  } catch (error) {
+    return {
+      _success: false,
+      message: `error: ${error.message}`,
+      debugUrl,
+      sessionUrl,
+      logs: logger.getLogs(),
+    };
+  } finally {
+    await stagehand.close();
+  }
+};

evals/tasks/osr_in_spif.ts

@@ -0,0 +1,47 @@
+import { EvalFunction } from "@/types/evals";
+
+export const osr_in_spif: EvalFunction = async ({
+  debugUrl,
+  sessionUrl,
+  stagehand,
+  logger,
+}) => {
+  // this eval is designed to test whether stagehand can successfully
+  // click inside an OSR (open mode shadow) root that is inside an
+  // SPIF (same process iframe)
+
+  const page = stagehand.page;
+  try {
+    await page.goto(
+      "https://browserbase.github.io/stagehand-eval-sites/sites/open-shadow-root-in-spif/",
+    );
+    await page.act({ action: "click the button", iframes: true });
+
+    const extraction = await page.extract({
+      instruction: "extract the entire page text",
+      iframes: true,
+    });
+
+    const pageText = extraction.extraction;
+
+    if (pageText.includes("button successfully clicked")) {
+      return {
+        _success: true,
+        message: `successfully clicked the button`,
+        debugUrl,
+        sessionUrl,
+        logs: logger.getLogs(),
+      };
+    }
+  } catch (error) {
+    return {
+      _success: false,
+      message: `error: ${error.message}`,
+      debugUrl,
+      sessionUrl,
+      logs: logger.getLogs(),
+    };
+  } finally {
+    await stagehand.close();
+  }
+};

evals/tasks/spif_in_csr.ts

@@ -0,0 +1,47 @@
+import { EvalFunction } from "@/types/evals";
+
+export const spif_in_csr: EvalFunction = async ({
+  debugUrl,
+  sessionUrl,
+  stagehand,
+  logger,
+}) => {
+  // this eval is designed to test whether stagehand can successfully
+  // click inside a SPIF (same process iframe) that is inside an
+  // CSR (closed mode shadow) root
+
+  const page = stagehand.page;
+  try {
+    await page.goto(
+      "https://browserbase.github.io/stagehand-eval-sites/sites/spif-in-closed-shadow-dom/",
+    );
+    await page.act({ action: "click the button", iframes: true });
+
+    const extraction = await page.extract({
+      instruction: "extract the entire page text",
+      iframes: true,
+    });
+
+    const pageText = extraction.extraction;
+
+    if (pageText.includes("button successfully clicked")) {
+      return {
+        _success: true,
+        message: `successfully clicked the button`,
+        debugUrl,
+        sessionUrl,
+        logs: logger.getLogs(),
+      };
+    }
+  } catch (error) {
+    return {
+      _success: false,
+      message: `error: ${error.message}`,
+      debugUrl,
+      sessionUrl,
+      logs: logger.getLogs(),
+    };
+  } finally {
+    await stagehand.close();
+  }
+};