add support for shadow doms (open & closed mode) (#954)

# why
- adds support for taking actions on shadow DOMs (both closed & open
mode)
- addresses the following issues:
   - #380  
   - #870  
   - #908  
   - #943  
# what changed
- This PR adds some injected JS which 
1. intercepts `Element.prototype.attachShadow` early and stashes closed
mode shadow roots in a `WeakMap`
2. provides a 'backdoor' for safely accessing these closed roots without
mutating anything in the actual DOM
- to access the 'backdoor', this PR adds a custom locator engine
`selectors.register('stagehand', …)`
- the engine does a DFS over:
   - regular DOM nodes,
   - open shadow roots via `el.shadowRoot`,
   - closed roots via `window.__stagehand__.getClosedRoot(el)`
   - returns a regular playwright locator
# note
- all the logic here is behind the `experimental` flag in the stagehand
constructor, so that we can give people access without breaking existing
behaviour
- this means that this feature is not available on the API (yet), and
you'll need to set `experimental: true` in order to use it
# test plan
- added 8 different evals. They are primarily aimed at testing shadow
dom interactions with the various types of shadow DOMs (open & closed
mode) and iframes (OOPIFs & SPIFs)
- will also run:
   - `regression` evals
   - `act` evals
   - `extract` evals
   - `observe` evals

Author: seanmcguire12

.changeset/calm-snails-carry.md

@@ -0,0 +1,5 @@
+---
+"@browserbasehq/stagehand": patch
+---
+
+add support for shadow DOMs (open & closed mode) when experimental: true

evals/evals.config.json

@@ -420,12 +420,48 @@
       "categories": ["agent"]
     },
     {
-      "name": "iframe_scroll",
+      "name": "osr_in_oopif",
+      "categories": ["act"]
+    },
+    {
+      "name": "csr_in_oopif",
+      "categories": ["act"]
+    },
+    {
+      "name": "csr_in_spif",
+      "categories": ["act"]
+    },
+    {
+      "name": "csr_in_spif",
+      "categories": ["act"]
+    },
+    {
+      "name": "spif_in_osr",
+      "categories": ["act"]
+    },
+    {
+      "name": "oopif_in_osr",
+      "categories": ["act"]
+    },
+    {
+      "name": "spif_in_csr",
+      "categories": ["act"]
+    },
+    {
+      "name": "oopif_in_csr",
+      "categories": ["act"]
+    },
+    {
+      "name": "osr_in_spif",
       "categories": ["act"]
     },
     {
       "name": "namespace_xpath",
       "categories": ["act"]
+    },
+    {
+      "name": "iframe_scroll",
+      "categories": ["act"]
     }
   ]
 }

evals/tasks/csr_in_oopif.ts

@@ -0,0 +1,54 @@
+import { EvalFunction } from "@/types/evals";
+
+export const csr_in_oopif: EvalFunction = async ({
+  debugUrl,
+  sessionUrl,
+  stagehand,
+  logger,
+}) => {
+  // this eval is designed to test whether stagehand can successfully
+  // click inside an CSR (closed mode shadow) root that is inside an
+  // OOPIF (out of process iframe)
+
+  const page = stagehand.page;
+  try {
+    await page.goto(
+      "https://browserbase.github.io/stagehand-eval-sites/sites/closed-shadow-root-in-oopif/",
+    );
+    await page.act({ action: "click the button", iframes: true });
+
+    const extraction = await page.extract({
+      instruction: "extract the entire page text",
+      iframes: true,
+    });
+
+    const pageText = extraction.extraction;
+
+    if (pageText.includes("button successfully clicked")) {
+      return {
+        _success: true,
+        message: `successfully clicked the button`,
+        debugUrl,
+        sessionUrl,
+        logs: logger.getLogs(),
+      };
+    }
+    return {
+      _success: false,
+      message: `unable to click on the button`,
+      debugUrl,
+      sessionUrl,
+      logs: logger.getLogs(),
+    };
+  } catch (error) {
+    return {
+      _success: false,
+      message: `error: ${error.message}`,
+      debugUrl,
+      sessionUrl,
+      logs: logger.getLogs(),
+    };
+  } finally {
+    await stagehand.close();
+  }
+};

evals/tasks/csr_in_spif.ts

@@ -0,0 +1,54 @@
+import { EvalFunction } from "@/types/evals";
+
+export const csr_in_spif: EvalFunction = async ({
+  debugUrl,
+  sessionUrl,
+  stagehand,
+  logger,
+}) => {
+  // this eval is designed to test whether stagehand can successfully
+  // click inside an CSR (closed mode shadow) root that is inside an
+  // SPIF (same process iframe)
+
+  const page = stagehand.page;
+  try {
+    await page.goto(
+      "https://browserbase.github.io/stagehand-eval-sites/sites/closed-shadow-dom-in-spif/",
+    );
+    await page.act({ action: "click the button", iframes: true });
+
+    const extraction = await page.extract({
+      instruction: "extract the entire page text",
+      iframes: true,
+    });
+
+    const pageText = extraction.extraction;
+
+    if (pageText.includes("button successfully clicked")) {
+      return {
+        _success: true,
+        message: `successfully clicked the button`,
+        debugUrl,
+        sessionUrl,
+        logs: logger.getLogs(),
+      };
+    }
+    return {
+      _success: false,
+      message: `unable to click on the button`,
+      debugUrl,
+      sessionUrl,
+      logs: logger.getLogs(),
+    };
+  } catch (error) {
+    return {
+      _success: false,
+      message: `error: ${error.message}`,
+      debugUrl,
+      sessionUrl,
+      logs: logger.getLogs(),
+    };
+  } finally {
+    await stagehand.close();
+  }
+};

evals/tasks/oopif_in_csr.ts

@@ -0,0 +1,57 @@
+import { EvalFunction } from "@/types/evals";
+
+export const oopif_in_csr: EvalFunction = async ({
+  debugUrl,
+  sessionUrl,
+  stagehand,
+  logger,
+}) => {
+  // this eval is designed to test whether stagehand can successfully
+  // fill a form inside a OOPIF (out of process iframe) that is inside an
+  // CSR (closed mode shadow) root
+
+  const page = stagehand.page;
+  try {
+    await page.goto(
+      "https://browserbase.github.io/stagehand-eval-sites/sites/oopif-in-open-shadow-dom/",
+    );
+    await page.act({
+      action: "fill 'nunya' into the first name field",
+      iframes: true,
+    });
+
+    const extraction = await page.extract({
+      instruction: "extract the entire page text",
+      iframes: true,
+    });
+
+    const pageText = extraction.extraction;
+
+    if (pageText.includes("nunya")) {
+      return {
+        _success: true,
+        message: `successfully filled the form`,
+        debugUrl,
+        sessionUrl,
+        logs: logger.getLogs(),
+      };
+    }
+    return {
+      _success: false,
+      message: `unable to fill the form`,
+      debugUrl,
+      sessionUrl,
+      logs: logger.getLogs(),
+    };
+  } catch (error) {
+    return {
+      _success: false,
+      message: `error: ${error.message}`,
+      debugUrl,
+      sessionUrl,
+      logs: logger.getLogs(),
+    };
+  } finally {
+    await stagehand.close();
+  }
+};

evals/tasks/oopif_in_osr.ts

@@ -0,0 +1,57 @@
+import { EvalFunction } from "@/types/evals";
+
+export const oopif_in_osr: EvalFunction = async ({
+  debugUrl,
+  sessionUrl,
+  stagehand,
+  logger,
+}) => {
+  // this eval is designed to test whether stagehand can successfully
+  // fill a form inside a OOPIF (out of process iframe) that is inside an
+  // OSR (open mode shadow) root
+
+  const page = stagehand.page;
+  try {
+    await page.goto(
+      "https://browserbase.github.io/stagehand-eval-sites/sites/oopif-in-open-shadow-dom/",
+    );
+    await page.act({
+      action: "fill 'nunya' into the first name field",
+      iframes: true,
+    });
+
+    const extraction = await page.extract({
+      instruction: "extract the entire page text",
+      iframes: true,
+    });
+
+    const pageText = extraction.extraction;
+
+    if (pageText.includes("nunya")) {
+      return {
+        _success: true,
+        message: `successfully filled the form`,
+        debugUrl,
+        sessionUrl,
+        logs: logger.getLogs(),
+      };
+    }
+    return {
+      _success: false,
+      message: `unable to fill the form`,
+      debugUrl,
+      sessionUrl,
+      logs: logger.getLogs(),
+    };
+  } catch (error) {
+    return {
+      _success: false,
+      message: `error: ${error.message}`,
+      debugUrl,
+      sessionUrl,
+      logs: logger.getLogs(),
+    };
+  } finally {
+    await stagehand.close();
+  }
+};

evals/tasks/osr_in_oopif.ts

@@ -0,0 +1,54 @@
+import { EvalFunction } from "@/types/evals";
+
+export const osr_in_oopif: EvalFunction = async ({
+  debugUrl,
+  sessionUrl,
+  stagehand,
+  logger,
+}) => {
+  // this eval is designed to test whether stagehand can successfully
+  // click inside an OSR (open mode shadow) root that is inside an
+  // OOPIF (out of process iframe)
+
+  const page = stagehand.page;
+  try {
+    await page.goto(
+      "https://browserbase.github.io/stagehand-eval-sites/sites/open-shadow-root-in-oopif/",
+    );
+    await page.act({ action: "click the button", iframes: true });
+
+    const extraction = await page.extract({
+      instruction: "extract the entire page text",
+      iframes: true,
+    });
+
+    const pageText = extraction.extraction;
+
+    if (pageText.includes("button successfully clicked")) {
+      return {
+        _success: true,
+        message: `successfully clicked the button`,
+        debugUrl,
+        sessionUrl,
+        logs: logger.getLogs(),
+      };
+    }
+    return {
+      _success: false,
+      message: `unable to click on the button`,
+      debugUrl,
+      sessionUrl,
+      logs: logger.getLogs(),
+    };
+  } catch (error) {
+    return {
+      _success: false,
+      message: `error: ${error.message}`,
+      debugUrl,
+      sessionUrl,
+      logs: logger.getLogs(),
+    };
+  } finally {
+    await stagehand.close();
+  }
+};

evals/tasks/osr_in_spif.ts

@@ -0,0 +1,54 @@
+import { EvalFunction } from "@/types/evals";
+
+export const osr_in_spif: EvalFunction = async ({
+  debugUrl,
+  sessionUrl,
+  stagehand,
+  logger,
+}) => {
+  // this eval is designed to test whether stagehand can successfully
+  // click inside an OSR (open mode shadow) root that is inside an
+  // SPIF (same process iframe)
+
+  const page = stagehand.page;
+  try {
+    await page.goto(
+      "https://browserbase.github.io/stagehand-eval-sites/sites/open-shadow-root-in-spif/",
+    );
+    await page.act({ action: "click the button", iframes: true });
+
+    const extraction = await page.extract({
+      instruction: "extract the entire page text",
+      iframes: true,
+    });
+
+    const pageText = extraction.extraction;
+
+    if (pageText.includes("button successfully clicked")) {
+      return {
+        _success: true,
+        message: `successfully clicked the button`,
+        debugUrl,
+        sessionUrl,
+        logs: logger.getLogs(),
+      };
+    }
+    return {
+      _success: false,
+      message: `unable to click on the button`,
+      debugUrl,
+      sessionUrl,
+      logs: logger.getLogs(),
+    };
+  } catch (error) {
+    return {
+      _success: false,
+      message: `error: ${error.message}`,
+      debugUrl,
+      sessionUrl,
+      logs: logger.getLogs(),
+    };
+  } finally {
+    await stagehand.close();
+  }
+};