put shadow dom support behind experimental flag

Author: seanmcguire12

lib/StagehandPage.ts

@@ -123,18 +123,21 @@ export class StagehandPage {
         logger: this.stagehand.logger,
         stagehandPage: this,
         selfHeal: this.stagehand.selfHeal,
+        experimental: this.stagehand.experimental,
       });
       this.extractHandler = new StagehandExtractHandler({
         stagehand: this.stagehand,
         logger: this.stagehand.logger,
         stagehandPage: this,
         userProvidedInstructions,
+        experimental: this.stagehand.experimental,
       });
       this.observeHandler = new StagehandObserveHandler({
         stagehand: this.stagehand,
         logger: this.stagehand.logger,
         stagehandPage: this,
         userProvidedInstructions,
+        experimental: this.stagehand.experimental,
       });
     }
   }

lib/a11y/utils.ts

@@ -118,11 +118,13 @@ const lc = (raw: string): string => {
 /**
  * Build mappings from CDP backendNodeIds to HTML tag names and relative XPaths.
  *
+ * @param experimental - Whether to use experimental behaviour.
  * @param sp - The StagehandPage wrapper for Playwright and CDP calls.
  * @param targetFrame - Optional Playwright.Frame whose DOM subtree to map; defaults to main frame.
  * @returns A Promise resolving to BackendIdMaps containing tagNameMap and xpathMap.
  */
 export async function buildBackendIdMaps(
+  experimental: boolean,
   sp: StagehandPage,
   targetFrame?: Frame,
 ): Promise<BackendIdMaps> {
@@ -169,14 +171,22 @@ export async function buildBackendIdMaps(
 
       let iframeNode: DOMNode | undefined;
       const locate = (n: DOMNode): boolean => {
-        if (n.backendNodeId === backendNodeId) {
-          iframeNode = n;
-          return true;
+        if (experimental) {
+          if (n.backendNodeId === backendNodeId) {
+            iframeNode = n;
+            return true;
+          }
+          if (n.shadowRoots?.some(locate)) return true;
+          if (n.children?.some(locate)) return true;
+          if (n.contentDocument && locate(n.contentDocument)) return true;
+          return false;
+        } else {
+          if (n.backendNodeId === backendNodeId) return (iframeNode = n), true;
+          return (
+            (n.children?.some(locate) ?? false) ||
+            (n.contentDocument ? locate(n.contentDocument) : false)
+          );
         }
-        if (n.shadowRoots?.some(locate)) return true;
-        if (n.children?.some(locate)) return true;
-        if (n.contentDocument && locate(n.contentDocument)) return true;
-        return false;
       };
 
       if (!locate(root) || !iframeNode?.contentDocument) {
@@ -218,7 +228,7 @@ export async function buildBackendIdMaps(
         stack.push({ node: node.contentDocument, path: "", fid: childFid });
       }
 
-      if (node.shadowRoots?.length) {
+      if (node.shadowRoots?.length && experimental) {
         for (const shadowRoot of node.shadowRoots) {
           stack.push({
             node: shadowRoot,
@@ -502,13 +512,15 @@ export async function getCDPFrameId(
  * @returns A Promise resolving to a TreeResult with the hierarchical AX tree and related metadata.
  */
 export async function getAccessibilityTree(
+  experimental: boolean,
   stagehandPage: StagehandPage,
   logger: (log: LogLine) => void,
   selector?: string,
   targetFrame?: Frame,
 ): Promise<TreeResult> {
   // 0. DOM helpers (maps, xpath)
   const { tagNameMap, xpathMap } = await buildBackendIdMaps(
+    experimental,
     stagehandPage,
     targetFrame,
   );
@@ -717,7 +729,7 @@ export async function getFrameRootBackendNodeId(
  * @param frame - The Playwright.Frame whose iframe element to locate.
  * @returns A Promise resolving to the XPath of the iframe element, or "/" if no frame provided.
  */
-export async function getFrameRootXpath(
+export async function getFrameRootXpathWithShadow(
   frame: Frame | undefined,
 ): Promise<string> {
   // Return root path when no frame context is provided
@@ -768,6 +780,34 @@ export async function getFrameRootXpath(
   });
 }
 
+export async function getFrameRootXpath(
+  frame: Frame | undefined,
+): Promise<string> {
+  // Return root path when no frame context is provided
+  if (!frame) {
+    return "/";
+  }
+  // Obtain the element handle of the iframe in the embedding document
+  const handle = await frame.frameElement();
+  // Evaluate the element's absolute XPath within the page context
+  return handle.evaluate((node: Element) => {
+    const pos = (el: Element) => {
+      let i = 1;
+      for (
+        let sib = el.previousElementSibling;
+        sib;
+        sib = sib.previousElementSibling
+      )
+        if (sib.tagName === el.tagName) i += 1;
+      return i;
+    };
+    const segs: string[] = [];
+    for (let el: Element | null = node; el; el = el.parentElement)
+      segs.unshift(`${el.tagName.toLowerCase()}[${pos(el)}]`);
+    return `/${segs.join("/")}`;
+  });
+}
+
 /**
  * Inject simplified subtree outlines into the main frame outline for nested iframes.
  * Walks the main tree text, looks for EncodedId labels, and inserts matching subtrees.
@@ -872,6 +912,7 @@ export function injectSubtrees(
  * @returns A Promise resolving to CombinedA11yResult with combined tree text, xpath map, and URL map.
  */
 export async function getAccessibilityTreeWithFrames(
+  experimental: boolean,
   stagehandPage: StagehandPage,
   logger: (l: LogLine) => void,
   rootXPath?: string,
@@ -919,6 +960,7 @@ export async function getAccessibilityTreeWithFrames(
 
     try {
       const res = await getAccessibilityTree(
+        experimental,
         stagehandPage,
         logger,
         selector,
@@ -931,7 +973,13 @@ export async function getAccessibilityTreeWithFrames(
           ? null
           : await getFrameRootBackendNodeId(stagehandPage, frame);
 
-      const frameXpath = frame === main ? "/" : await getFrameRootXpath(frame);
+      let frameXpath;
+      if (experimental) {
+        frameXpath =
+          frame === main ? "/" : await getFrameRootXpathWithShadow(frame);
+      } else {
+        frameXpath = frame === main ? "/" : await getFrameRootXpath(frame);
+      }
 
       // Resolve the CDP frameId for this Playwright Frame (undefined for main)
       const frameId = await getCDPFrameId(stagehandPage, frame);

lib/handlers/actHandler.ts

@@ -18,6 +18,7 @@ import {
   methodHandlerMap,
   fallbackLocatorMethod,
   deepLocator,
+  deepLocatorWithShadow,
 } from "./handlerUtils/actHandlerUtils";
 import { StagehandObserveHandler } from "@/lib/handlers/observeHandler";
 import { StagehandInvalidArgumentError } from "@/types/stagehandErrors";
@@ -30,19 +31,23 @@ export class StagehandActHandler {
   private readonly stagehandPage: StagehandPage;
   private readonly logger: (logLine: LogLine) => void;
   private readonly selfHeal: boolean;
+  private readonly experimental: boolean;
 
   constructor({
     logger,
     stagehandPage,
     selfHeal,
+    experimental,
   }: {
     logger: (logLine: LogLine) => void;
     stagehandPage: StagehandPage;
     selfHeal: boolean;
+    experimental: boolean;
   }) {
     this.logger = logger;
     this.stagehandPage = stagehandPage;
     this.selfHeal = selfHeal;
+    this.experimental = experimental;
   }
 
   /**
@@ -311,7 +316,13 @@ export class StagehandActHandler {
     domSettleTimeoutMs?: number,
   ) {
     const xpath = rawXPath.replace(/^xpath=/i, "").trim();
-    const locator = await deepLocator(this.stagehandPage.page, xpath);
+    let locator;
+    if (this.experimental) {
+      locator = await deepLocatorWithShadow(this.stagehandPage.page, xpath);
+    } else {
+      locator = deepLocator(this.stagehandPage.page, xpath);
+    }
+
     const initialUrl = this.stagehandPage.page.url();
 
     this.logger({

lib/handlers/extractHandler.ts

@@ -18,12 +18,14 @@ export class StagehandExtractHandler {
   private readonly stagehandPage: StagehandPage;
   private readonly logger: (logLine: LogLine) => void;
   private readonly userProvidedInstructions?: string;
+  private readonly experimental: boolean;
 
   constructor({
     stagehand,
     logger,
     stagehandPage,
     userProvidedInstructions,
+    experimental,
   }: {
     stagehand: Stagehand;
     logger: (message: {
@@ -34,11 +36,13 @@ export class StagehandExtractHandler {
     }) => void;
     stagehandPage: StagehandPage;
     userProvidedInstructions?: string;
+    experimental: boolean;
   }) {
     this.stagehand = stagehand;
     this.logger = logger;
     this.stagehandPage = stagehandPage;
     this.userProvidedInstructions = userProvidedInstructions;
+    this.experimental = experimental;
   }
 
   public async extract<T extends z.AnyZodObject>({
@@ -97,7 +101,11 @@ export class StagehandExtractHandler {
     domSettleTimeoutMs?: number,
   ): Promise<{ page_text?: string }> {
     await this.stagehandPage._waitForSettledDom(domSettleTimeoutMs);
-    const tree = await getAccessibilityTree(this.stagehandPage, this.logger);
+    const tree = await getAccessibilityTree(
+      this.experimental,
+      this.stagehandPage,
+      this.logger,
+    );
     this.logger({
       category: "extraction",
       message: "Getting accessibility tree data",
@@ -147,6 +155,7 @@ export class StagehandExtractHandler {
       discoveredIframes,
     } = await (iframes
       ? getAccessibilityTreeWithFrames(
+          this.experimental,
           this.stagehandPage,
           this.logger,
           targetXpath,
@@ -156,14 +165,17 @@ export class StagehandExtractHandler {
           combinedXpathMap: {} as Record<EncodedId, string>,
           discoveredIframes: [] as undefined,
         }))
-      : getAccessibilityTree(this.stagehandPage, this.logger, targetXpath).then(
-          ({ simplified, idToUrl, iframes: frameNodes }) => ({
-            combinedTree: simplified,
-            combinedUrlMap: idToUrl as Record<EncodedId, string>,
-            combinedXpathMap: {} as Record<EncodedId, string>,
-            discoveredIframes: frameNodes,
-          }),
-        ));
+      : getAccessibilityTree(
+          this.experimental,
+          this.stagehandPage,
+          this.logger,
+          targetXpath,
+        ).then(({ simplified, idToUrl, iframes: frameNodes }) => ({
+          combinedTree: simplified,
+          combinedUrlMap: idToUrl as Record<EncodedId, string>,
+          combinedXpathMap: {} as Record<EncodedId, string>,
+          discoveredIframes: frameNodes,
+        })));
 
     this.logger({
       category: "extraction",

lib/handlers/handlerUtils/actHandlerUtils.ts

@@ -101,7 +101,7 @@ async function resolveShadowSegment(
   return hostLoc.locator(`stagehand=${id}`);
 }
 
-export async function deepLocator(
+export async function deepLocatorWithShadow(
   root: Page | FrameLocator,
   xpath: string,
 ): Promise<Locator> {
@@ -179,6 +179,35 @@ export async function deepLocator(
   );
 }
 
+export function deepLocator(root: Page | FrameLocator, xpath: string): Locator {
+  // 1 ─ prepend with slash if not already included
+  if (!xpath.startsWith("/")) xpath = "/" + xpath;
+
+  // 2 ─ split into steps, accumulate until we hit an iframe step
+  const steps = xpath.split("/").filter(Boolean); // tokens
+  let ctx: Page | FrameLocator = root;
+  let buffer: string[] = [];
+
+  const flushIntoFrame = () => {
+    if (buffer.length === 0) return;
+    const selector = "xpath=/" + buffer.join("/");
+    ctx = (ctx as Page | FrameLocator).frameLocator(selector);
+    buffer = [];
+  };
+
+  for (const step of steps) {
+    buffer.push(step);
+    if (IFRAME_STEP_RE.test(step)) {
+      // we've included the <iframe> element in buffer ⇒ descend
+      flushIntoFrame();
+    }
+  }
+
+  // 3 ─ whatever is left in buffer addresses the target *inside* the last ctx
+  const finalSelector = "xpath=/" + buffer.join("/");
+  return (ctx as Page | FrameLocator).locator(finalSelector);
+}
+
 /**
  * A mapping of playwright methods that may be chosen by the LLM to their
  * implementation.