fix: incorrect nested iframe xpaths (#891)

Author: seanmcguire12

.changeset/rich-bobcats-sneeze.md

@@ -0,0 +1,5 @@
+---
+"@browserbasehq/stagehand": patch
+---
+
+fix: nested iframe xpath bug

evals/evals.config.json

@@ -382,6 +382,10 @@
     {
       "name": "hidden_input_dropdown",
       "categories": ["act"]
+    },
+    {
+      "name": "nested_iframes_2",
+      "categories": ["act"]
     }
   ]
 }

evals/tasks/nested_iframes_2.ts

@@ -0,0 +1,48 @@
+import { EvalFunction } from "@/types/evals";
+import { FrameLocator } from "playwright";
+
+export const nested_iframes_2: EvalFunction = async ({
+  debugUrl,
+  sessionUrl,
+  stagehand,
+  logger,
+}) => {
+  const page = stagehand.page;
+  try {
+    await page.goto(
+      "https://browserbase.github.io/stagehand-eval-sites/sites/nested-iframes-2/",
+    );
+
+    await page.act({
+      action: "click the button called 'click me (inner 2)'",
+      iframes: true,
+    });
+
+    const inner: FrameLocator = page
+      .frameLocator('iframe[src="iframe2.html"]')
+      .frameLocator('iframe[src="inner2.html"]');
+
+    const messageText = await inner.locator("#msg").textContent();
+
+    const passed: boolean =
+      messageText.toLowerCase().trim() ===
+      "clicked the button in the second inner iframe";
+
+    return {
+      _success: passed,
+      logs: logger.getLogs(),
+      debugUrl,
+      sessionUrl,
+    };
+  } catch (error) {
+    return {
+      _success: false,
+      logs: logger.getLogs(),
+      debugUrl,
+      sessionUrl,
+      error,
+    };
+  } finally {
+    await stagehand.close();
+  }
+};

lib/a11y/utils.ts

@@ -894,6 +894,7 @@ export async function getAccessibilityTreeWithFrames(
       const frameId = await getCDPFrameId(stagehandPage, frame);
 
       snapshots.push({
+        frame,
         tree: res.simplified.trimEnd(),
         xpathMap: res.xpathMap as Record<EncodedId, string>,
         urlMap: res.idToUrl as Record<string, string>,
@@ -918,15 +919,15 @@ export async function getAccessibilityTreeWithFrames(
   const combinedXpathMap: Record<EncodedId, string> = {};
   const combinedUrlMap: Record<EncodedId, string> = {};
 
-  const seg = new Map<Frame | null, string>();
-  for (const s of snapshots) seg.set(s.parentFrame, s.frameXpath);
+  const seg = new Map<Frame, string>();
+  for (const s of snapshots) seg.set(s.frame, s.frameXpath);
 
   /* recursively build the full prefix for a frame */
   function fullPrefix(f: Frame | null): string {
-    if (!f) return ""; // reached main
+    if (!f || f === main) return "";
     const parent = f.parentFrame();
     const above = fullPrefix(parent);
-    const hop = seg.get(parent) ?? "";
+    const hop = seg.get(f) ?? "";
     return hop === "/"
       ? above
       : above

types/context.ts

@@ -105,6 +105,7 @@ export interface CombinedA11yResult {
 }
 
 export interface FrameSnapshot {
+  frame: Frame;
   tree: string;
   xpathMap: Record<EncodedId, string>;
   urlMap: Record<EncodedId, string>;