[fix] ensure selfHeal respects arguments (#897)

# why
On self healing we were finding the selector, but nor remembering the
arguments previously passed; so either they were empty or the LLM was
hallucinating them. This PR fixes this

# what changed
Updated the logic from selfHealing to reuse the previously passed
arguments and method

# test plan
- [x] Added evals to act and regression (`evals/tasks/heal_*.ts`)

---------

Co-authored-by: Sean McGuire <[email protected]>

Author: miguelg719

.changeset/chilly-geckos-burn.md

@@ -0,0 +1,5 @@
+---
+"@browserbasehq/stagehand": patch
+---
+
+Fix selfHeal to remember intially received arguments

evals/evals.config.json

@@ -386,6 +386,18 @@
     {
       "name": "nested_iframes_2",
       "categories": ["act"]
+    },
+    {
+      "name": "heal_scroll_50",
+      "categories": ["act"]
+    },
+    {
+      "name": "heal_simple_google_search",
+      "categories": ["regression", "act"]
+    },
+    {
+      "name": "heal_custom_dropdown",
+      "categories": ["act"]
     }
   ]
 }

evals/initStagehand.ts

@@ -48,6 +48,7 @@ const StagehandConfig = {
       },
     },
   },
+  selfHeal: true,
 };
 
 /**

evals/tasks/heal_custom_dropdown.ts

@@ -0,0 +1,65 @@
+import { EvalFunction } from "@/types/evals";
+
+export const heal_custom_dropdown: EvalFunction = async ({
+  debugUrl,
+  sessionUrl,
+  stagehand,
+  logger,
+}) => {
+  /**
+   * This eval is meant to test whether we do not incorrectly attempt
+   * the selectOptionFromDropdown method (defined in actHandlerUtils.ts) on a
+   * 'dropdown' that is not a <select> element.
+   *
+   * This kind of dropdown must be clicked to be expanded before being interacted
+   * with.
+   */
+
+  try {
+    const page = stagehand.page;
+    await page.goto(
+      "https://browserbase.github.io/stagehand-eval-sites/sites/expand-dropdown/",
+    );
+
+    await page.act({
+      description: "The dropdown",
+      selector: "/html/not-a-dropdown",
+      arguments: [],
+      method: "click",
+    });
+
+    // we are expecting stagehand to click the dropdown to expand it,
+    // and therefore the available options should now be contained in the full
+    // a11y tree.
+
+    // to test, we'll grab the full a11y tree, and make sure it contains 'Canada'
+    const extraction = await page.extract();
+    const fullTree = extraction.page_text;
+
+    if (fullTree.includes("Canada")) {
+      return {
+        _success: true,
+        debugUrl,
+        sessionUrl,
+        logs: logger.getLogs(),
+      };
+    }
+    return {
+      _success: false,
+      message: "unable to expand the dropdown",
+      debugUrl,
+      sessionUrl,
+      logs: logger.getLogs(),
+    };
+  } catch (error) {
+    return {
+      _success: false,
+      message: `error attempting to select an option from the dropdown: ${error.message}`,
+      debugUrl,
+      sessionUrl,
+      logs: logger.getLogs(),
+    };
+  } finally {
+    await stagehand.close();
+  }
+};

evals/tasks/heal_scroll_50.ts

@@ -0,0 +1,60 @@
+import { EvalFunction } from "@/types/evals";
+
+export const heal_scroll_50: EvalFunction = async ({
+  debugUrl,
+  sessionUrl,
+  stagehand,
+  logger,
+}) => {
+  try {
+    await stagehand.page.goto(
+      "https://browserbase.github.io/stagehand-eval-sites/sites/aigrant/",
+    );
+    await stagehand.page.act({
+      description: "the element to scroll on",
+      selector: "/html/body/div/div/button",
+      arguments: ["50%"],
+      method: "scrollTo",
+    });
+
+    await new Promise((resolve) => setTimeout(resolve, 5000));
+
+    // Get the current scroll position and total scroll height
+    const scrollInfo = await stagehand.page.evaluate(() => {
+      return {
+        scrollTop: window.scrollY + window.innerHeight / 2,
+        scrollHeight: document.documentElement.scrollHeight,
+      };
+    });
+
+    const halfwayScroll = scrollInfo.scrollHeight / 2;
+    const halfwayReached =
+      Math.abs(scrollInfo.scrollTop - halfwayScroll) <= 200;
+    const evaluationResult = halfwayReached
+      ? {
+          _success: true,
+          logs: logger.getLogs(),
+          debugUrl,
+          sessionUrl,
+        }
+      : {
+          _success: false,
+          logs: logger.getLogs(),
+          debugUrl,
+          sessionUrl,
+          message: `Scroll position (${scrollInfo.scrollTop}px) is not halfway down the page (${halfwayScroll}px).`,
+        };
+
+    return evaluationResult;
+  } catch (error) {
+    return {
+      _success: false,
+      error: error,
+      logs: logger.getLogs(),
+      debugUrl,
+      sessionUrl,
+    };
+  } finally {
+    await stagehand.close();
+  }
+};

evals/tasks/heal_simple_google_search.ts

@@ -0,0 +1,45 @@
+import { EvalFunction } from "@/types/evals";
+
+export const heal_simple_google_search: EvalFunction = async ({
+  debugUrl,
+  sessionUrl,
+  stagehand,
+  logger,
+}) => {
+  try {
+    await stagehand.page.goto(
+      "https://browserbase.github.io/stagehand-eval-sites/sites/google/",
+    );
+
+    await stagehand.page.act({
+      description: "The search bar",
+      selector: "/html/not-the-search-bar",
+      arguments: ["OpenAI"],
+      method: "fill",
+    });
+
+    await stagehand.page.act("click the google search button");
+
+    const expectedUrl =
+      "https://browserbase.github.io/stagehand-eval-sites/sites/google/openai.html";
+    const currentUrl = stagehand.page.url();
+
+    return {
+      _success: currentUrl.startsWith(expectedUrl),
+      currentUrl,
+      debugUrl,
+      sessionUrl,
+      logs: logger.getLogs(),
+    };
+  } catch (error) {
+    return {
+      _success: false,
+      error: error,
+      debugUrl,
+      sessionUrl,
+      logs: logger.getLogs(),
+    };
+  } finally {
+    await stagehand.close();
+  }
+};

lib/handlers/actHandler.ts

@@ -147,10 +147,35 @@ export class StagehandActHandler {
           : method
             ? `${method} ${observe.description}`
             : observe.description;
-        // Call act with the ObserveResult description
-        return await this.stagehandPage.act({
-          action: actCommand,
+        const instruction = buildActObservePrompt(
+          actCommand,
+          Object.values(SupportedPlaywrightAction),
+          {},
+        );
+        const observeResults = await this.stagehandPage.observe({
+          instruction,
         });
+        if (observeResults.length === 0) {
+          return {
+            success: false,
+            message: `Failed to self heal act: No observe results found for action`,
+            action: actCommand,
+          };
+        }
+        const element: ObserveResult = observeResults[0];
+        await this._performPlaywrightMethod(
+          // override previously provided method and arguments
+          observe.method,
+          observe.arguments,
+          // only update selector
+          element.selector,
+          domSettleTimeoutMs,
+        );
+        return {
+          success: true,
+          message: `Action [${element.method}] performed successfully on selector: ${element.selector}`,
+          action: observe.description || `ObserveResult action (${method})`,
+        };
       } catch (err) {
         this.logger({
           category: "action",
@@ -282,9 +307,10 @@ export class StagehandActHandler {
   private async _performPlaywrightMethod(
     method: string,
     args: unknown[],
-    xpath: string,
+    rawXPath: string,
     domSettleTimeoutMs?: number,
   ) {
+    const xpath = rawXPath.replace(/^xpath=/i, "").trim();
     const locator = deepLocator(this.stagehandPage.page, xpath).first();
     const initialUrl = this.stagehandPage.page.url();
 

lib/handlers/handlerUtils/actHandlerUtils.ts

@@ -8,12 +8,8 @@ import { StagehandClickError } from "@/types/stagehandErrors";
 
 const IFRAME_STEP_RE = /^iframe(\[[^\]]+])?$/i;
 
-export function deepLocator(
-  root: Page | FrameLocator,
-  rawXPath: string,
-): Locator {
-  // 1 ─ strip optional 'xpath=' prefix and whitespace
-  let xpath = rawXPath.replace(/^xpath=/i, "").trim();
+export function deepLocator(root: Page | FrameLocator, xpath: string): Locator {
+  // 1 ─ prepend with slash if not already included
   if (!xpath.startsWith("/")) xpath = "/" + xpath;
 
   // 2 ─ split into steps, accumulate until we hit an iframe step
@@ -79,8 +75,7 @@ export async function scrollToNextChunk(ctx: MethodHandlerContext) {
       ({ xpath }) => {
         const elementNode = getNodeFromXpath(xpath);
         if (!elementNode || elementNode.nodeType !== Node.ELEMENT_NODE) {
-          console.warn(`Could not locate element to scroll by its height.`);
-          return Promise.resolve();
+          throw Error(`Could not locate element to scroll on.`);
         }
 
         const element = elementNode as HTMLElement;
@@ -143,8 +138,7 @@ export async function scrollToPreviousChunk(ctx: MethodHandlerContext) {
       ({ xpath }) => {
         const elementNode = getNodeFromXpath(xpath);
         if (!elementNode || elementNode.nodeType !== Node.ELEMENT_NODE) {
-          console.warn(`Could not locate element to scroll by its height.`);
-          return Promise.resolve();
+          throw Error(`Could not locate element to scroll on.`);
         }
 
         const element = elementNode as HTMLElement;
@@ -246,8 +240,7 @@ export async function scrollElementToPercentage(ctx: MethodHandlerContext) {
 
         const elementNode = getNodeFromXpath(xpath);
         if (!elementNode || elementNode.nodeType !== Node.ELEMENT_NODE) {
-          console.warn(`Could not locate element to scroll on.`);
-          return;
+          throw Error(`Could not locate element to scroll on.`);
         }
 
         const element = elementNode as HTMLElement;