cdp debug logging [STG-593]

.gitignore

@@ -19,3 +19,4 @@ eval-summary.json
 package-lock.json
 evals/deterministic/tests/BrowserContext/tmp-test.har
 lib/version.ts
+*.log

evals/deterministic/tests/page/livePageProxy.test.ts

@@ -8,13 +8,16 @@ test.describe("StagehandPage - live page proxy", () => {
     await stagehand.init();
 
     const page = stagehand.page;
-    await page.goto("https://browserbase.github.io/stagehand-eval-sites/sites/five-tab/");
+    await page.goto(
+      "https://browserbase.github.io/stagehand-eval-sites/sites/five-tab/",
+    );
     await page.locator("body > button").click();
-    await new Promise(resolve => setTimeout(resolve, 1000));
-    await page.waitForURL('**/page2.html', {waitUntil: "commit"});
+    await new Promise((resolve) => setTimeout(resolve, 1000));
+    await page.waitForURL("**/page2.html", { waitUntil: "commit" });
     // await new Promise(resolve => setTimeout(resolve, 1000));
     const currentURL = page.url();
-    const expectedURL = "https://browserbase.github.io/stagehand-eval-sites/sites/five-tab/page2.html";
+    const expectedURL =
+      "https://browserbase.github.io/stagehand-eval-sites/sites/five-tab/page2.html";
 
     expect(currentURL).toBe(expectedURL);
 
@@ -26,9 +29,11 @@ test.describe("StagehandPage - live page proxy", () => {
     await stagehand.init();
 
     const page = stagehand.page;
-    await page.goto("https://browserbase.github.io/stagehand-eval-sites/sites/five-tab/");
+    await page.goto(
+      "https://browserbase.github.io/stagehand-eval-sites/sites/five-tab/",
+    );
     await page.locator("body > button").click();
-    await new Promise(resolve => setTimeout(resolve, 1000));
+    await new Promise((resolve) => setTimeout(resolve, 1000));
 
     const expectedNumPages = 2;
     const actualNumPages = stagehand.context.pages().length;

examples/example.ts

@@ -6,6 +6,7 @@
  */
 import { Stagehand } from "@browserbasehq/stagehand";
 import StagehandConfig from "../stagehand.config";
+import { createStagehandApiLogger } from "../lib/stagehandApiLogger";
 
 async function example(stagehand: Stagehand) {
   /**
@@ -18,6 +19,7 @@ async function example(stagehand: Stagehand) {
 
 (async () => {
   const stagehand = new Stagehand({
+    logger: createStagehandApiLogger(),
     ...StagehandConfig,
   });
   await stagehand.init();

examples/test-api-logger.ts

@@ -0,0 +1,44 @@
+import { Stagehand } from "../lib/index";
+import { createStagehandApiLogger } from "../lib/stagehandApiLogger";
+
+async function testApiLogger() {
+  console.log("Starting test with custom sh:api logger...\n");
+
+  const stagehand = new Stagehand({
+    env: "LOCAL",
+    logger: createStagehandApiLogger(),
+    localBrowserLaunchOptions: {
+      headless: false,
+    },
+  });
+
+  try {
+    await stagehand.init();
+    const page = stagehand.page;
+
+    console.log("\nNavigating to example.com...");
+    await page.goto("https://example.com");
+
+    console.log("\nExtracting page title...");
+    const title = await page.extract({
+      instruction: "Extract the main heading of the page",
+    });
+    console.log("Extracted title:", title);
+
+    console.log("\nPerforming a simple action...");
+    await page.act({
+      action: "click on the 'More information' link",
+    });
+
+    console.log("\nObserving the page...");
+    const observation = await page.observe();
+    console.log("Observation result:", observation);
+  } catch (error) {
+    console.error("Error during test:", error);
+  } finally {
+    await stagehand.close();
+  }
+}
+
+// Run the test
+testApiLogger().catch(console.error);

lib/StagehandContext.ts

@@ -1,3 +1,4 @@
+import "./debug";
 import type {
   BrowserContext as PlaywrightContext,
   Page as PlaywrightPage,

lib/StagehandPage.ts

@@ -1,3 +1,4 @@
+import "./debug";
 import { Browserbase } from "@browserbasehq/sdk";
 import type { CDPSession, Page as PlaywrightPage, Frame } from "playwright";
 import { chromium } from "playwright";
@@ -33,6 +34,7 @@ import {
 import { StagehandAPIError } from "@/types/stagehandApiErrors";
 import { scriptContent } from "@/lib/dom/build/scriptContent";
 import type { Protocol } from "devtools-protocol";
+import { markStagehandCDPCall } from "./debug";
 
 export class StagehandPage {
   private stagehand: Stagehand;
@@ -351,13 +353,13 @@ ${scriptContent} \
 
           // Handle goto specially
           if (prop === "goto") {
-            const rawGoto: typeof target.goto =
-              Object.getPrototypeOf(target).goto.bind(target);
             return async (url: string, options: GotoOptions) => {
               this.intContext.setActivePage(this);
+
+              // Use the raw page directly for navigation
               const result = this.api
                 ? await this.api.goto(url, options)
-                : await rawGoto(url, options);
+                : await target.goto(url, options);
 
               this.stagehand.addToHistory("navigate", { url, options }, result);
 
@@ -381,7 +383,24 @@ ${scriptContent} \
                   });
                 }
                 await target.waitForLoadState("domcontentloaded");
-                await this._waitForSettledDom();
+                // Skip DOM settling during initial navigation
+                if (this.initialized) {
+                  try {
+                    await this._waitForSettledDom();
+                  } catch (err) {
+                    this.stagehand.log({
+                      category: "navigation",
+                      message: "Failed to wait for settled DOM, continuing",
+                      level: 2,
+                      auxiliary: {
+                        error: {
+                          value: (err as Error).message,
+                          type: "string",
+                        },
+                      },
+                    });
+                  }
+                }
               }
               return result;
             };
@@ -482,8 +501,13 @@ ${scriptContent} \
     const hasDoc = !!(await this.page.title().catch(() => false));
     if (!hasDoc) await this.page.waitForLoadState("domcontentloaded");
 
+    markStagehandCDPCall("Network.enable");
     await client.send("Network.enable");
+
+    markStagehandCDPCall("Page.enable");
     await client.send("Page.enable");
+
+    markStagehandCDPCall("Target.setAutoAttach");
     await client.send("Target.setAutoAttach", {
       autoAttach: true,
       waitForDebuggerOnStart: false,
@@ -964,7 +988,9 @@ ${scriptContent} \
     target: PlaywrightPage | Frame = this.page,
   ): Promise<CDPSession> {
     const cached = this.cdpClients.get(target);
-    if (cached) return cached;
+    if (cached) {
+      return cached;
+    }
 
     try {
       const session = await this.context.newCDPSession(target);
@@ -1000,10 +1026,15 @@ ${scriptContent} \
   ): Promise<T> {
     const client = await this.getCDPClient(target ?? this.page);
 
-    return client.send(
+    // Mark this as a Stagehand CDP call
+    markStagehandCDPCall(method);
+
+    const result = (await client.send(
       method as Parameters<CDPSession["send"]>[0],
       params as Parameters<CDPSession["send"]>[1],
-    ) as Promise<T>;
+    )) as T;
+
+    return result;
   }
 
   /** Enable a CDP domain (e.g. `"Network"` or `"DOM"`) on the chosen target. */

lib/a11y/utils.ts

@@ -14,6 +14,7 @@ import {
 } from "../../types/context";
 import { StagehandPage } from "../StagehandPage";
 import { LogLine } from "../../types/log";
+import { markStagehandCDPCall } from "../debug";
 import {
   ContentFrameNotFoundError,
   StagehandDomProcessError,
@@ -144,10 +145,15 @@ export async function buildBackendIdMaps(
 
   try {
     // 1. full DOM tree
-    const { root } = (await session.send("DOM.getDocument", {
+    markStagehandCDPCall("DOM.getDocument");
+    const result = (await session.send("DOM.getDocument", {
       depth: -1,
       pierce: true,
-    })) as { root: DOMNode };
+    })) as {
+      root: DOMNode;
+    };
+
+    const { root } = result;
 
     // 2. pick start node + root frame-id
     let startNode: DOMNode = root;
@@ -462,7 +468,9 @@ export async function getCDPFrameId(
   try {
     const sess = await sp.context.newCDPSession(frame); // throws if detached
 
+    markStagehandCDPCall("Page.getFrameTree");
     const ownResp = (await sess.send("Page.getFrameTree")) as unknown;
+
     const { frameTree } = ownResp as { frameTree: CdpFrameTree };
 
     return frameTree.frame.id; // root of OOPIF
@@ -684,10 +692,13 @@ export async function getFrameRootBackendNodeId(
   }
 
   // Retrieve the DOM node that owns the frame via CDP
-  const { backendNodeId } = (await cdp.send("DOM.getFrameOwner", {
+  markStagehandCDPCall("DOM.getFrameOwner");
+  const frameOwnerResult = (await cdp.send("DOM.getFrameOwner", {
     frameId: fid,
   })) as FrameOwnerResult;
 
+  const { backendNodeId } = frameOwnerResult;
+
   return backendNodeId ?? null;
 }