feat: expose previous navigations to the MCP (#419)

We should consider changing the order of element shown in the from the
collectors to be show newest first rather then the current oldest first
approach.

Closes #88

Author: Lightning00Blade

docs/tool-reference.md

@@ -269,6 +269,7 @@
 
 **Parameters:**
 
+- **includePreviousNavigations** (boolean) _(optional)_: Whether to include requests from previous navigations.
 - **pageIdx** (integer) _(optional)_: Page number to return (0-based). When omitted, returns the first page.
 - **pageSize** (integer) _(optional)_: Maximum number of requests to return. When omitted, returns all requests.
 - **resourceTypes** (array) _(optional)_: Filter requests to only return requests of the specified resource types. When omitted or empty, returns all requests.

src/McpContext.ts

@@ -141,9 +141,9 @@ export class McpContext implements Context {
     return context;
   }
 
-  getNetworkRequests(): HTTPRequest[] {
+  getNetworkRequests(includePreviousNavigations?: boolean): HTTPRequest[] {
     const page = this.getSelectedPage();
-    return this.#networkCollector.getData(page);
+    return this.#networkCollector.getData(page, includePreviousNavigations);
   }
 
   getConsoleData(): Array<ConsoleMessage | Error> {
@@ -465,4 +465,21 @@ export class McpContext implements Context {
 
     return locator.wait();
   }
+
+  /**
+   * We need to ignore favicon request as they make our test flaky
+   */
+  async setUpNetworkCollectorForTesting() {
+    this.#networkCollector = new NetworkCollector(this.browser, collect => {
+      return {
+        request: req => {
+          if (req.url().includes('favicon.ico')) {
+            return;
+          }
+          collect(req);
+        },
+      } as ListenerMap;
+    });
+    await this.#networkCollector.init();
+  }
 }

src/McpResponse.ts

@@ -42,6 +42,7 @@ export class McpResponse implements Response {
     include: boolean;
     pagination?: PaginationOptions;
     resourceTypes?: ResourceType[];
+    includePreviousNavigations?: boolean;
   };
   #consoleDataOptions?: {
     include: boolean;
@@ -62,6 +63,7 @@ export class McpResponse implements Response {
     value: boolean,
     options?: PaginationOptions & {
       resourceTypes?: ResourceType[];
+      includePreviousNavigations?: boolean;
     },
   ): void {
     if (!value) {
@@ -79,6 +81,7 @@ export class McpResponse implements Response {
             }
           : undefined,
       resourceTypes: options?.resourceTypes,
+      includePreviousNavigations: options?.includePreviousNavigations,
     };
   }
 
@@ -346,7 +349,9 @@ Call ${handleDialog.name} to handle it before continuing.`);
     response.push(...this.#formatConsoleData(data.consoleData));
 
     if (this.#networkRequestsOptions?.include) {
-      let requests = context.getNetworkRequests();
+      let requests = context.getNetworkRequests(
+        this.#networkRequestsOptions?.includePreviousNavigations,
+      );
 
       // Apply resource type filtering if specified
       if (this.#networkRequestsOptions.resourceTypes?.length) {

src/PageCollector.ts

@@ -133,12 +133,21 @@ export class PageCollector<T> {
     this.storage.delete(page);
   }
 
-  getData(page: Page): T[] {
+  getData(page: Page, includePreviousNavigations?: boolean): T[] {
     const navigations = this.storage.get(page);
     if (!navigations) {
       return [];
     }
-    return navigations[0];
+
+    if (!includePreviousNavigations) {
+      return navigations[0];
+    }
+
+    const data: T[] = [];
+    for (let index = this.#maxNavigationSaved; index >= 0; index--) {
+      data.push(...navigations[index]);
+    }
+    return data;
   }
 
   getIdForResource(resource: WithSymbolId<T>): number {
@@ -164,14 +173,19 @@ export class PageCollector<T> {
 }
 
 export class NetworkCollector extends PageCollector<HTTPRequest> {
-  constructor(browser: Browser) {
-    super(browser, collect => {
+  constructor(
+    browser: Browser,
+    listeners: (
+      collector: (item: HTTPRequest) => void,
+    ) => ListenerMap<PageEvents> = collect => {
       return {
         request: req => {
           collect(req);
         },
       } as ListenerMap;
-    });
+    },
+  ) {
+    super(browser, listeners);
   }
   override splitAfterNavigation(page: Page) {
     const navigations = this.storage.get(page) ?? [];
@@ -190,7 +204,7 @@ export class NetworkCollector extends PageCollector<HTTPRequest> {
     // Keep all requests since the last navigation request including that
     // navigation request itself.
     // Keep the reference
-    if (lastRequestIdx) {
+    if (lastRequestIdx !== -1) {
       const fromCurrentNavigation = requests.splice(lastRequestIdx);
       navigations.unshift(fromCurrentNavigation);
     } else {

src/tools/ToolDefinition.ts

@@ -53,6 +53,7 @@ export interface Response {
     value: boolean,
     options?: PaginationOptions & {
       resourceTypes?: string[];
+      includePreviousNavigations?: boolean;
     },
   ): void;
   setIncludeConsoleData(

src/tools/network.ts

@@ -62,12 +62,18 @@ export const listNetworkRequests = defineTool({
       .describe(
         'Filter requests to only return requests of the specified resource types. When omitted or empty, returns all requests.',
       ),
+    includePreviousNavigations: zod
+      .boolean()
+      .default(false)
+      .optional()
+      .describe('Whether to include requests from previous navigations.'),
   },
   handler: async (request, response) => {
     response.setIncludeNetworkRequests(true, {
       pageSize: request.params.pageSize,
       pageIdx: request.params.pageIdx,
       resourceTypes: request.params.resourceTypes,
+      includePreviousNavigations: request.params.includePreviousNavigations,
     });
   },
 });

tests/PageCollector.test.ts

@@ -250,4 +250,40 @@ describe('NetworkCollector', () => {
     assert.equal(collector.getData(page)[0], navRequest);
     assert.equal(collector.getData(page)[1], request2);
   });
+
+  it('correctly picks up after multiple back to back navigations', async () => {
+    const browser = getMockBrowser();
+    const page = (await browser.pages())[0];
+    const mainFrame = page.mainFrame();
+    const navRequest = getMockRequest({
+      navigationRequest: true,
+      frame: page.mainFrame(),
+    });
+    const navRequest2 = getMockRequest({
+      navigationRequest: true,
+      frame: page.mainFrame(),
+    });
+    const request = getMockRequest();
+
+    const collector = new NetworkCollector(browser);
+    await collector.init();
+    page.emit('request', navRequest);
+    assert.equal(collector.getData(page)[0], navRequest);
+
+    page.emit('framenavigated', mainFrame);
+    assert.equal(collector.getData(page).length, 1);
+    assert.equal(collector.getData(page)[0], navRequest);
+
+    page.emit('request', navRequest2);
+    assert.equal(collector.getData(page).length, 2);
+    assert.equal(collector.getData(page)[0], navRequest);
+    assert.equal(collector.getData(page)[1], navRequest2);
+
+    page.emit('framenavigated', mainFrame);
+    assert.equal(collector.getData(page).length, 1);
+    assert.equal(collector.getData(page)[0], navRequest2);
+
+    page.emit('request', request);
+    assert.equal(collector.getData(page).length, 2);
+  });
 });

tests/tools/network.test.js.snapshot

@@ -0,0 +1,36 @@
+exports[`network > network_get_request > should get request from previous navigations 1`] = `
+# get_request response
+## Request http://localhost:<port>/one
+Status:  [success - 200]
+### Request Headers
+- accept-language:en-US,en;q=0.9
+- upgrade-insecure-requests:1
+- user-agent:<user-agent>
+- sec-ch-ua:"Chromium";v="<version>", "Not?A_Brand";v="8"
+- sec-ch-ua-mobile:?0
+- sec-ch-ua-platform:"<os>"
+### Response Headers
+- connection:keep-alive
+- content-length:239
+- content-type:text/html; charset=utf-8
+- date:<long date>
+- keep-alive:timeout=5
+### Response Body
+<not available anymore>
+`;
+
+exports[`network > network_list_requests > list requests form current navigations only 1`] = `
+# list_request response
+## Network requests
+Showing 1-1 of 1 (Page 1 of 1).
+reqid=3 GET http://localhost:<port>/three [success - 200]
+`;
+
+exports[`network > network_list_requests > list requests from previous navigations 1`] = `
+# list_request response
+## Network requests
+Showing 1-3 of 3 (Page 1 of 1).
+reqid=1 GET http://localhost:<port>/one [success - 200]
+reqid=2 GET http://localhost:<port>/two [success - 200]
+reqid=3 GET http://localhost:<port>/three [success - 200]
+`;

tests/tools/network.test.ts

@@ -10,9 +10,11 @@ import {
   getNetworkRequest,
   listNetworkRequests,
 } from '../../src/tools/network.js';
-import {withBrowser} from '../utils.js';
+import {serverHooks} from '../server.js';
+import {html, withBrowser, stabilizeResponseOutput} from '../utils.js';
 
 describe('network', () => {
+  const server = serverHooks();
   describe('network_list_requests', () => {
     it('list requests', async () => {
       await withBrowser(async (response, context) => {
@@ -21,23 +23,72 @@ describe('network', () => {
         assert.strictEqual(response.networkRequestsPageIdx, undefined);
       });
     });
+
+    it('list requests form current navigations only', async t => {
+      server.addHtmlRoute('/one', html`<main>First</main>`);
+      server.addHtmlRoute('/two', html`<main>Second</main>`);
+      server.addHtmlRoute('/three', html`<main>Third</main>`);
+
+      await withBrowser(async (response, context) => {
+        await context.setUpNetworkCollectorForTesting();
+        const page = context.getSelectedPage();
+        await page.goto(server.getRoute('/one'));
+        await page.goto(server.getRoute('/two'));
+        await page.goto(server.getRoute('/three'));
+        await listNetworkRequests.handler(
+          {
+            params: {},
+          },
+          response,
+          context,
+        );
+        const responseData = await response.handle('list_request', context);
+        t.assert.snapshot?.(stabilizeResponseOutput(responseData[0].text));
+      });
+    });
+
+    it('list requests from previous navigations', async t => {
+      server.addHtmlRoute('/one', html`<main>First</main>`);
+      server.addHtmlRoute('/two', html`<main>Second</main>`);
+      server.addHtmlRoute('/three', html`<main>Third</main>`);
+
+      await withBrowser(async (response, context) => {
+        await context.setUpNetworkCollectorForTesting();
+        const page = context.getSelectedPage();
+        await page.goto(server.getRoute('/one'));
+        await page.goto(server.getRoute('/two'));
+        await page.goto(server.getRoute('/three'));
+        await listNetworkRequests.handler(
+          {
+            params: {
+              includePreviousNavigations: true,
+            },
+          },
+          response,
+          context,
+        );
+        const responseData = await response.handle('list_request', context);
+        t.assert.snapshot?.(stabilizeResponseOutput(responseData[0].text));
+      });
+    });
   });
   describe('network_get_request', () => {
     it('attaches request', async () => {
       await withBrowser(async (response, context) => {
-        const page = await context.getSelectedPage();
+        const page = context.getSelectedPage();
         await page.goto('data:text/html,<div>Hello MCP</div>');
         await getNetworkRequest.handler(
           {params: {reqid: 1}},
           response,
           context,
         );
+
         assert.equal(response.attachedNetworkRequestId, 1);
       });
     });
     it('should not add the request list', async () => {
       await withBrowser(async (response, context) => {
-        const page = await context.getSelectedPage();
+        const page = context.getSelectedPage();
         await page.goto('data:text/html,<div>Hello MCP</div>');
         await getNetworkRequest.handler(
           {params: {reqid: 1}},
@@ -47,5 +98,30 @@ describe('network', () => {
         assert(!response.includeNetworkRequests);
       });
     });
+    it('should get request from previous navigations', async t => {
+      server.addHtmlRoute('/one', html`<main>First</main>`);
+      server.addHtmlRoute('/two', html`<main>Second</main>`);
+      server.addHtmlRoute('/three', html`<main>Third</main>`);
+
+      await withBrowser(async (response, context) => {
+        await context.setUpNetworkCollectorForTesting();
+        const page = context.getSelectedPage();
+        await page.goto(server.getRoute('/one'));
+        await page.goto(server.getRoute('/two'));
+        await page.goto(server.getRoute('/three'));
+        await getNetworkRequest.handler(
+          {
+            params: {
+              reqid: 1,
+            },
+          },
+          response,
+          context,
+        );
+        const responseData = await response.handle('get_request', context);
+
+        t.assert.snapshot?.(stabilizeResponseOutput(responseData[0].text));
+      });
+    });
   });
 });

tests/tools/snapshot.test.ts

@@ -21,7 +21,7 @@ describe('snapshot', () => {
   describe('browser_wait_for', () => {
     it('should work', async () => {
       await withBrowser(async (response, context) => {
-        const page = await context.getSelectedPage();
+        const page = context.getSelectedPage();
 
         await page.setContent(
           html`<main><span>Hello</span><span> </span><div>World</div></main>`,
@@ -98,7 +98,7 @@ describe('snapshot', () => {
 
     it('should work with iframe content', async () => {
       await withBrowser(async (response, context) => {
-        const page = await context.getSelectedPage();
+        const page = context.getSelectedPage();
 
         await page.setContent(
           html`<h1>Top level</h1>