diff --git a/docs/tool-reference.md b/docs/tool-reference.md index 68c1f9b3..d7cbc2f4 100644 --- a/docs/tool-reference.md +++ b/docs/tool-reference.md @@ -269,6 +269,7 @@ **Parameters:** +- **includePreviousNavigations** (boolean) _(optional)_: Whether to include requests from previous navigations. - **pageIdx** (integer) _(optional)_: Page number to return (0-based). When omitted, returns the first page. - **pageSize** (integer) _(optional)_: Maximum number of requests to return. When omitted, returns all requests. - **resourceTypes** (array) _(optional)_: Filter requests to only return requests of the specified resource types. When omitted or empty, returns all requests. diff --git a/src/McpContext.ts b/src/McpContext.ts index 4f7641d5..dd9186a6 100644 --- a/src/McpContext.ts +++ b/src/McpContext.ts @@ -141,9 +141,9 @@ export class McpContext implements Context { return context; } - getNetworkRequests(): HTTPRequest[] { + getNetworkRequests(includePreviousNavigations?: boolean): HTTPRequest[] { const page = this.getSelectedPage(); - return this.#networkCollector.getData(page); + return this.#networkCollector.getData(page, includePreviousNavigations); } getConsoleData(): Array { @@ -465,4 +465,21 @@ export class McpContext implements Context { return locator.wait(); } + + /** + * We need to ignore favicon request as they make our test flaky + */ + async setUpNetworkCollectorForTesting() { + this.#networkCollector = new NetworkCollector(this.browser, collect => { + return { + request: req => { + if (req.url().includes('favicon.ico')) { + return; + } + collect(req); + }, + } as ListenerMap; + }); + await this.#networkCollector.init(); + } } diff --git a/src/McpResponse.ts b/src/McpResponse.ts index 31bbdcc5..03a81465 100644 --- a/src/McpResponse.ts +++ b/src/McpResponse.ts @@ -42,6 +42,7 @@ export class McpResponse implements Response { include: boolean; pagination?: PaginationOptions; resourceTypes?: ResourceType[]; + includePreviousNavigations?: boolean; }; #consoleDataOptions?: { include: boolean; @@ -62,6 +63,7 @@ export class McpResponse implements Response { value: boolean, options?: PaginationOptions & { resourceTypes?: ResourceType[]; + includePreviousNavigations?: boolean; }, ): void { if (!value) { @@ -79,6 +81,7 @@ export class McpResponse implements Response { } : undefined, resourceTypes: options?.resourceTypes, + includePreviousNavigations: options?.includePreviousNavigations, }; } @@ -346,7 +349,9 @@ Call ${handleDialog.name} to handle it before continuing.`); response.push(...this.#formatConsoleData(data.consoleData)); if (this.#networkRequestsOptions?.include) { - let requests = context.getNetworkRequests(); + let requests = context.getNetworkRequests( + this.#networkRequestsOptions?.includePreviousNavigations, + ); // Apply resource type filtering if specified if (this.#networkRequestsOptions.resourceTypes?.length) { diff --git a/src/PageCollector.ts b/src/PageCollector.ts index 44d1363f..39168dc8 100644 --- a/src/PageCollector.ts +++ b/src/PageCollector.ts @@ -133,12 +133,21 @@ export class PageCollector { this.storage.delete(page); } - getData(page: Page): T[] { + getData(page: Page, includePreviousNavigations?: boolean): T[] { const navigations = this.storage.get(page); if (!navigations) { return []; } - return navigations[0]; + + if (!includePreviousNavigations) { + return navigations[0]; + } + + const data: T[] = []; + for (let index = this.#maxNavigationSaved; index >= 0; index--) { + data.push(...navigations[index]); + } + return data; } getIdForResource(resource: WithSymbolId): number { @@ -164,14 +173,19 @@ export class PageCollector { } export class NetworkCollector extends PageCollector { - constructor(browser: Browser) { - super(browser, collect => { + constructor( + browser: Browser, + listeners: ( + collector: (item: HTTPRequest) => void, + ) => ListenerMap = collect => { return { request: req => { collect(req); }, } as ListenerMap; - }); + }, + ) { + super(browser, listeners); } override splitAfterNavigation(page: Page) { const navigations = this.storage.get(page) ?? []; @@ -190,7 +204,7 @@ export class NetworkCollector extends PageCollector { // Keep all requests since the last navigation request including that // navigation request itself. // Keep the reference - if (lastRequestIdx) { + if (lastRequestIdx !== -1) { const fromCurrentNavigation = requests.splice(lastRequestIdx); navigations.unshift(fromCurrentNavigation); } else { diff --git a/src/tools/ToolDefinition.ts b/src/tools/ToolDefinition.ts index 19415dd2..966cf77b 100644 --- a/src/tools/ToolDefinition.ts +++ b/src/tools/ToolDefinition.ts @@ -53,6 +53,7 @@ export interface Response { value: boolean, options?: PaginationOptions & { resourceTypes?: string[]; + includePreviousNavigations?: boolean; }, ): void; setIncludeConsoleData( diff --git a/src/tools/network.ts b/src/tools/network.ts index d47fa996..d684d05b 100644 --- a/src/tools/network.ts +++ b/src/tools/network.ts @@ -62,12 +62,18 @@ export const listNetworkRequests = defineTool({ .describe( 'Filter requests to only return requests of the specified resource types. When omitted or empty, returns all requests.', ), + includePreviousNavigations: zod + .boolean() + .default(false) + .optional() + .describe('Whether to include requests from previous navigations.'), }, handler: async (request, response) => { response.setIncludeNetworkRequests(true, { pageSize: request.params.pageSize, pageIdx: request.params.pageIdx, resourceTypes: request.params.resourceTypes, + includePreviousNavigations: request.params.includePreviousNavigations, }); }, }); diff --git a/tests/PageCollector.test.ts b/tests/PageCollector.test.ts index a916fdc2..5da1d31a 100644 --- a/tests/PageCollector.test.ts +++ b/tests/PageCollector.test.ts @@ -250,4 +250,40 @@ describe('NetworkCollector', () => { assert.equal(collector.getData(page)[0], navRequest); assert.equal(collector.getData(page)[1], request2); }); + + it('correctly picks up after multiple back to back navigations', async () => { + const browser = getMockBrowser(); + const page = (await browser.pages())[0]; + const mainFrame = page.mainFrame(); + const navRequest = getMockRequest({ + navigationRequest: true, + frame: page.mainFrame(), + }); + const navRequest2 = getMockRequest({ + navigationRequest: true, + frame: page.mainFrame(), + }); + const request = getMockRequest(); + + const collector = new NetworkCollector(browser); + await collector.init(); + page.emit('request', navRequest); + assert.equal(collector.getData(page)[0], navRequest); + + page.emit('framenavigated', mainFrame); + assert.equal(collector.getData(page).length, 1); + assert.equal(collector.getData(page)[0], navRequest); + + page.emit('request', navRequest2); + assert.equal(collector.getData(page).length, 2); + assert.equal(collector.getData(page)[0], navRequest); + assert.equal(collector.getData(page)[1], navRequest2); + + page.emit('framenavigated', mainFrame); + assert.equal(collector.getData(page).length, 1); + assert.equal(collector.getData(page)[0], navRequest2); + + page.emit('request', request); + assert.equal(collector.getData(page).length, 2); + }); }); diff --git a/tests/tools/network.test.js.snapshot b/tests/tools/network.test.js.snapshot new file mode 100644 index 00000000..0a74003c --- /dev/null +++ b/tests/tools/network.test.js.snapshot @@ -0,0 +1,36 @@ +exports[`network > network_get_request > should get request from previous navigations 1`] = ` +# get_request response +## Request http://localhost:/one +Status: [success - 200] +### Request Headers +- accept-language:en-US,en;q=0.9 +- upgrade-insecure-requests:1 +- user-agent: +- sec-ch-ua:"Chromium";v="", "Not?A_Brand";v="8" +- sec-ch-ua-mobile:?0 +- sec-ch-ua-platform:"" +### Response Headers +- connection:keep-alive +- content-length:239 +- content-type:text/html; charset=utf-8 +- date: +- keep-alive:timeout=5 +### Response Body + +`; + +exports[`network > network_list_requests > list requests form current navigations only 1`] = ` +# list_request response +## Network requests +Showing 1-1 of 1 (Page 1 of 1). +reqid=3 GET http://localhost:/three [success - 200] +`; + +exports[`network > network_list_requests > list requests from previous navigations 1`] = ` +# list_request response +## Network requests +Showing 1-3 of 3 (Page 1 of 1). +reqid=1 GET http://localhost:/one [success - 200] +reqid=2 GET http://localhost:/two [success - 200] +reqid=3 GET http://localhost:/three [success - 200] +`; diff --git a/tests/tools/network.test.ts b/tests/tools/network.test.ts index fa6439dd..c5f666b9 100644 --- a/tests/tools/network.test.ts +++ b/tests/tools/network.test.ts @@ -10,9 +10,11 @@ import { getNetworkRequest, listNetworkRequests, } from '../../src/tools/network.js'; -import {withBrowser} from '../utils.js'; +import {serverHooks} from '../server.js'; +import {html, withBrowser, stabilizeResponseOutput} from '../utils.js'; describe('network', () => { + const server = serverHooks(); describe('network_list_requests', () => { it('list requests', async () => { await withBrowser(async (response, context) => { @@ -21,23 +23,72 @@ describe('network', () => { assert.strictEqual(response.networkRequestsPageIdx, undefined); }); }); + + it('list requests form current navigations only', async t => { + server.addHtmlRoute('/one', html`
First
`); + server.addHtmlRoute('/two', html`
Second
`); + server.addHtmlRoute('/three', html`
Third
`); + + await withBrowser(async (response, context) => { + await context.setUpNetworkCollectorForTesting(); + const page = context.getSelectedPage(); + await page.goto(server.getRoute('/one')); + await page.goto(server.getRoute('/two')); + await page.goto(server.getRoute('/three')); + await listNetworkRequests.handler( + { + params: {}, + }, + response, + context, + ); + const responseData = await response.handle('list_request', context); + t.assert.snapshot?.(stabilizeResponseOutput(responseData[0].text)); + }); + }); + + it('list requests from previous navigations', async t => { + server.addHtmlRoute('/one', html`
First
`); + server.addHtmlRoute('/two', html`
Second
`); + server.addHtmlRoute('/three', html`
Third
`); + + await withBrowser(async (response, context) => { + await context.setUpNetworkCollectorForTesting(); + const page = context.getSelectedPage(); + await page.goto(server.getRoute('/one')); + await page.goto(server.getRoute('/two')); + await page.goto(server.getRoute('/three')); + await listNetworkRequests.handler( + { + params: { + includePreviousNavigations: true, + }, + }, + response, + context, + ); + const responseData = await response.handle('list_request', context); + t.assert.snapshot?.(stabilizeResponseOutput(responseData[0].text)); + }); + }); }); describe('network_get_request', () => { it('attaches request', async () => { await withBrowser(async (response, context) => { - const page = await context.getSelectedPage(); + const page = context.getSelectedPage(); await page.goto('data:text/html,
Hello MCP
'); await getNetworkRequest.handler( {params: {reqid: 1}}, response, context, ); + assert.equal(response.attachedNetworkRequestId, 1); }); }); it('should not add the request list', async () => { await withBrowser(async (response, context) => { - const page = await context.getSelectedPage(); + const page = context.getSelectedPage(); await page.goto('data:text/html,
Hello MCP
'); await getNetworkRequest.handler( {params: {reqid: 1}}, @@ -47,5 +98,30 @@ describe('network', () => { assert(!response.includeNetworkRequests); }); }); + it('should get request from previous navigations', async t => { + server.addHtmlRoute('/one', html`
First
`); + server.addHtmlRoute('/two', html`
Second
`); + server.addHtmlRoute('/three', html`
Third
`); + + await withBrowser(async (response, context) => { + await context.setUpNetworkCollectorForTesting(); + const page = context.getSelectedPage(); + await page.goto(server.getRoute('/one')); + await page.goto(server.getRoute('/two')); + await page.goto(server.getRoute('/three')); + await getNetworkRequest.handler( + { + params: { + reqid: 1, + }, + }, + response, + context, + ); + const responseData = await response.handle('get_request', context); + + t.assert.snapshot?.(stabilizeResponseOutput(responseData[0].text)); + }); + }); }); }); diff --git a/tests/tools/snapshot.test.ts b/tests/tools/snapshot.test.ts index 31857ff5..718ea998 100644 --- a/tests/tools/snapshot.test.ts +++ b/tests/tools/snapshot.test.ts @@ -21,7 +21,7 @@ describe('snapshot', () => { describe('browser_wait_for', () => { it('should work', async () => { await withBrowser(async (response, context) => { - const page = await context.getSelectedPage(); + const page = context.getSelectedPage(); await page.setContent( html`
Hello
World
`, @@ -98,7 +98,7 @@ describe('snapshot', () => { it('should work with iframe content', async () => { await withBrowser(async (response, context) => { - const page = await context.getSelectedPage(); + const page = context.getSelectedPage(); await page.setContent( html`

Top level

diff --git a/tests/utils.ts b/tests/utils.ts index 7adaaf53..17e86ca2 100644 --- a/tests/utils.ts +++ b/tests/utils.ts @@ -130,3 +130,26 @@ export function html( `; } + +export function stabilizeResponseOutput(text: unknown) { + if (typeof text !== 'string') { + throw new Error('Input must be string'); + } + let output = text; + const dateRegEx = /.{3}, \d{2} .{3} \d{4} \d{2}:\d{2}:\d{2} [A-Z]{3}/g; + output = output.replaceAll(dateRegEx, ''); + + const localhostRegEx = /http:\/\/localhost:\d{5}\//g; + output = output.replaceAll(localhostRegEx, 'http://localhost:/'); + + const userAgentRegEx = /user-agent:.*\n/g; + output = output.replaceAll(userAgentRegEx, 'user-agent:\n'); + + const chUaRegEx = /sec-ch-ua:"Chromium";v="\d{3}"/g; + output = output.replaceAll(chUaRegEx, 'sec-ch-ua:"Chromium";v=""'); + + // sec-ch-ua-platform:"Linux" + const chUaPlatformRegEx = /sec-ch-ua-platform:"[a-zA-Z]*"/g; + output = output.replaceAll(chUaPlatformRegEx, 'sec-ch-ua-platform:""'); + return output; +}