-
Notifications
You must be signed in to change notification settings - Fork 1k
feat: Add pagination and request type filtering to list_network_requests #107
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
5be157e
bbcc6db
08b5661
5922f00
11deb4d
fb2b8c9
a8020cc
bc30736
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -6,6 +6,7 @@ | |
|
|
||
| import z from 'zod'; | ||
| import {Dialog, ElementHandle, Page} from 'puppeteer-core'; | ||
| import type {FilterableResourceType} from '../utils/networkUtils.js'; | ||
| import {ToolCategories} from './categories.js'; | ||
| import {TraceResult} from '../trace-processing/parse.js'; | ||
|
|
||
|
|
@@ -39,10 +40,19 @@ export type ImageContentData = { | |
| mimeType: string; | ||
| }; | ||
|
|
||
| export type NetworkRequestsOptions = { | ||
| pageSize?: number; | ||
| pageToken?: string | null; | ||
| requestType?: FilterableResourceType | FilterableResourceType[] | null; | ||
|
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. It appears that |
||
| }; | ||
|
|
||
| export interface Response { | ||
| appendResponseLine(value: string): void; | ||
| setIncludePages(value: boolean): void; | ||
| setIncludeNetworkRequests(value: boolean): void; | ||
| setIncludeNetworkRequests( | ||
| value: boolean, | ||
| options?: NetworkRequestsOptions, | ||
| ): void; | ||
| setIncludeConsoleData(value: boolean): void; | ||
| setIncludeSnapshot(value: boolean): void; | ||
| attachImage(value: ImageContentData): void; | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,183 @@ | ||
| /** | ||
| * @license | ||
| * Copyright 2025 Google LLC | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| */ | ||
| import {type HTTPRequest, type ResourceType} from 'puppeteer-core'; | ||
|
|
||
| export const FILTERABLE_RESOURCE_TYPES = [ | ||
| 'document', | ||
| 'stylesheet', | ||
| 'image', | ||
| 'media', | ||
| 'font', | ||
| 'script', | ||
| 'xhr', | ||
| 'fetch', | ||
| 'prefetch', | ||
| 'websocket', | ||
| 'preflight', | ||
| 'other', | ||
| ] as const satisfies readonly ResourceType[]; | ||
|
|
||
| export type FilterableResourceType = (typeof FILTERABLE_RESOURCE_TYPES)[number]; | ||
|
|
||
| export type NetworkRequestsListingOptions = { | ||
| pageSize?: number; | ||
| pageToken?: string; | ||
| requestType?: FilterableResourceType | FilterableResourceType[]; | ||
| }; | ||
|
|
||
| export type NetworkRequestsListingResult = { | ||
| requests: readonly HTTPRequest[]; | ||
| nextPageToken?: string; | ||
| previousPageToken?: string; | ||
| startIndex: number; | ||
| endIndex: number; | ||
| invalidToken: boolean; | ||
| total: number; | ||
| appliedRequestType?: FilterableResourceType | FilterableResourceType[]; | ||
| }; | ||
|
|
||
| const DEFAULT_PAGE_SIZE = 20; | ||
| const FILTERABLE_RESOURCE_TYPES_SET = new Set<FilterableResourceType>( | ||
| FILTERABLE_RESOURCE_TYPES, | ||
| ); | ||
|
|
||
| export function isFilterableResourceType( | ||
| value: ResourceType | string, | ||
| ): value is FilterableResourceType { | ||
| return FILTERABLE_RESOURCE_TYPES_SET.has(value as FilterableResourceType); | ||
| } | ||
|
|
||
| export function sanitizeRequestTypeFilter( | ||
| requestType?: string | string[] | null, | ||
| ): FilterableResourceType | FilterableResourceType[] | undefined { | ||
| if (requestType === undefined || requestType === null) { | ||
| return undefined; | ||
| } | ||
|
|
||
| const values = Array.isArray(requestType) ? requestType : [requestType]; | ||
| const sanitized = values.filter(isFilterableResourceType); | ||
|
|
||
| if (!sanitized.length) { | ||
| return undefined; | ||
| } | ||
|
|
||
| return Array.isArray(requestType) ? sanitized : sanitized[0]; | ||
| } | ||
|
|
||
| export function filterNetworkRequests( | ||
| requests: readonly HTTPRequest[], | ||
| requestType?: FilterableResourceType | FilterableResourceType[], | ||
| ): readonly HTTPRequest[] { | ||
| if (!requestType) { | ||
| return requests; | ||
| } | ||
|
|
||
| const normalizedTypes = new Set<FilterableResourceType>( | ||
| Array.isArray(requestType) ? requestType : [requestType], | ||
| ); | ||
|
|
||
| if (!normalizedTypes.size) { | ||
| return requests; | ||
| } | ||
|
|
||
| return requests.filter(request => { | ||
| const type = request.resourceType(); | ||
| if (!isFilterableResourceType(type)) { | ||
| return false; | ||
| } | ||
| return normalizedTypes.has(type); | ||
| }); | ||
| } | ||
|
|
||
| export function paginateNetworkRequests( | ||
|
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. let's make paginate function generic to support any list of objects. |
||
| requests: readonly HTTPRequest[], | ||
| options?: NetworkRequestsListingOptions, | ||
| ): NetworkRequestsListingResult { | ||
| const sanitizedOptions = options ?? {}; | ||
| const filteredRequests = filterNetworkRequests( | ||
|
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. let's filter before the paginate function. |
||
| requests, | ||
| sanitizedOptions.requestType, | ||
| ); | ||
| const total = filteredRequests.length; | ||
|
|
||
| const hasPaginationOptions = hasPagination(sanitizedOptions); | ||
|
|
||
| if (!hasPaginationOptions) { | ||
| return { | ||
| requests: filteredRequests, | ||
| nextPageToken: undefined, | ||
| previousPageToken: undefined, | ||
| startIndex: 0, | ||
| endIndex: total, | ||
| invalidToken: false, | ||
| total, | ||
| appliedRequestType: sanitizedOptions.requestType, | ||
| }; | ||
| } | ||
|
|
||
| const pageSize = validatePageSize(sanitizedOptions.pageSize, total); | ||
| const {startIndex, invalidToken} = resolveStartIndex( | ||
| sanitizedOptions.pageToken, | ||
| total, | ||
| ); | ||
|
|
||
| const pageRequests = filteredRequests.slice( | ||
| startIndex, | ||
| startIndex + pageSize, | ||
| ); | ||
| const endIndex = startIndex + pageRequests.length; | ||
|
|
||
| const nextPageToken = endIndex < total ? String(endIndex) : undefined; | ||
| const previousPageToken = | ||
| startIndex > 0 ? String(Math.max(startIndex - pageSize, 0)) : undefined; | ||
|
|
||
| return { | ||
| requests: pageRequests, | ||
| nextPageToken, | ||
| previousPageToken, | ||
| startIndex, | ||
| endIndex, | ||
| invalidToken, | ||
| total, | ||
| appliedRequestType: sanitizedOptions.requestType, | ||
| }; | ||
| } | ||
|
|
||
| function hasPagination(options: NetworkRequestsListingOptions): boolean { | ||
| return ( | ||
| options.pageSize !== undefined || | ||
| (options.pageToken !== undefined && options.pageToken !== null) | ||
| ); | ||
| } | ||
|
|
||
| function validatePageSize(pageSize: number | undefined, total: number): number { | ||
|
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Would not the JSON schema already validate the page size? If not, can we update the JSON schema. |
||
| if (pageSize === undefined) { | ||
| return total || DEFAULT_PAGE_SIZE; | ||
| } | ||
| if (!Number.isInteger(pageSize) || pageSize <= 0) { | ||
| return DEFAULT_PAGE_SIZE; | ||
| } | ||
| return Math.min(pageSize, Math.max(total, 1)); | ||
| } | ||
|
|
||
| function resolveStartIndex( | ||
| pageToken: string | undefined, | ||
| total: number, | ||
| ): { | ||
| startIndex: number; | ||
| invalidToken: boolean; | ||
| } { | ||
| if (pageToken === undefined || pageToken === null) { | ||
| return {startIndex: 0, invalidToken: false}; | ||
| } | ||
|
|
||
| const parsed = Number.parseInt(pageToken, 10); | ||
| if (Number.isNaN(parsed) || parsed < 0 || parsed >= total) { | ||
| return {startIndex: 0, invalidToken: total > 0}; | ||
| } | ||
|
|
||
| return {startIndex: parsed, invalidToken: false}; | ||
| } | ||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
We do not need to sanitize the values as we can rely on the types and JSON schema.
Could we replace this with the following?