feat: add request.body async getters

Author: hbenl

docs/src/api/class-request.md

@@ -23,6 +23,32 @@ request is  issued to a redirected url.
 
 An object with all the request HTTP headers associated with this request. The header names are lower-cased.
 
+## async method: Request.body
+* since: v1.58
+- returns: <[null]|[string]>
+
+The request body, if present.
+
+## async method: Request.bodyBuffer
+* since: v1.58
+- returns: <[null]|[Buffer]>
+
+The request body in a binary form. Returns null if the request has no body.
+
+## async method: Request.bodyJSON
+* since: v1.58
+* langs: js, python
+- returns: <[null]|[Serializable]>
+
+Returns the request body as a parsed JSON object. If the request `Content-Type` is `application/x-www-form-urlencoded`, this method returns a key/value object parsed from the form data. Otherwise, it parses the body as JSON.
+
+## async method: Request.bodyJSON
+* since: v1.58
+* langs: csharp
+- returns: <[null]|[JsonElement]>
+
+Returns the request body as a parsed JSON object. If the request `Content-Type` is `application/x-www-form-urlencoded`, this method returns a key/value object parsed from the form data. Otherwise, it parses the body as JSON.
+
 ## method: Request.failure
 * since: v1.8
 - returns: <[null]|[string]>
@@ -149,35 +175,33 @@ Request's method (GET, POST, etc.)
 
 ## method: Request.postData
 * since: v1.8
+* discouraged: Use [`method: Request.body`] instead.
 - returns: <[null]|[string]>
 
-Request's post body, if any.
+The request body, if present.
 
 ## method: Request.postDataBuffer
 * since: v1.8
+* discouraged: Use [`method: Request.bodyBuffer`] instead.
 - returns: <[null]|[Buffer]>
 
-Request's post body in a binary form, if any.
+The request body in a binary form. Returns null if the request has no body.
 
 ## method: Request.postDataJSON
 * since: v1.8
 * langs: js, python
+* discouraged: Use [`method: Request.bodyJSON`] instead.
 - returns: <[null]|[Serializable]>
 
-Returns parsed request's body for `form-urlencoded` and JSON as a fallback if any.
-
-When the response is `application/x-www-form-urlencoded` then a key/value object of the values will be returned.
-Otherwise it will be parsed as JSON.
+Returns the request body as a parsed JSON object. If the request `Content-Type` is `application/x-www-form-urlencoded`, this method returns a key/value object parsed from the form data. Otherwise, it parses the body as JSON.
 
 ## method: Request.postDataJSON
 * since: v1.12
 * langs: csharp
+* discouraged: Use [`method: Request.bodyJSON`] instead.
 - returns: <[null]|[JsonElement]>
 
-Returns parsed request's body for `form-urlencoded` and JSON as a fallback if any.
-
-When the response is `application/x-www-form-urlencoded` then a key/value object of the values will be returned.
-Otherwise it will be parsed as JSON.
+Returns the request body as a parsed JSON object. If the request `Content-Type` is `application/x-www-form-urlencoded`, this method returns a key/value object parsed from the form data. Otherwise, it parses the body as JSON.
 
 ## method: Request.redirectedFrom
 * since: v1.8

packages/playwright-client/types/types.d.ts

@@ -20616,6 +20616,23 @@ export interface Request {
    */
   allHeaders(): Promise<{ [key: string]: string; }>;
 
+  /**
+   * The request body, if present.
+   */
+  body(): Promise<null|string>;
+
+  /**
+   * The request body in a binary form. Returns null if the request has no body.
+   */
+  bodyBuffer(): Promise<null|Buffer>;
+
+  /**
+   * Returns the request body as a parsed JSON object. If the request `Content-Type` is
+   * `application/x-www-form-urlencoded`, this method returns a key/value object parsed from the form data. Otherwise,
+   * it parses the body as JSON.
+   */
+  bodyJSON(): Promise<null|Serializable>;
+
   /**
    * The method returns `null` unless this request has failed, as reported by `requestfailed` event.
    *
@@ -20713,20 +20730,25 @@ export interface Request {
   method(): string;
 
   /**
-   * Request's post body, if any.
+   * **NOTE** Use [request.body()](https://playwright.dev/docs/api/class-request#request-body) instead.
+   *
+   * The request body, if present.
    */
   postData(): null|string;
 
   /**
-   * Request's post body in a binary form, if any.
+   * **NOTE** Use [request.bodyBuffer()](https://playwright.dev/docs/api/class-request#request-body-buffer) instead.
+   *
+   * The request body in a binary form. Returns null if the request has no body.
    */
   postDataBuffer(): null|Buffer;
 
   /**
-   * Returns parsed request's body for `form-urlencoded` and JSON as a fallback if any.
+   * **NOTE** Use [request.bodyJSON()](https://playwright.dev/docs/api/class-request#request-body-json) instead.
    *
-   * When the response is `application/x-www-form-urlencoded` then a key/value object of the values will be returned.
-   * Otherwise it will be parsed as JSON.
+   * Returns the request body as a parsed JSON object. If the request `Content-Type` is
+   * `application/x-www-form-urlencoded`, this method returns a key/value object parsed from the form data. Otherwise,
+   * it parses the body as JSON.
    */
   postDataJSON(): null|Serializable;
 

packages/playwright-core/src/client/network.ts

@@ -134,6 +134,18 @@ export class Request extends ChannelOwner<channels.RequestChannel> implements ap
     return this._fallbackOverrides.method || this._initializer.method;
   }
 
+  async body(): Promise<string | null> {
+    return (this._fallbackOverrides.postDataBuffer || (await this._channel.body()).body)?.toString('utf-8') || null;
+  }
+
+  async bodyBuffer(): Promise<Buffer | null> {
+    return this._fallbackOverrides.postDataBuffer || (await this._channel.body()).body || null;
+  }
+
+  async bodyJSON(): Promise<Object | null> {
+    return this._postDataJSON(await this.body());
+  }
+
   postData(): string | null {
     return (this._fallbackOverrides.postDataBuffer || this._initializer.postData)?.toString('utf-8') || null;
   }
@@ -144,6 +156,10 @@ export class Request extends ChannelOwner<channels.RequestChannel> implements ap
 
   postDataJSON(): Object | null {
     const postData = this.postData();
+    return this._postDataJSON(postData);
+  }
+
+  private _postDataJSON(postData: string | null): Object | null {
     if (!postData)
       return null;
 

packages/playwright-core/src/protocol/validator.ts

@@ -2249,6 +2249,10 @@ scheme.RequestInitializer = tObject({
   hasResponse: tBoolean,
 });
 scheme.RequestResponseEvent = tOptional(tObject({}));
+scheme.RequestBodyParams = tOptional(tObject({}));
+scheme.RequestBodyResult = tObject({
+  body: tOptional(tBinary),
+});
 scheme.RequestResponseParams = tOptional(tObject({}));
 scheme.RequestResponseResult = tObject({
   response: tOptional(tChannel(['Response'])),

packages/playwright-core/src/server/bidi/bidiBrowser.ts

@@ -75,7 +75,7 @@ export class BidiBrowser extends Browser {
     });
 
     await browser._browserSession.send('network.addDataCollector', {
-      dataTypes: [bidi.Network.DataType.Response],
+      dataTypes: [bidi.Network.DataType.Request, bidi.Network.DataType.Response],
       maxEncodedDataSize: 20_000_000, // same default as in CDP: https://source.chromium.org/chromium/chromium/src/+/main:third_party/blink/renderer/core/inspector/inspector_network_agent.cc;l=134;drc=4128411589187a396829a827f59a655bed876aa7
     });
 

packages/playwright-core/src/server/bidi/bidiNetworkManager.ts

@@ -79,7 +79,8 @@ export class BidiNetworkManager {
         route = new BidiRouteImpl(this._session, param.request.request);
       }
     }
-    const request = new BidiRequest(frame, redirectedFrom, param, route);
+    const getRequestBody = param.request.bodySize ? () => getNetworkData(this._session, param.request, bidi.Network.DataType.Request) : null;
+    const request = new BidiRequest(frame, redirectedFrom, param, getRequestBody, route);
     this._requests.set(request._id, request);
     this._page.frameManager.requestStarted(request.request, route);
   }
@@ -88,11 +89,7 @@ export class BidiNetworkManager {
     const request = this._requests.get(params.request.request);
     if (!request)
       return;
-    const getResponseBody = async () => {
-      const { bytes } = await this._session.send('network.getData', { request: params.request.request, dataType: bidi.Network.DataType.Response });
-      const encoding = bytes.type === 'base64' ? 'base64' : 'utf8';
-      return Buffer.from(bytes.value, encoding);
-    };
+    const getResponseBody = () => getNetworkData(this._session, params.request, bidi.Network.DataType.Response);
     const timings = params.request.timings;
     const startTime = timings.requestTime;
     function relativeToStart(time: number): number {
@@ -236,14 +233,12 @@ class BidiRequest {
   // store the first and only Route in the chain (if any).
   _originalRequestRoute: BidiRouteImpl | undefined;
 
-  constructor(frame: frames.Frame, redirectedFrom: BidiRequest | null, payload: bidi.Network.BeforeRequestSentParameters, route: BidiRouteImpl | undefined) {
+  constructor(frame: frames.Frame, redirectedFrom: BidiRequest | null, payload: bidi.Network.BeforeRequestSentParameters, getBody: (() => Promise<Buffer>) | null, route: BidiRouteImpl | undefined) {
     this._id = payload.request.request;
     if (redirectedFrom)
       redirectedFrom._redirectedTo = this;
-    // TODO: missing in the spec?
-    const postDataBuffer = null;
     this.request = new network.Request(frame._page.browserContext, frame, null, redirectedFrom ? redirectedFrom.request : null, payload.navigation ?? undefined, payload.request.url,
-        resourceTypeFromBidi(payload.request.destination, payload.request.initiatorType, payload.initiator?.type), payload.request.method, postDataBuffer, fromBidiHeaders(payload.request.headers));
+        resourceTypeFromBidi(payload.request.destination, payload.request.initiatorType, payload.initiator?.type), payload.request.method, null, fromBidiHeaders(payload.request.headers), getBody);
     // "raw" headers are the same as "provisional" headers in Bidi.
     this.request.setRawRequestHeaders(null);
     this.request._setBodySize(payload.request.bodySize || 0);
@@ -390,3 +385,9 @@ function resourceTypeFromBidi(requestDestination: string, requestInitiatorType:
     default: return 'other';
   }
 }
+
+async function getNetworkData(session: BidiSession, request: bidi.Network.RequestData, dataType: bidi.Network.DataType) {
+  const { bytes } = await session.send('network.getData', { request: request.request, dataType });
+  const encoding = bytes.type === 'base64' ? 'base64' : 'utf8';
+  return Buffer.from(bytes.value, encoding);
+}

packages/playwright-core/src/server/chromium/crNetworkManager.ts

@@ -598,14 +598,23 @@ class InterceptableRequest {
       headers,
       method,
       url,
+      hasPostData = false,
       postDataEntries = null,
     } = requestPausedEvent ? requestPausedEvent.request : requestWillBeSentEvent.request;
     let postDataBuffer = null;
+    let getPostData: (() => Promise<Buffer>) | null = null;
     const entries = postDataEntries?.filter(entry => entry.bytes);
-    if (entries && entries.length)
+    if (entries && entries.length) {
       postDataBuffer = Buffer.concat(entries.map(entry => Buffer.from(entry.bytes!, 'base64')));
+    } else if (hasPostData) {
+      getPostData = async () => {
+        const requestId = requestPausedEvent ? requestPausedEvent.requestId : requestWillBeSentEvent.requestId;
+        const { postData } = await session.send('Network.getRequestPostData', { requestId });
+        return Buffer.from(postData);
+      };
+    }
 
-    this.request = new network.Request(context, frame, serviceWorker, redirectedFrom?.request || null, documentId, url, toResourceType(requestWillBeSentEvent.type || 'Other'), method, postDataBuffer,  headersOverride || headersObjectToArray(headers));
+    this.request = new network.Request(context, frame, serviceWorker, redirectedFrom?.request || null, documentId, url, toResourceType(requestWillBeSentEvent.type || 'Other'), method, postDataBuffer,  headersOverride || headersObjectToArray(headers), getPostData);
   }
 }
 

packages/playwright-core/src/server/dispatchers/networkDispatchers.ts

@@ -66,6 +66,11 @@ export class RequestDispatcher extends Dispatcher<Request, channels.RequestChann
     this.addObjectListener(Request.Events.Response, () => this._dispatchEvent('response', {}));
   }
 
+  async body(params: channels.RequestBodyParams, progress: Progress): Promise<channels.RequestBodyResult> {
+    const body = await this._object.body();
+    return { body: body === null ? undefined : body };
+  }
+
   async rawRequestHeaders(params: channels.RequestRawRequestHeadersParams, progress: Progress): Promise<channels.RequestRawRequestHeadersResult> {
     return { headers: await progress.race(this._object.rawRequestHeaders()) };
   }

packages/playwright-core/src/server/network.ts

@@ -179,6 +179,7 @@ export class Request extends SdkObject {
   private _resourceType: ResourceType;
   private _method: string;
   private _postData: Buffer | null;
+  private _getPostData: (() => Promise<Buffer>) | null;
   readonly _headers: HeadersArray;
   readonly _frame: frames.Frame | null = null;
   readonly _serviceWorker: pages.Worker | null = null;
@@ -195,7 +196,7 @@ export class Request extends SdkObject {
   };
 
   constructor(context: contexts.BrowserContext, frame: frames.Frame | null, serviceWorker: pages.Worker | null, redirectedFrom: Request | null, documentId: string | undefined,
-    url: string, resourceType: ResourceType, method: string, postData: Buffer | null, headers: HeadersArray) {
+    url: string, resourceType: ResourceType, method: string, postData: Buffer | null, headers: HeadersArray, getPostData: (() => Promise<Buffer>) | null = null) {
     super(frame || context, 'request');
     assert(!url.startsWith('data:'), 'Data urls should not fire requests');
     this._context = context;
@@ -209,6 +210,7 @@ export class Request extends SdkObject {
     this._resourceType = resourceType;
     this._method = method;
     this._postData = postData;
+    this._getPostData = getPostData;
     this._headers = headers;
     this._isFavicon = url.endsWith('/favicon.ico') || !!redirectedFrom?._isFavicon;
   }
@@ -243,6 +245,14 @@ export class Request extends SdkObject {
     return this._overrides?.postData || this._postData;
   }
 
+  async body(): Promise<Buffer | null> {
+    if (this._overrides?.postData)
+      return this._overrides.postData;
+    if (this._getPostData)
+      return await this._getPostData();
+    return this._postData;
+  }
+
   headers(): HeadersArray {
     return this._overrides?.headers || this._headers;
   }

packages/playwright-core/src/utils/isomorphic/protocolMetainfo.ts

@@ -239,6 +239,7 @@ export const methodMetainfo = new Map<string, { internal?: boolean, title?: stri
   ['ElementHandle.uncheck', { title: 'Uncheck', slowMo: true, snapshot: true, pausesBeforeInput: true, }],
   ['ElementHandle.waitForElementState', { title: 'Wait for state', snapshot: true, pausesBeforeAction: true, }],
   ['ElementHandle.waitForSelector', { title: 'Wait for selector', snapshot: true, }],
+  ['Request.body', { title: 'Get request body', group: 'getter', }],
   ['Request.response', { internal: true, }],
   ['Request.rawRequestHeaders', { internal: true, }],
   ['Route.redirectNavigationRequest', { internal: true, }],