feat: expose multipart/form-data parsing options (#13766)

When sending REST API requests with multipart/form-data, e.g. PATCH or
POST within the admin panel, a request body larger than 1MB throws the
following error:

```
Unterminated string in JSON at position...
```

This is because there are sensible defaults imposed by the HTML form
data parser (currently using
[busboy](https://github.com/fastify/busboy)). If your documents exceed
this limit, you may run into this error when editing them within the
admin panel.

To support large documents over 1MB, use the new `bodyParser` property
on the root config:

```ts
import { buildConfig } from 'payload'

const config = buildConfig({
  // ...
  bodyParser: {
    limits: {
      fieldSize: 2 * 1024 * 1024, // This will allow requests containing up to 2MB of multipart/form-data
    }
  }
}
```

---
- To see the specific tasks where the Asana app for GitHub is being
used, see below:
  - https://app.asana.com/0/0/1211317005907885

Author: jacobsfletch

packages/next/src/routes/rest/index.ts

@@ -14,7 +14,7 @@ const handlerBuilder =
   ): Promise<Response> => {
     const awaitedConfig = await config
 
-    // Add this endpoint only when using Next.js, still can be overriden.
+    // Add this endpoint only when using Next.js, still can be overridden.
     if (
       initedOGEndpoint === false &&
       !awaitedConfig.endpoints.some(

packages/payload/src/config/types.ts

@@ -275,6 +275,7 @@ export type InitOptions = {
   disableOnInit?: boolean
 
   importMap?: ImportMap
+
   /**
    * A function that is called immediately following startup that receives the Payload instance as it's only argument.
    */
@@ -980,6 +981,7 @@ export type Config = {
     /** The slug of a Collection that you want to be used to log in to the Admin dashboard. */
     user?: string
   }
+
   /**
    * Configure authentication-related Payload-wide settings.
    */
@@ -993,6 +995,15 @@ export type Config = {
   /** Custom Payload bin scripts can be injected via the config. */
   bin?: BinScriptConfig[]
   blocks?: Block[]
+  /**
+   * Pass additional options to the parser used to process `multipart/form-data` requests.
+   * For example, a PATCH request containing HTML form data.
+   * For example, you may want to increase the `limits` imposed by the parser.
+   * Currently using @link {https://www.npmjs.com/package/busboy|busboy} under the hood.
+   *
+   * @experimental This property is experimental and may change in future releases. Use at your own discretion.
+   */
+  bodyParser?: Partial<BusboyConfig>
   /**
    * Manage the datamodel of your application
    *
@@ -1024,10 +1035,8 @@ export type Config = {
   cors?: '*' | CORSConfig | string[]
   /** A whitelist array of URLs to allow Payload cookies to be accepted from as a form of CSRF protection. */
   csrf?: string[]
-
   /** Extension point to add your custom data. Server only. */
   custom?: Record<string, any>
-
   /** Pass in a database adapter for use on this project. */
   db: DatabaseAdapterResult
   /** Enable to expose more detailed error information. */

packages/payload/src/uploads/fetchAPI-multipart/index.ts

@@ -7,7 +7,7 @@ import { isEligibleRequest } from './isEligibleRequest.js'
 import { processMultipart } from './processMultipart.js'
 import { debugLog } from './utilities.js'
 
-const DEFAULT_OPTIONS: FetchAPIFileUploadOptions = {
+const DEFAULT_UPLOAD_OPTIONS: FetchAPIFileUploadOptions = {
   abortOnLimit: false,
   createParentPath: false,
   debug: false,
@@ -53,16 +53,22 @@ type FetchAPIFileUpload = (args: {
   options?: FetchAPIFileUploadOptions
   request: Request
 }) => Promise<FetchAPIFileUploadResponse>
-export const fetchAPIFileUpload: FetchAPIFileUpload = async ({ options, request }) => {
-  const uploadOptions: FetchAPIFileUploadOptions = { ...DEFAULT_OPTIONS, ...options }
+
+export const processMultipartFormdata: FetchAPIFileUpload = async ({
+  options: incomingOptions,
+  request,
+}) => {
+  const options: FetchAPIFileUploadOptions = { ...DEFAULT_UPLOAD_OPTIONS, ...incomingOptions }
+
   if (!isEligibleRequest(request)) {
-    debugLog(uploadOptions, 'Request is not eligible for file upload!')
+    debugLog(options, 'Request is not eligible for file upload!')
+
     return {
       error: new APIError('Request is not eligible for file upload', 500),
       fields: undefined!,
       files: undefined!,
     }
   } else {
-    return processMultipart({ options: uploadOptions, request })
+    return processMultipart({ options, request })
   }
 }

packages/payload/src/utilities/addDataAndFileToRequest.ts

@@ -1,7 +1,7 @@
 import type { PayloadRequest } from '../types/index.js'
 
 import { APIError } from '../errors/APIError.js'
-import { fetchAPIFileUpload } from '../uploads/fetchAPI-multipart/index.js'
+import { processMultipartFormdata } from '../uploads/fetchAPI-multipart/index.js'
 
 type AddDataAndFileToRequest = (req: PayloadRequest) => Promise<void>
 
@@ -24,12 +24,15 @@ export const addDataAndFileToRequest: AddDataAndFileToRequest = async (req) => {
         req.payload.logger.error(error)
       } finally {
         req.data = data
-        // @ts-expect-error
+        // @ts-expect-error attach json method to request
         req.json = () => Promise.resolve(data)
       }
     } else if (bodyByteSize && contentType?.includes('multipart/')) {
-      const { error, fields, files } = await fetchAPIFileUpload({
-        options: payload.config.upload,
+      const { error, fields, files } = await processMultipartFormdata({
+        options: {
+          ...(payload.config.bodyParser || {}),
+          ...(payload.config.upload || {}),
+        },
         request: req as Request,
       })
 

pnpm-lock.yaml

@@ -0,0 +1,19 @@
+import type { CollectionConfig } from 'payload'
+
+export const largeDocumentsCollectionSlug = 'large-documents'
+
+export const LargeDocuments: CollectionConfig = {
+  slug: largeDocumentsCollectionSlug,
+  fields: [
+    {
+      name: 'array',
+      type: 'array',
+      fields: [
+        {
+          name: 'text',
+          type: 'text',
+        },
+      ],
+    },
+  ],
+}

test/collections-rest/collections/LargeDocuments.ts

@@ -6,6 +6,7 @@ import { APIError, type CollectionConfig, type Endpoint } from 'payload'
 
 import { buildConfigWithDefaults } from '../buildConfigWithDefaults.js'
 import { devUser } from '../credentials.js'
+import { LargeDocuments } from './collections/LargeDocuments.js'
 
 export interface Relation {
   id: string
@@ -280,7 +281,13 @@ export default buildConfigWithDefaults({
       ],
       disableBulkEdit: true,
     },
+    LargeDocuments,
   ],
+  bodyParser: {
+    limits: {
+      fieldSize: 2 * 1024 * 1024, // 2MB
+    },
+  },
   endpoints: [
     {
       handler: async ({ payload }) => {

test/collections-rest/config.ts

@@ -1,6 +1,7 @@
 import type { Payload, SanitizedCollectionConfig } from 'payload'
 
 import { randomBytes, randomUUID } from 'crypto'
+import { serialize } from 'object-to-formdata'
 import path from 'path'
 import { APIError, NotFound } from 'payload'
 import { fileURLToPath } from 'url'
@@ -9,7 +10,9 @@ import type { NextRESTClient } from '../helpers/NextRESTClient.js'
 import type { Relation } from './config.js'
 import type { Post } from './payload-types.js'
 
+import { getFormDataSize } from '../helpers/getFormDataSize.js'
 import { initPayloadInt } from '../helpers/initPayloadInt.js'
+import { largeDocumentsCollectionSlug } from './collections/LargeDocuments.js'
 import {
   customIdNumberSlug,
   customIdSlug,
@@ -119,6 +122,47 @@ describe('collections-rest', () => {
       expect(doc.description).toEqual(description) // Check was not modified
     })
 
+    it('can handle REST API requests with over 1mb of multipart/form-data', async () => {
+      const doc = await payload.create({
+        collection: largeDocumentsCollectionSlug,
+        data: {},
+      })
+
+      const arrayData = new Array(500).fill({ text: randomUUID().repeat(100) })
+
+      // Now use the REST API and attempt to PATCH the document with a payload over 1mb
+      const dataToSerialize: Record<string, unknown> = {
+        _payload: JSON.stringify({
+          title: 'Hello, world!',
+          // fill with long, random string of text to exceed 1mb
+          array: arrayData,
+        }),
+      }
+
+      const formData: FormData = serialize(dataToSerialize, {
+        indices: true,
+        nullsAsUndefineds: false,
+      })
+
+      // Ensure the form data we are about to send is greater than the default limit (1mb)
+      // But less than the increased limit that we've set in the root config (2mb)
+      const docSize = getFormDataSize(formData)
+      expect(docSize).toBeGreaterThan(1 * 1024 * 1024)
+      expect(docSize).toBeLessThan(2 * 1024 * 1024)
+
+      // This request should not fail with error: "Unterminated string in JSON at position..."
+      // This is because we set `bodyParser.limits.fieldSize` to 2mb in the root config
+      const res = await restClient
+        .PATCH(`/${largeDocumentsCollectionSlug}/${doc.id}?limit=1`, {
+          body: formData,
+        })
+        .then((res) => res.json())
+
+      expect(res).not.toHaveProperty('errors')
+      expect(res.doc.id).toEqual(doc.id)
+      expect(res.doc.array[0].text).toEqual(arrayData[0].text)
+    })
+
     describe('Bulk operations', () => {
       it('should bulk update', async () => {
         for (let i = 0; i < 11; i++) {

test/collections-rest/int.spec.ts

@@ -76,6 +76,7 @@ export interface Config {
     'error-on-hooks': ErrorOnHook;
     endpoints: Endpoint;
     'disabled-bulk-edit-docs': DisabledBulkEditDoc;
+    'large-documents': LargeDocument;
     users: User;
     'payload-locked-documents': PayloadLockedDocument;
     'payload-preferences': PayloadPreference;
@@ -92,6 +93,7 @@ export interface Config {
     'error-on-hooks': ErrorOnHooksSelect<false> | ErrorOnHooksSelect<true>;
     endpoints: EndpointsSelect<false> | EndpointsSelect<true>;
     'disabled-bulk-edit-docs': DisabledBulkEditDocsSelect<false> | DisabledBulkEditDocsSelect<true>;
+    'large-documents': LargeDocumentsSelect<false> | LargeDocumentsSelect<true>;
     users: UsersSelect<false> | UsersSelect<true>;
     'payload-locked-documents': PayloadLockedDocumentsSelect<false> | PayloadLockedDocumentsSelect<true>;
     'payload-preferences': PayloadPreferencesSelect<false> | PayloadPreferencesSelect<true>;
@@ -259,6 +261,21 @@ export interface DisabledBulkEditDoc {
   updatedAt: string;
   createdAt: string;
 }
+/**
+ * This interface was referenced by `Config`'s JSON-Schema
+ * via the `definition` "large-documents".
+ */
+export interface LargeDocument {
+  id: string;
+  array?:
+    | {
+        text?: string | null;
+        id?: string | null;
+      }[]
+    | null;
+  updatedAt: string;
+  createdAt: string;
+}
 /**
  * This interface was referenced by `Config`'s JSON-Schema
  * via the `definition` "users".
@@ -274,6 +291,13 @@ export interface User {
   hash?: string | null;
   loginAttempts?: number | null;
   lockUntil?: string | null;
+  sessions?:
+    | {
+        id: string;
+        createdAt?: string | null;
+        expiresAt: string;
+      }[]
+    | null;
   password?: string | null;
 }
 /**
@@ -319,6 +343,10 @@ export interface PayloadLockedDocument {
         relationTo: 'disabled-bulk-edit-docs';
         value: string | DisabledBulkEditDoc;
       } | null)
+    | ({
+        relationTo: 'large-documents';
+        value: string | LargeDocument;
+      } | null)
     | ({
         relationTo: 'users';
         value: string | User;
@@ -471,6 +499,20 @@ export interface DisabledBulkEditDocsSelect<T extends boolean = true> {
   updatedAt?: T;
   createdAt?: T;
 }
+/**
+ * This interface was referenced by `Config`'s JSON-Schema
+ * via the `definition` "large-documents_select".
+ */
+export interface LargeDocumentsSelect<T extends boolean = true> {
+  array?:
+    | T
+    | {
+        text?: T;
+        id?: T;
+      };
+  updatedAt?: T;
+  createdAt?: T;
+}
 /**
  * This interface was referenced by `Config`'s JSON-Schema
  * via the `definition` "users_select".
@@ -485,6 +527,13 @@ export interface UsersSelect<T extends boolean = true> {
   hash?: T;
   loginAttempts?: T;
   lockUntil?: T;
+  sessions?:
+    | T
+    | {
+        id?: T;
+        createdAt?: T;
+        expiresAt?: T;
+      };
 }
 /**
  * This interface was referenced by `Config`'s JSON-Schema

test/collections-rest/payload-types.ts

@@ -12,6 +12,7 @@ import {
 import * as qs from 'qs-esm'
 
 import { devUser } from '../credentials.js'
+import { getFormDataSize } from './getFormDataSize.js'
 
 type ValidPath = `/${string}`
 type RequestOptions = {
@@ -94,17 +95,26 @@ export class NextRESTClient {
   }
 
   private buildHeaders(options: FileArg & RequestInit & RequestOptions): Headers {
-    const defaultHeaders = {
-      'Content-Type': 'application/json',
+    // Only set `Content-Type` to `application/json` if body is not `FormData`
+    const isFormData =
+      options &&
+      typeof options.body !== 'undefined' &&
+      typeof FormData !== 'undefined' &&
+      options.body instanceof FormData
+
+    const headers = new Headers(options.headers || {})
+
+    if (options?.file) {
+      headers.set('Content-Length', options.file.size.toString())
+    }
+
+    if (isFormData) {
+      headers.set('Content-Length', getFormDataSize(options.body as FormData).toString())
+    }
+
+    if (!isFormData && !headers.has('Content-Type')) {
+      headers.set('Content-Type', 'application/json')
     }
-    const headers = new Headers({
-      ...(options?.file
-        ? {
-            'Content-Length': options.file.size.toString(),
-          }
-        : defaultHeaders),
-      ...(options?.headers || {}),
-    })
 
     if (options.auth !== false && this.token) {
       headers.set('Authorization', `JWT ${this.token}`)
@@ -213,6 +223,7 @@ export class NextRESTClient {
       headers: this.buildHeaders(options),
       method: 'PATCH',
     })
+
     return this._PATCH(request, { params: Promise.resolve({ slug }) })
   }