From e363d506c60dc0ea5f64efa86c901865766e1c96 Mon Sep 17 00:00:00 2001 From: tak-amboss Date: Wed, 29 Oct 2025 09:02:41 +0100 Subject: [PATCH 1/3] feat: add bulkOperations.singleTransaction config option Add configuration to process bulk operations sequentially in separate transactions instead of all at once in a single transaction. This helps avoid transaction limitations with large datasets (e.g., DocumentDB cursor limits of 100 documents). - Add bulkOperations.singleTransaction config (default: false) - Supports global config default and per-collection override - Update and delete operations process documents sequentially when enabled - Maintains parallel processing for optimal performance when disabled --- .../src/collections/config/defaults.ts | 11 +++++++- .../src/collections/config/sanitize.ts | 2 +- .../payload/src/collections/config/types.ts | 13 ++++++++++ .../src/collections/operations/delete.ts | 24 +++++++++++++++++- .../src/collections/operations/update.ts | 25 ++++++++++++++++++- packages/payload/src/config/types.ts | 14 +++++++++++ 6 files changed, 85 insertions(+), 4 deletions(-) diff --git a/packages/payload/src/collections/config/defaults.ts b/packages/payload/src/collections/config/defaults.ts index 9e329073234..83e74ef752a 100644 --- a/packages/payload/src/collections/config/defaults.ts +++ b/packages/payload/src/collections/config/defaults.ts @@ -1,4 +1,5 @@ import type { IncomingAuthType, LoginWithUsernameOptions } from '../../auth/types.js' +import type { Config } from '../../config/types.js' import type { CollectionConfig } from './types.js' import { defaultAccess } from '../../auth/defaultAccess.js' @@ -54,7 +55,10 @@ export const defaults: Partial = { versions: false, } -export const addDefaultsToCollectionConfig = (collection: CollectionConfig): CollectionConfig => { +export const addDefaultsToCollectionConfig = ( + collection: CollectionConfig, + config?: Config, +): CollectionConfig => { collection.access = { create: defaultAccess, delete: defaultAccess, @@ -105,6 +109,11 @@ export const addDefaultsToCollectionConfig = (collection: CollectionConfig): Col ...(collection.hooks || {}), } + collection.bulkOperations = { + singleTransaction: config?.bulkOperations?.singleTransaction ?? false, + ...(collection.bulkOperations || {}), + } + collection.timestamps = collection.timestamps ?? true collection.upload = collection.upload ?? false collection.versions = collection.versions ?? false diff --git a/packages/payload/src/collections/config/sanitize.ts b/packages/payload/src/collections/config/sanitize.ts index b073c8ee992..14da240431e 100644 --- a/packages/payload/src/collections/config/sanitize.ts +++ b/packages/payload/src/collections/config/sanitize.ts @@ -47,7 +47,7 @@ export const sanitizeCollection = async ( // Make copy of collection config // ///////////////////////////////// - const sanitized: CollectionConfig = addDefaultsToCollectionConfig(collection) + const sanitized: CollectionConfig = addDefaultsToCollectionConfig(collection, config) // ///////////////////////////////// // Sanitize fields diff --git a/packages/payload/src/collections/config/types.ts b/packages/payload/src/collections/config/types.ts index c392a124426..71afeaafadb 100644 --- a/packages/payload/src/collections/config/types.ts +++ b/packages/payload/src/collections/config/types.ts @@ -505,6 +505,19 @@ export type CollectionConfig = { * Use `true` to enable with default options */ auth?: boolean | IncomingAuthType + /** + * Configuration for bulk operations + */ + bulkOperations?: { + /** + * When true, bulk operations will process documents one at a time + * in separate transactions instead of all at once in a single transaction. + * Useful for avoiding database-side transaction limitations. + * + * @default false + */ + singleTransaction?: boolean + } /** Extension point to add your custom data. Server only. */ custom?: Record /** diff --git a/packages/payload/src/collections/operations/delete.ts b/packages/payload/src/collections/operations/delete.ts index 3ad006b19c0..d3d256ac82f 100644 --- a/packages/payload/src/collections/operations/delete.ts +++ b/packages/payload/src/collections/operations/delete.ts @@ -146,6 +146,12 @@ export const deleteOperation = async < const { id } = doc try { + // Each document gets its own transaction when singleTransaction is enabled + let docShouldCommit = false + if (collectionConfig.bulkOperations?.singleTransaction) { + docShouldCommit = await initTransaction(req) + } + // ///////////////////////////////////// // Handle potentially locked documents // ///////////////////////////////////// @@ -278,9 +284,15 @@ export const deleteOperation = async < // ///////////////////////////////////// // 8. Return results // ///////////////////////////////////// + if (docShouldCommit) { + await commitTransaction(req) + } return result } catch (error) { + if (collectionConfig.bulkOperations?.singleTransaction) { + await killTransaction(req) + } errors.push({ id: doc.id, message: error instanceof Error ? error.message : 'Unknown error', @@ -289,7 +301,17 @@ export const deleteOperation = async < return null }) - const awaitedDocs = await Promise.all(promises) + // Process sequentially when using single transaction mode to avoid shared state issues + // Process in parallel when using one transaction for better performance + let awaitedDocs + if (collectionConfig.bulkOperations?.singleTransaction) { + awaitedDocs = [] + for (const promise of promises) { + awaitedDocs.push(await promise) + } + } else { + awaitedDocs = await Promise.all(promises) + } // ///////////////////////////////////// // Delete Preferences diff --git a/packages/payload/src/collections/operations/update.ts b/packages/payload/src/collections/operations/update.ts index 6d0ad1b72b9..1379e1c431b 100644 --- a/packages/payload/src/collections/operations/update.ts +++ b/packages/payload/src/collections/operations/update.ts @@ -231,6 +231,12 @@ export const updateOperation = async < const { id } = docWithLocales try { + // Each document gets its own transaction when singleTransaction is enabled + let docShouldCommit = false + if (collectionConfig.bulkOperations?.singleTransaction) { + docShouldCommit = await initTransaction(req) + } + const select = sanitizeSelect({ fields: collectionConfig.flattenedFields, forceSelect: collectionConfig.forceSelect, @@ -263,8 +269,15 @@ export const updateOperation = async < showHiddenFields: showHiddenFields!, }) + if (docShouldCommit) { + await commitTransaction(req) + } + return updatedDoc } catch (error) { + if (collectionConfig.bulkOperations?.singleTransaction) { + await killTransaction(req) + } errors.push({ id, message: error instanceof Error ? error.message : 'Unknown error', @@ -279,7 +292,17 @@ export const updateOperation = async < req, }) - const awaitedDocs = await Promise.all(promises) + // Process sequentially when using single transaction mode to avoid shared state issues + // Process in parallel when using one transaction for better performance + let awaitedDocs: (DataFromCollectionSlug | null)[] + if (collectionConfig.bulkOperations?.singleTransaction) { + awaitedDocs = [] + for (const promise of promises) { + awaitedDocs.push(await promise) + } + } else { + awaitedDocs = await Promise.all(promises) + } let result = { docs: awaitedDocs.filter(Boolean), diff --git a/packages/payload/src/config/types.ts b/packages/payload/src/config/types.ts index 839d2e9e89b..ef23c14da69 100644 --- a/packages/payload/src/config/types.ts +++ b/packages/payload/src/config/types.ts @@ -1014,6 +1014,20 @@ export type Config = { * @experimental This property is experimental and may change in future releases. Use at your own risk. */ bodyParser?: Partial + /** + * Global default configuration for bulk operations across all collections. + * Can be overridden per collection. + */ + bulkOperations?: { + /** + * When true, bulk operations will process documents one at a time + * in separate transactions instead of all at once in a single transaction. + * Useful for avoiding transaction limitations with large datasets. + * + * @default false + */ + singleTransaction?: boolean + } /** * Manage the datamodel of your application * From 840f04500a0dcb10c70247696ac1222f5cf688c5 Mon Sep 17 00:00:00 2001 From: tak-amboss Date: Wed, 29 Oct 2025 09:08:19 +0100 Subject: [PATCH 2/3] docs: add bulkOperations.singleTransaction config documentation --- docs/configuration/collections.mdx | 1 + docs/configuration/overview.mdx | 75 +++++++++++++++--------------- 2 files changed, 39 insertions(+), 37 deletions(-) diff --git a/docs/configuration/collections.mdx b/docs/configuration/collections.mdx index 0eea5d52997..311c299b891 100644 --- a/docs/configuration/collections.mdx +++ b/docs/configuration/collections.mdx @@ -65,6 +65,7 @@ The following options are available: | `admin` | The configuration options for the Admin Panel. [More details](#admin-options). | | `access` | Provide Access Control functions to define exactly who should be able to do what with Documents in this Collection. [More details](../access-control/collections). | | `auth` | Specify options if you would like this Collection to feature authentication. [More details](../authentication/overview). | +| `bulkOperations` | Configure bulk operation behavior. Set `singleTransaction: true` to process documents one at a time in separate transactions. Useful for avoiding database transaction limits with large datasets. Defaults to global config. | | `custom` | Extension point for adding custom data (e.g. for plugins) | | `disableDuplicate` | When true, do not show the "Duplicate" button while editing documents within this Collection and prevent `duplicate` from all APIs. | | `defaultSort` | Pass a top-level field to sort by default in the Collection List View. Prefix the name of the field with a minus symbol ("-") to sort in descending order. Multiple fields can be specified by using a string array. | diff --git a/docs/configuration/overview.mdx b/docs/configuration/overview.mdx index dd697723cd6..93634c577cf 100644 --- a/docs/configuration/overview.mdx +++ b/docs/configuration/overview.mdx @@ -65,43 +65,44 @@ export default buildConfig({ The following options are available: -| Option | Description | -| -------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| **`admin`** | The configuration options for the Admin Panel, including Custom Components, Live Preview, etc. [More details](../admin/overview#admin-options). | -| **`bin`** | Register custom bin scripts for Payload to execute. [More Details](#custom-bin-scripts). | -| **`editor`** | The Rich Text Editor which will be used by `richText` fields. [More details](../rich-text/overview). | -| **`db`** \* | The Database Adapter which will be used by Payload. [More details](../database/overview). | -| **`serverURL`** | A string used to define the absolute URL of your app. This includes the protocol, for example `https://example.com`. No paths allowed, only protocol, domain and (optionally) port. | -| **`collections`** | An array of Collections for Payload to manage. [More details](./collections). | -| **`compatibility`** | Compatibility flags for earlier versions of Payload. [More details](#compatibility-flags). | -| **`globals`** | An array of Globals for Payload to manage. [More details](./globals). | -| **`cors`** | Cross-origin resource sharing (CORS) is a mechanism that accept incoming requests from given domains. You can also customize the `Access-Control-Allow-Headers` header. [More details](#cors). | -| **`localization`** | Opt-in to translate your content into multiple locales. [More details](./localization). | -| **`logger`** | Logger options, logger options with a destination stream, or an instantiated logger instance. [More details](https://getpino.io/#/docs/api?id=options). | -| **`loggingLevels`** | An object to override the level to use in the logger for Payload's errors. | -| **`graphQL`** | Manage GraphQL-specific functionality, including custom queries and mutations, query complexity limits, etc. [More details](../graphql/overview#graphql-options). | -| **`cookiePrefix`** | A string that will be prefixed to all cookies that Payload sets. | -| **`csrf`** | A whitelist array of URLs to allow Payload to accept cookies from. [More details](../authentication/cookies#csrf-attacks). | -| **`defaultDepth`** | If a user does not specify `depth` while requesting a resource, this depth will be used. [More details](../queries/depth). | -| **`defaultMaxTextLength`** | The maximum allowed string length to be permitted application-wide. Helps to prevent malicious public document creation. | -| `folders` | An optional object to configure global folder settings. [More details](../folders/overview). | -| `queryPresets` | An object that to configure Collection Query Presets. [More details](../query-presets/overview). | -| **`maxDepth`** | The maximum allowed depth to be permitted application-wide. This setting helps prevent against malicious queries. Defaults to `10`. [More details](../queries/depth). | -| **`indexSortableFields`** | Automatically index all sortable top-level fields in the database to improve sort performance and add database compatibility for Azure Cosmos and similar. | -| **`upload`** | Base Payload upload configuration. [More details](../upload/overview#payload-wide-upload-options). | -| **`routes`** | Control the routing structure that Payload binds itself to. [More details](../admin/overview#root-level-routes). | -| **`email`** | Configure the Email Adapter for Payload to use. [More details](../email/overview). | -| **`onInit`** | A function that is called immediately following startup that receives the Payload instance as its only argument. | -| **`debug`** | Enable to expose more detailed error information. | -| **`telemetry`** | Disable Payload telemetry by passing `false`. [More details](#telemetry). | -| **`hooks`** | An array of Root Hooks. [More details](../hooks/overview). | -| **`plugins`** | An array of Plugins. [More details](../plugins/overview). | -| **`endpoints`** | An array of Custom Endpoints added to the Payload router. [More details](../rest-api/overview#custom-endpoints). | -| **`custom`** | Extension point for adding custom data (e.g. for plugins). | -| **`i18n`** | Internationalization configuration. Pass all i18n languages you'd like the admin UI to support. Defaults to English-only. [More details](./i18n). | -| **`secret`** \* | A secure, unguessable string that Payload will use for any encryption workflows - for example, password salt / hashing. | -| **`sharp`** | If you would like Payload to offer cropping, focal point selection, and automatic media resizing, install and pass the Sharp module to the config here. | -| **`typescript`** | Configure TypeScript settings here. [More details](#typescript). | +| Option | Description | +| -------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| **`admin`** | The configuration options for the Admin Panel, including Custom Components, Live Preview, etc. [More details](../admin/overview#admin-options). | +| **`bin`** | Register custom bin scripts for Payload to execute. [More Details](#custom-bin-scripts). | +| **`bulkOperations`** | Global defaults for bulk operations across all collections. Set `singleTransaction: true` to process documents one at a time in separate transactions instead of all at once. Defaults to `false`. Can be overridden per collection. | +| **`editor`** | The Rich Text Editor which will be used by `richText` fields. [More details](../rich-text/overview). | +| **`db`** \* | The Database Adapter which will be used by Payload. [More details](../database/overview). | +| **`serverURL`** | A string used to define the absolute URL of your app. This includes the protocol, for example `https://example.com`. No paths allowed, only protocol, domain and (optionally) port. | +| **`collections`** | An array of Collections for Payload to manage. [More details](./collections). | +| **`compatibility`** | Compatibility flags for earlier versions of Payload. [More details](#compatibility-flags). | +| **`globals`** | An array of Globals for Payload to manage. [More details](./globals). | +| **`cors`** | Cross-origin resource sharing (CORS) is a mechanism that accept incoming requests from given domains. You can also customize the `Access-Control-Allow-Headers` header. [More details](#cors). | +| **`localization`** | Opt-in to translate your content into multiple locales. [More details](./localization). | +| **`logger`** | Logger options, logger options with a destination stream, or an instantiated logger instance. [More details](https://getpino.io/#/docs/api?id=options). | +| **`loggingLevels`** | An object to override the level to use in the logger for Payload's errors. | +| **`graphQL`** | Manage GraphQL-specific functionality, including custom queries and mutations, query complexity limits, etc. [More details](../graphql/overview#graphql-options). | +| **`cookiePrefix`** | A string that will be prefixed to all cookies that Payload sets. | +| **`csrf`** | A whitelist array of URLs to allow Payload to accept cookies from. [More details](../authentication/cookies#csrf-attacks). | +| **`defaultDepth`** | If a user does not specify `depth` while requesting a resource, this depth will be used. [More details](../queries/depth). | +| **`defaultMaxTextLength`** | The maximum allowed string length to be permitted application-wide. Helps to prevent malicious public document creation. | +| `folders` | An optional object to configure global folder settings. [More details](../folders/overview). | +| `queryPresets` | An object that to configure Collection Query Presets. [More details](../query-presets/overview). | +| **`maxDepth`** | The maximum allowed depth to be permitted application-wide. This setting helps prevent against malicious queries. Defaults to `10`. [More details](../queries/depth). | +| **`indexSortableFields`** | Automatically index all sortable top-level fields in the database to improve sort performance and add database compatibility for Azure Cosmos and similar. | +| **`upload`** | Base Payload upload configuration. [More details](../upload/overview#payload-wide-upload-options). | +| **`routes`** | Control the routing structure that Payload binds itself to. [More details](../admin/overview#root-level-routes). | +| **`email`** | Configure the Email Adapter for Payload to use. [More details](../email/overview). | +| **`onInit`** | A function that is called immediately following startup that receives the Payload instance as its only argument. | +| **`debug`** | Enable to expose more detailed error information. | +| **`telemetry`** | Disable Payload telemetry by passing `false`. [More details](#telemetry). | +| **`hooks`** | An array of Root Hooks. [More details](../hooks/overview). | +| **`plugins`** | An array of Plugins. [More details](../plugins/overview). | +| **`endpoints`** | An array of Custom Endpoints added to the Payload router. [More details](../rest-api/overview#custom-endpoints). | +| **`custom`** | Extension point for adding custom data (e.g. for plugins). | +| **`i18n`** | Internationalization configuration. Pass all i18n languages you'd like the admin UI to support. Defaults to English-only. [More details](./i18n). | +| **`secret`** \* | A secure, unguessable string that Payload will use for any encryption workflows - for example, password salt / hashing. | +| **`sharp`** | If you would like Payload to offer cropping, focal point selection, and automatic media resizing, install and pass the Sharp module to the config here. | +| **`typescript`** | Configure TypeScript settings here. [More details](#typescript). | _\* An asterisk denotes that a property is required._ From a178e74710f77f9a88042aa30664667f7e49405c Mon Sep 17 00:00:00 2001 From: tak-amboss Date: Wed, 29 Oct 2025 09:43:28 +0100 Subject: [PATCH 3/3] test: add singleTransaction tests for bulk operations --- test/database/int.spec.ts | 54 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/test/database/int.spec.ts b/test/database/int.spec.ts index 172f722d476..49636fac931 100644 --- a/test/database/int.spec.ts +++ b/test/database/int.spec.ts @@ -2011,6 +2011,60 @@ describe('database', () => { expect(worldDocs).toHaveLength(5) }) + it('should bulk update with singleTransaction: true', async () => { + const collectionConfig = payload.config.collections.find((c) => c.slug === collection) + + assert(collectionConfig) + + const originalConfig = collectionConfig.bulkOperations + collectionConfig.bulkOperations = { singleTransaction: true } + + try { + const posts = await Promise.all([ + payload.create({ collection, data: { title: 'test1' } }), + payload.create({ collection, data: { title: 'test2' } }), + payload.create({ collection, data: { title: 'test3' } }), + ]) + + const result = await payload.update({ + collection, + data: { title: 'updated' }, + where: { id: { in: posts.map((p) => p.id) } }, + }) + + expect(result.docs).toHaveLength(3) + expect(result.errors).toHaveLength(0) + } finally { + collectionConfig.bulkOperations = originalConfig + } + }) + + it('should bulk delete with singleTransaction: true', async () => { + const collectionConfig = payload.config.collections.find((c) => c.slug === collection) + + assert(collectionConfig) + + const originalConfig = collectionConfig.bulkOperations + collectionConfig.bulkOperations = { singleTransaction: true } + + try { + const posts = await Promise.all([ + payload.create({ collection, data: { title: 'toDelete1' } }), + payload.create({ collection, data: { title: 'toDelete2' } }), + ]) + + const result = await payload.delete({ + collection, + where: { id: { in: posts.map((p) => p.id) } }, + }) + + expect(result.docs).toHaveLength(2) + expect(result.errors).toHaveLength(0) + } finally { + collectionConfig.bulkOperations = originalConfig + } + }) + it('should CRUD point field', async () => { const result = await payload.create({ collection: 'default-values',