diff --git a/source/client-side-encryption/limits/limits-encryptedFields.json b/source/client-side-encryption/limits/limits-encryptedFields.json new file mode 100644 index 0000000000..c52a0271e1 --- /dev/null +++ b/source/client-side-encryption/limits/limits-encryptedFields.json @@ -0,0 +1,14 @@ +{ + "fields": [ + { + "keyId": { + "$binary": { + "base64": "LOCALAAAAAAAAAAAAAAAAA==", + "subType": "04" + } + }, + "path": "foo", + "bsonType": "string" + } + ] +} \ No newline at end of file diff --git a/source/client-side-encryption/limits/limits-qe-doc.json b/source/client-side-encryption/limits/limits-qe-doc.json new file mode 100644 index 0000000000..71efbf4068 --- /dev/null +++ b/source/client-side-encryption/limits/limits-qe-doc.json @@ -0,0 +1,3 @@ +{ + "foo": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" +} \ No newline at end of file diff --git a/source/client-side-encryption/tests/README.md b/source/client-side-encryption/tests/README.md index e0677bbab7..7768b03cc0 100644 --- a/source/client-side-encryption/tests/README.md +++ b/source/client-side-encryption/tests/README.md @@ -563,10 +563,13 @@ First, perform the setup. 2. Using `client`, drop and create the collection `db.coll` configured with the included JSON schema [limits/limits-schema.json](../limits/limits-schema.json). -3. Using `client`, drop the collection `keyvault.datakeys`. Insert the document +3. Using `client`, drop and create the collection `db.coll2` configured with the included encryptedFields + [limits/limits-encryptedFields.json](../limits/limits-encryptedFields.json). + +4. Using `client`, drop the collection `keyvault.datakeys`. Insert the document [limits/limits-key.json](../limits/limits-key.json) -4. Create a MongoClient configured with auto encryption (referred to as `client_encrypted`) +5. Create a MongoClient configured with auto encryption (referred to as `client_encrypted`) Configure with the `local` KMS provider as follows: @@ -578,19 +581,19 @@ First, perform the setup. Using `client_encrypted` perform the following operations: -1. Insert `{ "_id": "over_2mib_under_16mib", "unencrypted": }`. +1. Insert `{ "_id": "over_2mib_under_16mib", "unencrypted": }` into `coll`. Expect this to succeed since this is still under the `maxBsonObjectSize` limit. 2. Insert the document [limits/limits-doc.json](../limits/limits-doc.json) concatenated with - `{ "_id": "encryption_exceeds_2mib", "unencrypted": < the string "a" repeated (2097152 - 2000) times > }` Note: + `{ "_id": "encryption_exceeds_2mib", "unencrypted": < the string "a" repeated (2097152 - 2000) times > }` into `coll`. Note: limits-doc.json is a 1005 byte BSON document that encrypts to a ~10,000 byte document. Expect this to succeed since after encryption this still is below the normal maximum BSON document size. Note, before auto encryption this document is under the 2 MiB limit. After encryption it exceeds the 2 MiB limit, but does NOT exceed the 16 MiB limit. -3. Bulk insert the following: +3. Use MongoCollection.bulkWrite to insert the following into `coll`: - `{ "_id": "over_2mib_1", "unencrypted": }` - `{ "_id": "over_2mib_2", "unencrypted": }` @@ -598,7 +601,7 @@ Using `client_encrypted` perform the following operations: Expect the bulk write to succeed and split after first doc (i.e. two inserts occur). This may be verified using [command monitoring](../../command-logging-and-monitoring/command-logging-and-monitoring.md). -4. Bulk insert the following: +4. Use MongoCollection.bulkWrite insert the following into `coll`: - The document [limits/limits-doc.json](../limits/limits-doc.json) concatenated with `{ "_id": "encryption_exceeds_2mib_1", "unencrypted": < the string "a" repeated (2097152 - 2000) times > }` @@ -608,15 +611,36 @@ Using `client_encrypted` perform the following operations: Expect the bulk write to succeed and split after first doc (i.e. two inserts occur). This may be verified using [command logging and monitoring](../../command-logging-and-monitoring/command-logging-and-monitoring.md). -5. Insert `{ "_id": "under_16mib", "unencrypted": `. +5. Insert `{ "_id": "under_16mib", "unencrypted": ` into `coll`. Expect this to succeed since this is still (just) under the `maxBsonObjectSize` limit. 6. Insert the document [limits/limits-doc.json](../limits/limits-doc.json) concatenated with - `{ "_id": "encryption_exceeds_16mib", "unencrypted": < the string "a" repeated (16777216 - 2000) times > }` + `{ "_id": "encryption_exceeds_16mib", "unencrypted": < the string "a" repeated (16777216 - 2000) times > }` into `coll`. Expect this to fail since encryption results in a document exceeding the `maxBsonObjectSize` limit. +> [!NOTE] +> MongoDB 8.0+ is required for MongoClient.bulkWrite + +7. Use MongoClient.bulkWrite to insert the following into `coll2`: + + - `{ "_id": "over_2mib_3", "unencrypted": }` + - `{ "_id": "over_2mib_4", "unencrypted": }` + + Expect the bulk write to succeed and split after first doc (i.e. two inserts occur). This may be verified using + [command logging and monitoring](../../command-logging-and-monitoring/command-logging-and-monitoring.md). + +8. Use MongoClient.bulkWrite to insert the following into `coll2`: + + - The document [limits/limits-qe-doc.json](../limits/limits-qe-doc.json) concatenated with + `{ "_id": "encryption_exceeds_2mib_3", "foo": < the string "a" repeated (2097152 - 2000 - 1500) times > }` + - The document [limits/limits-qe-doc.json](../limits/limits-qe-doc.json) concatenated with + `{ "_id": "encryption_exceeds_2mib_4", "foo": < the string "a" repeated (2097152 - 2000 - 1500) times > }` + + Expect the bulk write to succeed and split after first doc (i.e. two inserts occur). This may be verified using + [command logging and monitoring](../../command-logging-and-monitoring/command-logging-and-monitoring.md). + Optionally, if it is possible to mock the maxWriteBatchSize (i.e. the maximum number of documents in a batch) test that setting maxWriteBatchSize=1 and inserting the two documents `{ "_id": "a" }, { "_id": "b" }` with `client_encrypted` splits the operation into two inserts. diff --git a/source/crud/bulk-write.md b/source/crud/bulk-write.md index 28c5b847fd..3f5eea6718 100644 --- a/source/crud/bulk-write.md +++ b/source/crud/bulk-write.md @@ -459,7 +459,7 @@ class BulkWriteResult { * The results of each individual write operation that was successfully performed. * * This value will only be populated if the verboseResults option was set to true. - */ + */ verboseResults: Optional; /* rest of fields */ @@ -553,7 +553,9 @@ The `bulkWrite` server command has the following format: } ``` -Drivers MUST use document sequences ([`OP_MSG`](../message/OP_MSG.md) payload type 1) for the `ops` and `nsInfo` fields. +If auto-encryption is not enabled, drivers MUST use document sequences ([`OP_MSG`](../message/OP_MSG.md) payload type 1) for +the `ops` and `nsInfo` fields. If auto-encryption is enabled, drivers MUST NOT use document sequences and MUST append the +`ops` and `nsInfo` fields to the `bulkWrite` command document. The `bulkWrite` command is executed on the "admin" database. @@ -645,13 +647,6 @@ write concern containing the following message: > Cannot request unacknowledged write concern and ordered writes -## Auto-Encryption - -If `MongoClient.bulkWrite` is called on a `MongoClient` configured with `AutoEncryptionOpts`, drivers MUST return an -error with the message: "bulkWrite does not currently support automatic encryption". - -This is expected to be removed once [DRIVERS-2888](https://jira.mongodb.org/browse/DRIVERS-2888) is implemented. - ## Command Batching Drivers MUST accept an arbitrary number of operations as input to the `MongoClient.bulkWrite` method. Because the server @@ -672,8 +667,10 @@ multiple commands if the user provides more than `maxWriteBatchSize` operations ### Total Message Size -Drivers MUST ensure that the total size of the `OP_MSG` built for each `bulkWrite` command does not exceed -`maxMessageSizeBytes`. +#### Unencrypted bulk writes + +When auto-encryption is not enabled, drivers MUST ensure that the total size of the `OP_MSG` built for each `bulkWrite` +command does not exceed `maxMessageSizeBytes`. The upper bound for the size of an `OP_MSG` includes opcode-related bytes (e.g. the `OP_MSG` header) and operation-agnostic command field bytes (e.g. `txnNumber`, `lsid`). Drivers MUST limit the combined size of the @@ -727,6 +724,12 @@ was determined. Drivers MUST return an error if there is not room to add at least one operation to `ops`. +#### Auto-encrypted bulk writes + +Drivers MUST use the reduced size limit defined in +[Size limits for Write Commands](../client-side-encryption/client-side-encryption.md#size-limits-for-write-commands) +for the size of the `bulkWrite` command document when auto-encryption is enabled. + ## Handling the `bulkWrite` Server Response The server's response to `bulkWrite` has the following format: @@ -857,6 +860,12 @@ When a `getMore` fails with a retryable error when attempting to iterate the res entire `bulkWrite` command to receive a fresh cursor and retry iteration. This work was omitted to minimize the scope of the initial implementation and testing of the new bulk write API, but may be revisited in the future. +### Use document sequences for auto-encrypted bulk writes + +Auto-encryption does not currently support document sequences. This specification should be updated when +[DRIVERS-2859](https://jira.mongodb.org/browse/DRIVERS-2859) is completed to require use of document sequences for `ops` +and `nsInfo` when auto-encryption is enabled. + ## Q&A ### Is `bulkWrite` supported on Atlas Serverless? @@ -928,6 +937,8 @@ error in this specific situation does not seem helpful enough to require size ch ## **Changelog** +- 2025-08-13: Removed the requirement to error when QE is enabled. + - 2025-06-27: Added `rawData` option. - 2024-11-05: Updated the requirements regarding the size validation. diff --git a/source/crud/tests/unified/client-bulkWrite-qe.json b/source/crud/tests/unified/client-bulkWrite-qe.json new file mode 100644 index 0000000000..b732793400 --- /dev/null +++ b/source/crud/tests/unified/client-bulkWrite-qe.json @@ -0,0 +1,298 @@ +{ + "description": "client bulkWrite with queryable encryption", + "schemaVersion": "1.23", + "runOnRequirements": [ + { + "minServerVersion": "8.0", + "serverless": "forbid", + "csfle": true + } + ], + "createEntities": [ + { + "client": { + "id": "client0", + "observeEvents": [ + "commandStartedEvent", + "commandSucceededEvent" + ], + "autoEncryptOpts": { + "keyVaultNamespace": "keyvault.datakeys", + "kmsProviders": { + "local": { + "key": "Mng0NCt4ZHVUYUJCa1kxNkVyNUR1QURhZ2h2UzR2d2RrZzh0cFBwM3R6NmdWMDFBMUN3YkQ5aXRRMkhGRGdQV09wOGVNYUMxT2k3NjZKelhaQmRCZGJkTXVyZG9uSjFk" + } + } + } + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "crud-tests" + } + }, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": "coll0" + } + }, + { + "client": { + "id": "client1", + "observeEvents": [ + "commandStartedEvent" + ] + } + }, + { + "database": { + "id": "database1", + "client": "client0", + "databaseName": "keyvault" + } + }, + { + "collection": { + "id": "collection1", + "database": "database0", + "collectionName": "datakeys" + } + }, + { + "database": { + "id": "database2", + "client": "client1", + "databaseName": "crud-tests" + } + }, + { + "collection": { + "id": "collection2", + "database": "database2", + "collectionName": "coll0" + } + } + ], + "initialData": [ + { + "databaseName": "keyvault", + "collectionName": "datakeys", + "documents": [ + { + "_id": { + "$binary": { + "base64": "EjRWeBI0mHYSNBI0VniQEg==", + "subType": "04" + } + }, + "keyAltNames": [ + "local_key" + ], + "keyMaterial": { + "$binary": { + "base64": "sHe0kz57YW7v8g9VP9sf/+K1ex4JqKc5rf/URX3n3p8XdZ6+15uXPaSayC6adWbNxkFskuMCOifDoTT+rkqMtFkDclOy884RuGGtUysq3X7zkAWYTKi8QAfKkajvVbZl2y23UqgVasdQu3OVBQCrH/xY00nNAs/52e958nVjBuzQkSb1T8pKJAyjZsHJ60+FtnfafDZSTAIBJYn7UWBCwQ==", + "subType": "00" + } + }, + "creationDate": { + "$date": { + "$numberLong": "1641024000000" + } + }, + "updateDate": { + "$date": { + "$numberLong": "1641024000000" + } + }, + "status": 1, + "masterKey": { + "provider": "local" + } + } + ] + }, + { + "databaseName": "crud-tests", + "collectionName": "coll0", + "documents": [], + "createOptions": { + "encryptedFields": { + "fields": [ + { + "keyId": { + "$binary": { + "base64": "EjRWeBI0mHYSNBI0VniQEg==", + "subType": "04" + } + }, + "path": "encryptedInt", + "bsonType": "int", + "queries": { + "queryType": "equality", + "contention": { + "$numberLong": "0" + } + } + } + ] + } + } + } + ], + "_yamlAnchors": { + "namespace": "crud-tests.coll0" + }, + "tests": [ + { + "description": "client bulkWrite QE replaceOne", + "operations": [ + { + "object": "collection0", + "name": "insertMany", + "arguments": { + "documents": [ + { + "_id": 1, + "encryptedInt": 11 + }, + { + "_id": 2, + "encryptedInt": 22 + }, + { + "_id": 3, + "encryptedInt": 33 + } + ] + } + }, + { + "object": "client0", + "name": "clientBulkWrite", + "arguments": { + "models": [ + { + "replaceOne": { + "namespace": "crud-tests.coll0", + "filter": { + "encryptedInt": { + "$eq": 11 + } + }, + "replacement": { + "encryptedInt": 44 + } + } + } + ] + }, + "expectResult": { + "insertedCount": 0, + "upsertedCount": 0, + "matchedCount": 1, + "modifiedCount": 1, + "deletedCount": 0 + } + }, + { + "object": "collection0", + "name": "find", + "arguments": { + "filter": { + "encryptedInt": 44 + } + }, + "expectResult": [ + { + "_id": 1, + "encryptedInt": 44 + } + ] + }, + { + "object": "collection2", + "name": "find", + "arguments": { + "filter": {} + }, + "expectResult": [ + { + "_id": 1, + "encryptedInt": { + "$$type": "binData" + }, + "__safeContent__": { + "$$type": "array" + } + }, + { + "_id": 2, + "encryptedInt": { + "$$type": "binData" + }, + "__safeContent__": { + "$$type": "array" + } + }, + { + "_id": 3, + "encryptedInt": { + "$$type": "binData" + }, + "__safeContent__": { + "$$type": "array" + } + } + ] + } + ] + }, + { + "description": "client bulkWrite QE with multiple replace fails", + "operations": [ + { + "object": "client0", + "name": "clientBulkWrite", + "arguments": { + "models": [ + { + "replaceOne": { + "namespace": "crud-tests.coll0", + "filter": { + "encryptedInt": { + "$eq": 11 + } + }, + "replacement": { + "encryptedInt": 44 + } + } + }, + { + "replaceOne": { + "namespace": "crud-tests.coll0", + "filter": { + "encryptedInt": { + "$eq": 22 + } + }, + "replacement": { + "encryptedInt": 44 + } + } + } + ] + }, + "expectError": { + "isError": true, + "errorContains": "Only insert is supported in BulkWrite with multiple operations and Queryable Encryption" + } + } + ] + } + ] +} diff --git a/source/crud/tests/unified/client-bulkWrite-qe.yml b/source/crud/tests/unified/client-bulkWrite-qe.yml new file mode 100644 index 0000000000..5ee0fbe0a2 --- /dev/null +++ b/source/crud/tests/unified/client-bulkWrite-qe.yml @@ -0,0 +1,128 @@ +description: client bulkWrite with queryable encryption + +schemaVersion: "1.23" + +runOnRequirements: + - minServerVersion: "8.0" + serverless: forbid # Serverless does not support bulkWrite: CLOUDP-256344. + csfle: true + +createEntities: + - client: + id: &client0 client0 + observeEvents: + - commandStartedEvent + - commandSucceededEvent + autoEncryptOpts: + keyVaultNamespace: keyvault.datakeys + kmsProviders: + local: + key: Mng0NCt4ZHVUYUJCa1kxNkVyNUR1QURhZ2h2UzR2d2RrZzh0cFBwM3R6NmdWMDFBMUN3YkQ5aXRRMkhGRGdQV09wOGVNYUMxT2k3NjZKelhaQmRCZGJkTXVyZG9uSjFk + - database: + id: &database0 database0 + client: *client0 + databaseName: &database0Name crud-tests + - collection: + id: &collection0 collection0 + database: *database0 + collectionName: &collection0Name coll0 + - client: + id: &client1 client1 + observeEvents: + - commandStartedEvent + - database: + id: &database1 database1 + client: *client0 + databaseName: &database1Name keyvault + - collection: + id: &collection1 collection1 + database: *database0 + collectionName: &collection1Name datakeys + - database: + id: &database2 database2 + client: *client1 + databaseName: &database0Name crud-tests + - collection: + id: &collection2 collection2 + database: *database2 + collectionName: &collection0Name coll0 + + +initialData: + - databaseName: *database1Name + collectionName: *collection1Name + documents: + - _id: &local_key_id { $binary: { base64: EjRWeBI0mHYSNBI0VniQEg==, subType: "04" } } + keyAltNames: ["local_key"] + keyMaterial: { $binary: { base64: sHe0kz57YW7v8g9VP9sf/+K1ex4JqKc5rf/URX3n3p8XdZ6+15uXPaSayC6adWbNxkFskuMCOifDoTT+rkqMtFkDclOy884RuGGtUysq3X7zkAWYTKi8QAfKkajvVbZl2y23UqgVasdQu3OVBQCrH/xY00nNAs/52e958nVjBuzQkSb1T8pKJAyjZsHJ60+FtnfafDZSTAIBJYn7UWBCwQ==, subType: "00" } } + creationDate: { $date: { $numberLong: "1641024000000" } } + updateDate: { $date: { $numberLong: "1641024000000" } } + status: 1 + masterKey: &local_masterkey + provider: local + - databaseName: *database0Name + collectionName: *collection0Name + documents: [] + createOptions: + encryptedFields: &encrypted_fields {'fields': [{'keyId': {'$binary': {'base64': 'EjRWeBI0mHYSNBI0VniQEg==', 'subType': '04'}}, 'path': 'encryptedInt', 'bsonType': 'int', 'queries': {'queryType': 'equality', 'contention': {'$numberLong': '0'}}}]} + +_yamlAnchors: + namespace: &namespace "crud-tests.coll0" + +tests: + - description: client bulkWrite QE replaceOne + operations: + - object: *collection0 + name: insertMany + arguments: + documents: + - { _id: 1, encryptedInt: 11 } + - { _id: 2, encryptedInt: 22 } + - { _id: 3, encryptedInt: 33 } + - object: *client0 + name: clientBulkWrite + arguments: + models: + - replaceOne: + namespace: *namespace + filter: { encryptedInt: { $eq: 11 } } + replacement: { encryptedInt: 44 } + expectResult: + insertedCount: 0 + upsertedCount: 0 + matchedCount: 1 + modifiedCount: 1 + deletedCount: 0 + - object: *collection0 + name: find + arguments: + filter: { encryptedInt: 44 } + expectResult: + - _id: 1 + encryptedInt: 44 + - object: *collection2 + name: find + arguments: + filter: {} + expectResult: + - { _id: 1, encryptedInt: { $$type: binData }, __safeContent__: { $$type: array} } + - { _id: 2, encryptedInt: { $$type: binData }, __safeContent__: { $$type: array} } + - { _id: 3, encryptedInt: { $$type: binData }, __safeContent__: { $$type: array} } + - description: client bulkWrite QE with multiple replace fails + operations: + - object: *client0 + name: clientBulkWrite + arguments: + models: + - replaceOne: + namespace: *namespace + filter: { encryptedInt: { $eq: 11 } } + replacement: { encryptedInt: 44 } + - replaceOne: + namespace: *namespace + filter: { encryptedInt: { $eq: 22 } } + replacement: { encryptedInt: 44 } + expectError: + # Expect error from mongocryptd or crypt_shared + isError: true + errorContains: "Only insert is supported in BulkWrite with multiple operations and Queryable Encryption"