From 835e027180e2c822a90b3aeb0846d1f635fda76d Mon Sep 17 00:00:00 2001 From: Shane Harvey Date: Thu, 6 Mar 2025 15:03:17 -0800 Subject: [PATCH 01/10] PYTHON-4580 Add key_expiration_ms option for DEK cache lifetime --- pymongo/asynchronous/encryption.py | 17 +++++++++++++++-- pymongo/encryption_options.py | 10 ++++++++-- pymongo/synchronous/encryption.py | 17 +++++++++++++++-- 3 files changed, 38 insertions(+), 6 deletions(-) diff --git a/pymongo/asynchronous/encryption.py b/pymongo/asynchronous/encryption.py index 9d3ea67191..07d62a8a45 100644 --- a/pymongo/asynchronous/encryption.py +++ b/pymongo/asynchronous/encryption.py @@ -445,6 +445,7 @@ def _get_internal_client( bypass_encryption=opts._bypass_auto_encryption, encrypted_fields_map=encrypted_fields_map, bypass_query_analysis=opts._bypass_query_analysis, + key_expiration_ms=opts._key_expiration_ms, ), ) self._closed = False @@ -564,6 +565,7 @@ def __init__( key_vault_client: AsyncMongoClient[_DocumentTypeArg], codec_options: CodecOptions[_DocumentTypeArg], kms_tls_options: Optional[Mapping[str, Any]] = None, + key_expiration_ms: Optional[int] = None, ) -> None: """Explicit client-side field level encryption. @@ -630,7 +632,12 @@ def __init__( Or to supply a client certificate:: kms_tls_options={'kmip': {'tlsCertificateKeyFile': 'client.pem'}} + :param key_expiration_ms: The cache expiration time for data encryption keys. + Defaults to ``None`` which defers to libmongocrypt's default which is currently 60000. + Set to 0 to disable key expiration. + .. versionchanged:: 4.12 + Added the `key_expiration_ms` parameter. .. versionchanged:: 4.0 Added the `kms_tls_options` parameter and the "kmip" KMS provider. @@ -666,14 +673,19 @@ def __init__( key_vault_coll = key_vault_client[db][coll] opts = AutoEncryptionOpts( - kms_providers, key_vault_namespace, kms_tls_options=kms_tls_options + kms_providers, + key_vault_namespace, + kms_tls_options=kms_tls_options, + key_expiration_ms=key_expiration_ms, ) self._io_callbacks: Optional[_EncryptionIO] = _EncryptionIO( None, key_vault_coll, None, opts ) self._encryption = AsyncExplicitEncrypter( self._io_callbacks, - _create_mongocrypt_options(kms_providers=kms_providers, schema_map=None), + _create_mongocrypt_options( + kms_providers=kms_providers, schema_map=None, key_expiration_ms=key_expiration_ms + ), ) # Use the same key vault collection as the callback. assert self._io_callbacks.key_vault_coll is not None @@ -700,6 +712,7 @@ async def create_encrypted_collection( creation. :class:`~pymongo.errors.EncryptionError` will be raised if the collection already exists. + :param database: the database to create the collection :param name: the name of the collection to create :param encrypted_fields: Document that describes the encrypted fields for Queryable Encryption. The "keyId" may be set to ``None`` to auto-generate the data keys. For example: diff --git a/pymongo/encryption_options.py b/pymongo/encryption_options.py index 26dfbf5f03..a1c40dc7b2 100644 --- a/pymongo/encryption_options.py +++ b/pymongo/encryption_options.py @@ -57,6 +57,7 @@ def __init__( crypt_shared_lib_required: bool = False, bypass_query_analysis: bool = False, encrypted_fields_map: Optional[Mapping[str, Any]] = None, + key_expiration_ms: Optional[int] = None, ) -> None: """Options to configure automatic client-side field level encryption. @@ -191,9 +192,14 @@ def __init__( ] } } + :param key_expiration_ms: The cache expiration time for data encryption keys. + Defaults to ``None`` which defers to libmongocrypt's default which is currently 60000. + Set to 0 to disable key expiration. + .. versionchanged:: 4.12 + Added the `key_expiration_ms` parameter. .. versionchanged:: 4.2 - Added `encrypted_fields_map` `crypt_shared_lib_path`, `crypt_shared_lib_required`, + Added the `encrypted_fields_map`, `crypt_shared_lib_path`, `crypt_shared_lib_required`, and `bypass_query_analysis` parameters. .. versionchanged:: 4.0 @@ -210,7 +216,6 @@ def __init__( if encrypted_fields_map: validate_is_mapping("encrypted_fields_map", encrypted_fields_map) self._encrypted_fields_map = encrypted_fields_map - self._bypass_query_analysis = bypass_query_analysis self._crypt_shared_lib_path = crypt_shared_lib_path self._crypt_shared_lib_required = crypt_shared_lib_required self._kms_providers = kms_providers @@ -233,6 +238,7 @@ def __init__( # Maps KMS provider name to a SSLContext. self._kms_ssl_contexts = _parse_kms_tls_options(kms_tls_options) self._bypass_query_analysis = bypass_query_analysis + self._key_expiration_ms = key_expiration_ms class RangeOpts: diff --git a/pymongo/synchronous/encryption.py b/pymongo/synchronous/encryption.py index 7cbac1c509..71fa9fb36b 100644 --- a/pymongo/synchronous/encryption.py +++ b/pymongo/synchronous/encryption.py @@ -442,6 +442,7 @@ def _get_internal_client( bypass_encryption=opts._bypass_auto_encryption, encrypted_fields_map=encrypted_fields_map, bypass_query_analysis=opts._bypass_query_analysis, + key_expiration_ms=opts._key_expiration_ms, ), ) self._closed = False @@ -561,6 +562,7 @@ def __init__( key_vault_client: MongoClient[_DocumentTypeArg], codec_options: CodecOptions[_DocumentTypeArg], kms_tls_options: Optional[Mapping[str, Any]] = None, + key_expiration_ms: Optional[int] = None, ) -> None: """Explicit client-side field level encryption. @@ -627,7 +629,12 @@ def __init__( Or to supply a client certificate:: kms_tls_options={'kmip': {'tlsCertificateKeyFile': 'client.pem'}} + :param key_expiration_ms: The cache expiration time for data encryption keys. + Defaults to ``None`` which defers to libmongocrypt's default which is currently 60000. + Set to 0 to disable key expiration. + .. versionchanged:: 4.12 + Added the `key_expiration_ms` parameter. .. versionchanged:: 4.0 Added the `kms_tls_options` parameter and the "kmip" KMS provider. @@ -659,14 +666,19 @@ def __init__( key_vault_coll = key_vault_client[db][coll] opts = AutoEncryptionOpts( - kms_providers, key_vault_namespace, kms_tls_options=kms_tls_options + kms_providers, + key_vault_namespace, + kms_tls_options=kms_tls_options, + key_expiration_ms=key_expiration_ms, ) self._io_callbacks: Optional[_EncryptionIO] = _EncryptionIO( None, key_vault_coll, None, opts ) self._encryption = ExplicitEncrypter( self._io_callbacks, - _create_mongocrypt_options(kms_providers=kms_providers, schema_map=None), + _create_mongocrypt_options( + kms_providers=kms_providers, schema_map=None, key_expiration_ms=key_expiration_ms + ), ) # Use the same key vault collection as the callback. assert self._io_callbacks.key_vault_coll is not None @@ -693,6 +705,7 @@ def create_encrypted_collection( creation. :class:`~pymongo.errors.EncryptionError` will be raised if the collection already exists. + :param database: the database to create the collection :param name: the name of the collection to create :param encrypted_fields: Document that describes the encrypted fields for Queryable Encryption. The "keyId" may be set to ``None`` to auto-generate the data keys. For example: From e730515d65b3c0566648350e74c970f77687e9a8 Mon Sep 17 00:00:00 2001 From: Shane Harvey Date: Thu, 6 Mar 2025 15:29:29 -0800 Subject: [PATCH 02/10] PYTHON-4580 Add spec tests --- .../spec/legacy/fle2v2-Rangev2-Compact.json | 3 +- .../spec/legacy/keyCache.json | 270 ++++++++++++++++++ .../spec/legacy/timeoutMS.json | 4 +- .../spec/unified/keyCache.json | 198 +++++++++++++ 4 files changed, 471 insertions(+), 4 deletions(-) create mode 100644 test/client-side-encryption/spec/legacy/keyCache.json create mode 100644 test/client-side-encryption/spec/unified/keyCache.json diff --git a/test/client-side-encryption/spec/legacy/fle2v2-Rangev2-Compact.json b/test/client-side-encryption/spec/legacy/fle2v2-Rangev2-Compact.json index bba9f25535..59241927ca 100644 --- a/test/client-side-encryption/spec/legacy/fle2v2-Rangev2-Compact.json +++ b/test/client-side-encryption/spec/legacy/fle2v2-Rangev2-Compact.json @@ -6,8 +6,7 @@ "replicaset", "sharded", "load-balanced" - ], - "serverless": "forbid" + ] } ], "database_name": "default", diff --git a/test/client-side-encryption/spec/legacy/keyCache.json b/test/client-side-encryption/spec/legacy/keyCache.json new file mode 100644 index 0000000000..912ce80020 --- /dev/null +++ b/test/client-side-encryption/spec/legacy/keyCache.json @@ -0,0 +1,270 @@ +{ + "runOn": [ + { + "minServerVersion": "4.1.10" + } + ], + "database_name": "default", + "collection_name": "default", + "data": [], + "json_schema": { + "properties": { + "encrypted_w_altname": { + "encrypt": { + "keyId": "/altname", + "bsonType": "string", + "algorithm": "AEAD_AES_256_CBC_HMAC_SHA_512-Random" + } + }, + "encrypted_string": { + "encrypt": { + "keyId": [ + { + "$binary": { + "base64": "AAAAAAAAAAAAAAAAAAAAAA==", + "subType": "04" + } + } + ], + "bsonType": "string", + "algorithm": "AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic" + } + }, + "random": { + "encrypt": { + "keyId": [ + { + "$binary": { + "base64": "AAAAAAAAAAAAAAAAAAAAAA==", + "subType": "04" + } + } + ], + "bsonType": "string", + "algorithm": "AEAD_AES_256_CBC_HMAC_SHA_512-Random" + } + }, + "encrypted_string_equivalent": { + "encrypt": { + "keyId": [ + { + "$binary": { + "base64": "AAAAAAAAAAAAAAAAAAAAAA==", + "subType": "04" + } + } + ], + "bsonType": "string", + "algorithm": "AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic" + } + } + }, + "bsonType": "object" + }, + "key_vault_data": [ + { + "status": 1, + "_id": { + "$binary": { + "base64": "AAAAAAAAAAAAAAAAAAAAAA==", + "subType": "04" + } + }, + "masterKey": { + "provider": "aws", + "key": "arn:aws:kms:us-east-1:579766882180:key/89fcc2c4-08b0-4bd9-9f25-e30687b580d0", + "region": "us-east-1" + }, + "updateDate": { + "$date": { + "$numberLong": "1552949630483" + } + }, + "keyMaterial": { + "$binary": { + "base64": "AQICAHhQNmWG2CzOm1dq3kWLM+iDUZhEqnhJwH9wZVpuZ94A8gEqnsxXlR51T5EbEVezUqqKAAAAwjCBvwYJKoZIhvcNAQcGoIGxMIGuAgEAMIGoBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDHa4jo6yp0Z18KgbUgIBEIB74sKxWtV8/YHje5lv5THTl0HIbhSwM6EqRlmBiFFatmEWaeMk4tO4xBX65eq670I5TWPSLMzpp8ncGHMmvHqRajNBnmFtbYxN3E3/WjxmdbOOe+OXpnGJPcGsftc7cB2shRfA4lICPnE26+oVNXT6p0Lo20nY5XC7jyCO", + "subType": "00" + } + }, + "creationDate": { + "$date": { + "$numberLong": "1552949630483" + } + }, + "keyAltNames": [ + "altname", + "another_altname" + ] + } + ], + "tests": [ + { + "description": "Insert with deterministic encryption, then find it", + "clientOptions": { + "autoEncryptOpts": { + "kmsProviders": { + "aws": {} + }, + "keyExpirationMS": 1 + } + }, + "operations": [ + { + "name": "insertOne", + "arguments": { + "document": { + "_id": 1, + "encrypted_string": "string0" + } + } + }, + { + "name": "wait", + "object": "testRunner", + "arguments": { + "ms": 50 + } + }, + { + "name": "find", + "arguments": { + "filter": { + "_id": 1 + } + }, + "result": [ + { + "_id": 1, + "encrypted_string": "string0" + } + ] + } + ], + "expectations": [ + { + "command_started_event": { + "command": { + "listCollections": 1, + "filter": { + "name": "default" + } + }, + "command_name": "listCollections" + } + }, + { + "command_started_event": { + "command": { + "find": "datakeys", + "filter": { + "$or": [ + { + "_id": { + "$in": [ + { + "$binary": { + "base64": "AAAAAAAAAAAAAAAAAAAAAA==", + "subType": "04" + } + } + ] + } + }, + { + "keyAltNames": { + "$in": [] + } + } + ] + }, + "$db": "keyvault", + "readConcern": { + "level": "majority" + } + }, + "command_name": "find" + } + }, + { + "command_started_event": { + "command": { + "insert": "default", + "documents": [ + { + "_id": 1, + "encrypted_string": { + "$binary": { + "base64": "AQAAAAAAAAAAAAAAAAAAAAACwj+3zkv2VM+aTfk60RqhXq6a/77WlLwu/BxXFkL7EppGsju/m8f0x5kBDD3EZTtGALGXlym5jnpZAoSIkswHoA==", + "subType": "06" + } + } + } + ], + "ordered": true + }, + "command_name": "insert" + } + }, + { + "command_started_event": { + "command": { + "find": "default", + "filter": { + "_id": 1 + } + }, + "command_name": "find" + } + }, + { + "command_started_event": { + "command": { + "find": "datakeys", + "filter": { + "$or": [ + { + "_id": { + "$in": [ + { + "$binary": { + "base64": "AAAAAAAAAAAAAAAAAAAAAA==", + "subType": "04" + } + } + ] + } + }, + { + "keyAltNames": { + "$in": [] + } + } + ] + }, + "$db": "keyvault", + "readConcern": { + "level": "majority" + } + }, + "command_name": "find" + } + } + ], + "outcome": { + "collection": { + "data": [ + { + "_id": 1, + "encrypted_string": { + "$binary": { + "base64": "AQAAAAAAAAAAAAAAAAAAAAACwj+3zkv2VM+aTfk60RqhXq6a/77WlLwu/BxXFkL7EppGsju/m8f0x5kBDD3EZTtGALGXlym5jnpZAoSIkswHoA==", + "subType": "06" + } + } + } + ] + } + } + } + ] +} diff --git a/test/client-side-encryption/spec/legacy/timeoutMS.json b/test/client-side-encryption/spec/legacy/timeoutMS.json index 8411306224..b667767cfc 100644 --- a/test/client-side-encryption/spec/legacy/timeoutMS.json +++ b/test/client-side-encryption/spec/legacy/timeoutMS.json @@ -110,7 +110,7 @@ "listCollections" ], "blockConnection": true, - "blockTimeMS": 600 + "blockTimeMS": 60 } }, "clientOptions": { @@ -119,7 +119,7 @@ "aws": {} } }, - "timeoutMS": 500 + "timeoutMS": 50 }, "operations": [ { diff --git a/test/client-side-encryption/spec/unified/keyCache.json b/test/client-side-encryption/spec/unified/keyCache.json new file mode 100644 index 0000000000..a39701e286 --- /dev/null +++ b/test/client-side-encryption/spec/unified/keyCache.json @@ -0,0 +1,198 @@ +{ + "description": "keyCache-explicit", + "schemaVersion": "1.22", + "runOnRequirements": [ + { + "csfle": true + } + ], + "createEntities": [ + { + "client": { + "id": "client0", + "observeEvents": [ + "commandStartedEvent" + ] + } + }, + { + "clientEncryption": { + "id": "clientEncryption0", + "clientEncryptionOpts": { + "keyVaultClient": "client0", + "keyVaultNamespace": "keyvault.datakeys", + "kmsProviders": { + "local": { + "key": "OCTP9uKPPmvuqpHlqq83gPk4U6rUPxKVRRyVtrjFmVjdoa4Xzm1SzUbr7aIhNI42czkUBmrCtZKF31eaaJnxEBkqf0RFukA9Mo3NEHQWgAQ2cn9duOcRbaFUQo2z0/rB" + } + }, + "keyExpirationMS": 1 + } + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "keyvault" + } + }, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": "datakeys" + } + } + ], + "initialData": [ + { + "databaseName": "keyvault", + "collectionName": "datakeys", + "documents": [ + { + "_id": { + "$binary": { + "base64": "a+YWzdygTAG62/cNUkqZiQ==", + "subType": "04" + } + }, + "keyAltNames": [], + "keyMaterial": { + "$binary": { + "base64": "iocBkhO3YBokiJ+FtxDTS71/qKXQ7tSWhWbcnFTXBcMjarsepvALeJ5li+SdUd9ePuatjidxAdMo7vh1V2ZESLMkQWdpPJ9PaJjA67gKQKbbbB4Ik5F2uKjULvrMBnFNVRMup4JNUwWFQJpqbfMveXnUVcD06+pUpAkml/f+DSXrV3e5rxciiNVtz03dAG8wJrsKsFXWj6vTjFhsfknyBA==", + "subType": "00" + } + }, + "creationDate": { + "$date": { + "$numberLong": "1552949630483" + } + }, + "updateDate": { + "$date": { + "$numberLong": "1552949630483" + } + }, + "status": { + "$numberInt": "0" + }, + "masterKey": { + "provider": "local" + } + } + ] + } + ], + "tests": [ + { + "description": "decrypt, wait, and decrypt again", + "operations": [ + { + "name": "decrypt", + "object": "clientEncryption0", + "arguments": { + "value": { + "$binary": { + "base64": "AWvmFs3coEwButv3DVJKmYkCJ6lUzRX9R28WNlw5uyndb+8gurA+p8q14s7GZ04K2ZvghieRlAr5UwZbow3PMq27u5EIhDDczwBFcbdP1amllw==", + "subType": "06" + } + } + }, + "expectResult": "foobar" + }, + { + "name": "wait", + "object": "testRunner", + "arguments": { + "ms": 50 + } + }, + { + "name": "decrypt", + "object": "clientEncryption0", + "arguments": { + "value": { + "$binary": { + "base64": "AWvmFs3coEwButv3DVJKmYkCJ6lUzRX9R28WNlw5uyndb+8gurA+p8q14s7GZ04K2ZvghieRlAr5UwZbow3PMq27u5EIhDDczwBFcbdP1amllw==", + "subType": "06" + } + } + }, + "expectResult": "foobar" + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "command": { + "find": "datakeys", + "filter": { + "$or": [ + { + "_id": { + "$in": [ + { + "$binary": { + "base64": "a+YWzdygTAG62/cNUkqZiQ==", + "subType": "04" + } + } + ] + } + }, + { + "keyAltNames": { + "$in": [] + } + } + ] + }, + "$db": "keyvault", + "readConcern": { + "level": "majority" + } + } + } + }, + { + "commandStartedEvent": { + "command": { + "find": "datakeys", + "filter": { + "$or": [ + { + "_id": { + "$in": [ + { + "$binary": { + "base64": "a+YWzdygTAG62/cNUkqZiQ==", + "subType": "04" + } + } + ] + } + }, + { + "keyAltNames": { + "$in": [] + } + } + ] + }, + "$db": "keyvault", + "readConcern": { + "level": "majority" + } + } + } + } + ] + } + ] + } + ] +} From 03a2cf140a326a8720846d08e6fd93df58d1fece Mon Sep 17 00:00:00 2001 From: Shane Harvey Date: Thu, 6 Mar 2025 15:36:09 -0800 Subject: [PATCH 03/10] PYTHON-4580 Add changelog --- doc/changelog.rst | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/doc/changelog.rst b/doc/changelog.rst index cf5d5e8ff7..21e86953c6 100644 --- a/doc/changelog.rst +++ b/doc/changelog.rst @@ -1,7 +1,23 @@ Changelog ========= -Changes in Version 4.11.2 (YYYY/MM/DD) +Changes in Version 4.12.0 (YYYY/MM/DD) +-------------------------------------- + +PyMongo 4.12 brings a number of changes including: + +- Support for configuring DEK cache lifetime via the ``key_expiration_ms`` argument to + :class:`~pymongo.encryption_options.AutoEncryptionOpts`. + +Issues Resolved +............... + +See the `PyMongo 4.12 release notes in JIRA`_ for the list of resolved issues +in this release. + +.. _PyMongo 4.12 release notes in JIRA: https://jira.mongodb.org/secure/ReleaseNote.jspa?projectId=10004&version=41916 + +Changes in Version 4.11.2 (2025/03/05) -------------------------------------- Version 4.11.2 is a bug fix release. From db5cee8deeae769548da8f9b9602ec46d618fc76 Mon Sep 17 00:00:00 2001 From: Shane Harvey Date: Thu, 6 Mar 2025 15:39:48 -0800 Subject: [PATCH 04/10] TEMP REVERT ME: update pymongocrypt_source to test branch --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index ca76cfa2c0..a65b368334 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -59,7 +59,7 @@ mockupdb = [ "mockupdb@git+https://github.com/mongodb-labs/mongo-mockup-db@master" ] pymongocrypt_source = [ - "pymongocrypt@git+https://github.com/mongodb/libmongocrypt@master#subdirectory=bindings/python" + "pymongocrypt@git+https://github.com/ShaneHarvey/libmongocrypt@PYTHON-5191#subdirectory=bindings/python" ] perf = ["simplejson"] typing = [ From 9c00d542f81b2a2ad0bef76a22832a79725c0511 Mon Sep 17 00:00:00 2001 From: Shane Harvey Date: Thu, 6 Mar 2025 16:02:44 -0800 Subject: [PATCH 05/10] PYTHON-4580 Fix tests --- test/asynchronous/unified_format.py | 2 +- test/asynchronous/utils_spec_runner.py | 5 +++++ test/unified_format.py | 2 +- test/utils_spec_runner.py | 5 +++++ 4 files changed, 12 insertions(+), 2 deletions(-) diff --git a/test/asynchronous/unified_format.py b/test/asynchronous/unified_format.py index d4c3d40d20..f1807f6f59 100644 --- a/test/asynchronous/unified_format.py +++ b/test/asynchronous/unified_format.py @@ -439,7 +439,7 @@ class UnifiedSpecTestMixinV1(AsyncIntegrationTest): a class attribute ``TEST_SPEC``. """ - SCHEMA_VERSION = Version.from_string("1.21") + SCHEMA_VERSION = Version.from_string("1.22") RUN_ON_LOAD_BALANCER = True RUN_ON_SERVERLESS = True TEST_SPEC: Any diff --git a/test/asynchronous/utils_spec_runner.py b/test/asynchronous/utils_spec_runner.py index 7530ba36a7..14b9ae70ae 100644 --- a/test/asynchronous/utils_spec_runner.py +++ b/test/asynchronous/utils_spec_runner.py @@ -18,6 +18,7 @@ import asyncio import functools import os +import time import unittest from asyncio import iscoroutinefunction from collections import abc @@ -314,6 +315,10 @@ async def assert_index_not_exists(self, database, collection, index): coll = self.client[database][collection] self.assertNotIn(index, [doc["name"] async for doc in await coll.list_indexes()]) + async def wait(self, spec): + """Run the "wait" test operation.""" + await asyncio.sleep(spec["ms"] / 1000.0) + def assertErrorLabelsContain(self, exc, expected_labels): labels = [l for l in expected_labels if exc.has_error_label(l)] self.assertEqual(labels, expected_labels) diff --git a/test/unified_format.py b/test/unified_format.py index 293fbd97ca..7e505912f9 100644 --- a/test/unified_format.py +++ b/test/unified_format.py @@ -438,7 +438,7 @@ class UnifiedSpecTestMixinV1(IntegrationTest): a class attribute ``TEST_SPEC``. """ - SCHEMA_VERSION = Version.from_string("1.21") + SCHEMA_VERSION = Version.from_string("1.22") RUN_ON_LOAD_BALANCER = True RUN_ON_SERVERLESS = True TEST_SPEC: Any diff --git a/test/utils_spec_runner.py b/test/utils_spec_runner.py index ac4031e821..fb148e6733 100644 --- a/test/utils_spec_runner.py +++ b/test/utils_spec_runner.py @@ -18,6 +18,7 @@ import asyncio import functools import os +import time import unittest from asyncio import iscoroutinefunction from collections import abc @@ -314,6 +315,10 @@ def assert_index_not_exists(self, database, collection, index): coll = self.client[database][collection] self.assertNotIn(index, [doc["name"] for doc in coll.list_indexes()]) + def wait(self, spec): + """Run the "wait" test operation.""" + time.sleep(spec["ms"] / 1000.0) + def assertErrorLabelsContain(self, exc, expected_labels): labels = [l for l in expected_labels if exc.has_error_label(l)] self.assertEqual(labels, expected_labels) From 772ccebe6de9a7b90ce470a5ba9bc085d3b2cf2d Mon Sep 17 00:00:00 2001 From: Shane Harvey Date: Thu, 6 Mar 2025 16:23:52 -0800 Subject: [PATCH 06/10] PYTHON-4580 Fix wait() method --- test/asynchronous/utils_spec_runner.py | 4 ++-- test/utils_spec_runner.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/test/asynchronous/utils_spec_runner.py b/test/asynchronous/utils_spec_runner.py index 14b9ae70ae..f1c6deb690 100644 --- a/test/asynchronous/utils_spec_runner.py +++ b/test/asynchronous/utils_spec_runner.py @@ -315,9 +315,9 @@ async def assert_index_not_exists(self, database, collection, index): coll = self.client[database][collection] self.assertNotIn(index, [doc["name"] async for doc in await coll.list_indexes()]) - async def wait(self, spec): + async def wait(self, ms): """Run the "wait" test operation.""" - await asyncio.sleep(spec["ms"] / 1000.0) + await asyncio.sleep(ms / 1000.0) def assertErrorLabelsContain(self, exc, expected_labels): labels = [l for l in expected_labels if exc.has_error_label(l)] diff --git a/test/utils_spec_runner.py b/test/utils_spec_runner.py index fb148e6733..fe0ba6eb44 100644 --- a/test/utils_spec_runner.py +++ b/test/utils_spec_runner.py @@ -315,9 +315,9 @@ def assert_index_not_exists(self, database, collection, index): coll = self.client[database][collection] self.assertNotIn(index, [doc["name"] for doc in coll.list_indexes()]) - def wait(self, spec): + def wait(self, ms): """Run the "wait" test operation.""" - time.sleep(spec["ms"] / 1000.0) + time.sleep(ms / 1000.0) def assertErrorLabelsContain(self, exc, expected_labels): labels = [l for l in expected_labels if exc.has_error_label(l)] From 76fed07f7f864f65c83e9ec5528a9afe6d4bf09d Mon Sep 17 00:00:00 2001 From: Shane Harvey Date: Thu, 6 Mar 2025 16:37:19 -0800 Subject: [PATCH 07/10] PYTHON-4580 Pass key_expiration_ms to clientEncryption in tests --- test/asynchronous/unified_format.py | 1 + test/unified_format.py | 1 + 2 files changed, 2 insertions(+) diff --git a/test/asynchronous/unified_format.py b/test/asynchronous/unified_format.py index f1807f6f59..ce0b9979e2 100644 --- a/test/asynchronous/unified_format.py +++ b/test/asynchronous/unified_format.py @@ -378,6 +378,7 @@ async def drop(self: AsyncGridFSBucket, *args: Any, **kwargs: Any) -> None: opts["key_vault_client"], DEFAULT_CODEC_OPTIONS, opts.get("kms_tls_options", kms_tls_options), + opts.get("key_expiration_ms"), ) return elif entity_type == "thread": diff --git a/test/unified_format.py b/test/unified_format.py index 7e505912f9..682a6105f3 100644 --- a/test/unified_format.py +++ b/test/unified_format.py @@ -377,6 +377,7 @@ def drop(self: GridFSBucket, *args: Any, **kwargs: Any) -> None: opts["key_vault_client"], DEFAULT_CODEC_OPTIONS, opts.get("kms_tls_options", kms_tls_options), + opts.get("key_expiration_ms"), ) return elif entity_type == "thread": From bbe42fabde7a7c810bc686df85f8ddd8d3c42d4d Mon Sep 17 00:00:00 2001 From: Shane Harvey Date: Fri, 7 Mar 2025 16:00:56 -0800 Subject: [PATCH 08/10] PYTHON-4580 Compat with pymongocrypt <1.13 --- pymongo/asynchronous/encryption.py | 9 ++++----- pymongo/synchronous/encryption.py | 9 ++++----- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/pymongo/asynchronous/encryption.py b/pymongo/asynchronous/encryption.py index 07d62a8a45..ef8d817b2c 100644 --- a/pymongo/asynchronous/encryption.py +++ b/pymongo/asynchronous/encryption.py @@ -548,11 +548,10 @@ class QueryType(str, enum.Enum): def _create_mongocrypt_options(**kwargs: Any) -> MongoCryptOptions: - opts = MongoCryptOptions(**kwargs) - # Opt into range V2 encryption. - if hasattr(opts, "enable_range_v2"): - opts.enable_range_v2 = True - return opts + # For compat with pymongocrypt <1.13, avoid setting the default key_expiration_ms. + if kwargs.get("key_expiration_ms") is None: + kwargs.pop("key_expiration_ms", None) + return MongoCryptOptions(**kwargs) class AsyncClientEncryption(Generic[_DocumentType]): diff --git a/pymongo/synchronous/encryption.py b/pymongo/synchronous/encryption.py index 71fa9fb36b..a97534ed41 100644 --- a/pymongo/synchronous/encryption.py +++ b/pymongo/synchronous/encryption.py @@ -545,11 +545,10 @@ class QueryType(str, enum.Enum): def _create_mongocrypt_options(**kwargs: Any) -> MongoCryptOptions: - opts = MongoCryptOptions(**kwargs) - # Opt into range V2 encryption. - if hasattr(opts, "enable_range_v2"): - opts.enable_range_v2 = True - return opts + # For compat with pymongocrypt <1.13, avoid setting the default key_expiration_ms. + if kwargs.get("key_expiration_ms") is None: + kwargs.pop("key_expiration_ms", None) + return MongoCryptOptions(**kwargs) class ClientEncryption(Generic[_DocumentType]): From 20b12e0a1909ba627e538a067328aa77b19b2f05 Mon Sep 17 00:00:00 2001 From: Shane Harvey Date: Mon, 10 Mar 2025 13:30:59 -0700 Subject: [PATCH 09/10] Revert "TEMP REVERT ME: update pymongocrypt_source to test branch" This reverts commit db5cee8deeae769548da8f9b9602ec46d618fc76. --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index a65b368334..ca76cfa2c0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -59,7 +59,7 @@ mockupdb = [ "mockupdb@git+https://github.com/mongodb-labs/mongo-mockup-db@master" ] pymongocrypt_source = [ - "pymongocrypt@git+https://github.com/ShaneHarvey/libmongocrypt@PYTHON-5191#subdirectory=bindings/python" + "pymongocrypt@git+https://github.com/mongodb/libmongocrypt@master#subdirectory=bindings/python" ] perf = ["simplejson"] typing = [ From a7319ffb966c24d654711de14f0f9f10d61e2fbb Mon Sep 17 00:00:00 2001 From: Shane Harvey Date: Mon, 10 Mar 2025 16:36:28 -0700 Subject: [PATCH 10/10] PYTHON-4580 Update pymongocrypt --- uv.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uv.lock b/uv.lock index a2e951e76c..8b5d592dc0 100644 --- a/uv.lock +++ b/uv.lock @@ -1133,7 +1133,7 @@ wheels = [ [[package]] name = "pymongocrypt" version = "1.13.0.dev0" -source = { git = "https://github.com/mongodb/libmongocrypt?subdirectory=bindings%2Fpython&rev=master#90476d5db7737bab2ce1c198df5671a12dbaae1a" } +source = { git = "https://github.com/mongodb/libmongocrypt?subdirectory=bindings%2Fpython&rev=master#1e96c283162aa7789cf01f99f211e0ace8e6d49f" } dependencies = [ { name = "cffi" }, { name = "cryptography" },