From 68f176a08434b1b354600879259a977a27ff23ca Mon Sep 17 00:00:00 2001 From: Pauline Vos Date: Thu, 14 Aug 2025 17:12:33 +0200 Subject: [PATCH 1/2] Revert "Test against stable 2.1 version of the extension (#1698)" This reverts commit e76d9f5e4c95976ce3a600682f293b1c5e6c6f58. --- .../generated/build/build-extension.yml | 44 +++++++++++++++++++ .../templates/build/build-extension.yml | 11 +++++ .github/workflows/coding-standards.yml | 4 +- .github/workflows/generator.yml | 4 +- .github/workflows/static-analysis.yml | 4 +- .github/workflows/tests.yml | 4 +- 6 files changed, 67 insertions(+), 4 deletions(-) diff --git a/.evergreen/config/generated/build/build-extension.yml b/.evergreen/config/generated/build/build-extension.yml index 48e6b5a1f..48c7c26a3 100644 --- a/.evergreen/config/generated/build/build-extension.yml +++ b/.evergreen/config/generated/build/build-extension.yml @@ -7,6 +7,9 @@ tasks: vars: PHP_VERSION: "8.4" - func: "compile extension" + # TODO: Remove vars to switch to latest stable version when 2.1.0 is releeased + vars: + EXTENSION_BRANCH: "v2.x" - func: "upload extension" - name: "build-php-8.4-lowest" tags: ["build", "php8.4", "lowest", "pr", "tag"] @@ -15,6 +18,10 @@ tasks: vars: PHP_VERSION: "8.4" - func: "compile extension" + vars: + # TODO: Switch to 2.1.0 when it is released + # EXTENSION_VERSION: "2.0.0" + EXTENSION_BRANCH: "v2.x" - func: "upload extension" - name: "build-php-8.4-next-stable" tags: ["build", "php8.4", "next-stable", "pr", "tag"] @@ -23,6 +30,10 @@ tasks: vars: PHP_VERSION: "8.4" - func: "compile extension" + vars: + # TODO: Switch to v2.1 when 2.1.0 is released + # EXTENSION_VERSION: "v2.1" + EXTENSION_BRANCH: "v2.x" - func: "upload extension" - name: "build-php-8.4-next-minor" tags: ["build", "php8.4", "next-minor"] @@ -41,6 +52,9 @@ tasks: vars: PHP_VERSION: "8.3" - func: "compile extension" + # TODO: Remove vars to switch to latest stable version when 2.1.0 is releeased + vars: + EXTENSION_BRANCH: "v2.x" - func: "upload extension" - name: "build-php-8.3-lowest" tags: ["build", "php8.3", "lowest", "pr", "tag"] @@ -49,6 +63,10 @@ tasks: vars: PHP_VERSION: "8.3" - func: "compile extension" + vars: + # TODO: Switch to 2.1.0 when it is released + # EXTENSION_VERSION: "2.0.0" + EXTENSION_BRANCH: "v2.x" - func: "upload extension" - name: "build-php-8.3-next-stable" tags: ["build", "php8.3", "next-stable", "pr", "tag"] @@ -57,6 +75,10 @@ tasks: vars: PHP_VERSION: "8.3" - func: "compile extension" + vars: + # TODO: Switch to v2.1 when 2.1.0 is released + # EXTENSION_VERSION: "v2.1" + EXTENSION_BRANCH: "v2.x" - func: "upload extension" - name: "build-php-8.3-next-minor" tags: ["build", "php8.3", "next-minor"] @@ -75,6 +97,9 @@ tasks: vars: PHP_VERSION: "8.2" - func: "compile extension" + # TODO: Remove vars to switch to latest stable version when 2.1.0 is releeased + vars: + EXTENSION_BRANCH: "v2.x" - func: "upload extension" - name: "build-php-8.2-lowest" tags: ["build", "php8.2", "lowest", "pr", "tag"] @@ -83,6 +108,10 @@ tasks: vars: PHP_VERSION: "8.2" - func: "compile extension" + vars: + # TODO: Switch to 2.1.0 when it is released + # EXTENSION_VERSION: "2.0.0" + EXTENSION_BRANCH: "v2.x" - func: "upload extension" - name: "build-php-8.2-next-stable" tags: ["build", "php8.2", "next-stable", "pr", "tag"] @@ -91,6 +120,10 @@ tasks: vars: PHP_VERSION: "8.2" - func: "compile extension" + vars: + # TODO: Switch to v2.1 when 2.1.0 is released + # EXTENSION_VERSION: "v2.1" + EXTENSION_BRANCH: "v2.x" - func: "upload extension" - name: "build-php-8.2-next-minor" tags: ["build", "php8.2", "next-minor"] @@ -109,6 +142,9 @@ tasks: vars: PHP_VERSION: "8.1" - func: "compile extension" + # TODO: Remove vars to switch to latest stable version when 2.1.0 is releeased + vars: + EXTENSION_BRANCH: "v2.x" - func: "upload extension" - name: "build-php-8.1-lowest" tags: ["build", "php8.1", "lowest", "pr", "tag"] @@ -117,6 +153,10 @@ tasks: vars: PHP_VERSION: "8.1" - func: "compile extension" + vars: + # TODO: Switch to 2.1.0 when it is released + # EXTENSION_VERSION: "2.0.0" + EXTENSION_BRANCH: "v2.x" - func: "upload extension" - name: "build-php-8.1-next-stable" tags: ["build", "php8.1", "next-stable", "pr", "tag"] @@ -125,6 +165,10 @@ tasks: vars: PHP_VERSION: "8.1" - func: "compile extension" + vars: + # TODO: Switch to v2.1 when 2.1.0 is released + # EXTENSION_VERSION: "v2.1" + EXTENSION_BRANCH: "v2.x" - func: "upload extension" - name: "build-php-8.1-next-minor" tags: ["build", "php8.1", "next-minor"] diff --git a/.evergreen/config/templates/build/build-extension.yml b/.evergreen/config/templates/build/build-extension.yml index 39f478cac..c6b1f411c 100644 --- a/.evergreen/config/templates/build/build-extension.yml +++ b/.evergreen/config/templates/build/build-extension.yml @@ -5,6 +5,9 @@ vars: PHP_VERSION: "%phpVersion%" - func: "compile extension" + # TODO: Remove vars to switch to latest stable version when 2.1.0 is releeased + vars: + EXTENSION_BRANCH: "v2.x" - func: "upload extension" - name: "build-php-%phpVersion%-lowest" tags: ["build", "php%phpVersion%", "lowest", "pr", "tag"] @@ -13,6 +16,10 @@ vars: PHP_VERSION: "%phpVersion%" - func: "compile extension" + vars: + # TODO: Switch to 2.1.0 when it is released + # EXTENSION_VERSION: "2.0.0" + EXTENSION_BRANCH: "v2.x" - func: "upload extension" - name: "build-php-%phpVersion%-next-stable" tags: ["build", "php%phpVersion%", "next-stable", "pr", "tag"] @@ -21,6 +28,10 @@ vars: PHP_VERSION: "%phpVersion%" - func: "compile extension" + vars: + # TODO: Switch to v2.1 when 2.1.0 is released + # EXTENSION_VERSION: "v2.1" + EXTENSION_BRANCH: "v2.x" - func: "upload extension" - name: "build-php-%phpVersion%-next-minor" tags: ["build", "php%phpVersion%", "next-minor"] diff --git a/.github/workflows/coding-standards.yml b/.github/workflows/coding-standards.yml index bb6091805..3810a5ded 100644 --- a/.github/workflows/coding-standards.yml +++ b/.github/workflows/coding-standards.yml @@ -13,7 +13,9 @@ on: env: PHP_VERSION: "8.2" - DRIVER_VERSION: "stable" + # TODO: change to "stable" once 2.0.0 is released + # DRIVER_VERSION: "stable" + DRIVER_VERSION: "mongodb/mongo-php-driver@v2.x" jobs: phpcs: diff --git a/.github/workflows/generator.yml b/.github/workflows/generator.yml index e64e8feeb..008f8d84b 100644 --- a/.github/workflows/generator.yml +++ b/.github/workflows/generator.yml @@ -13,7 +13,9 @@ on: env: PHP_VERSION: "8.2" - DRIVER_VERSION: "stable" + # TODO: change to "stable" once 2.0.0 is released + # DRIVER_VERSION: "stable" + DRIVER_VERSION: "mongodb/mongo-php-driver@v2.x" jobs: diff: diff --git a/.github/workflows/static-analysis.yml b/.github/workflows/static-analysis.yml index 3447142a3..cf96ca895 100644 --- a/.github/workflows/static-analysis.yml +++ b/.github/workflows/static-analysis.yml @@ -19,7 +19,9 @@ on: env: PHP_VERSION: "8.2" - DRIVER_VERSION: "stable" + # TODO: change to "stable" once 2.0.0 is released + # DRIVER_VERSION: "stable" + DRIVER_VERSION: "mongodb/mongo-php-driver@v2.x" jobs: psalm: diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 59226167d..17bbfd742 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -12,7 +12,9 @@ on: - "feature/*" env: - DRIVER_VERSION: "stable" + # TODO: change to "stable" once 2.0.0 is released + # DRIVER_VERSION: "stable" + DRIVER_VERSION: "mongodb/mongo-php-driver@v2.x" jobs: phpunit: From d6e08a42969a392e5b68ad5ebac298a863985747 Mon Sep 17 00:00:00 2001 From: Pauline Vos Date: Thu, 14 Aug 2025 16:43:36 +0200 Subject: [PATCH 2/2] Implement prose tests for `$lookup` support As laid out in the spec: https://github.com/mongodb/specifications/commit/527e22d5090ec48bf1e144c45fc831de0f1935f6 --- .../Prose25_LookupTest.php | 372 ++++++++++++++++++ tests/drivers-evergreen-tools | 2 +- tests/specifications | 2 +- 3 files changed, 374 insertions(+), 2 deletions(-) create mode 100644 tests/SpecTests/ClientSideEncryption/Prose25_LookupTest.php diff --git a/tests/SpecTests/ClientSideEncryption/Prose25_LookupTest.php b/tests/SpecTests/ClientSideEncryption/Prose25_LookupTest.php new file mode 100644 index 000000000..6abfbda13 --- /dev/null +++ b/tests/SpecTests/ClientSideEncryption/Prose25_LookupTest.php @@ -0,0 +1,372 @@ +isStandalone()) { + $this->markTestSkipped('Lookup tests require replica sets'); + } + + $this->skipIfServerVersion('<', '7.0.0', 'Lookup encryption tests require MongoDB 7.0 or later'); + + $key1Document = $this->decodeJson(file_get_contents(self::$dataDir . '/key-doc.json')); + $this->key1Id = $key1Document->_id; + + $encryptedClient = $this->getEncryptedClient(); + + // Drop the key vault collection and insert key1Document with a majority write concern + self::insertKeyVaultData($encryptedClient, [$key1Document]); + + $this->refreshCollections($encryptedClient); + } + + private function getEncryptedClient(): Client + { + $autoEncryptionOpts = [ + 'keyVaultNamespace' => 'keyvault.datakeys', + 'kmsProviders' => ['local' => ['key' => new Binary(base64_decode(self::LOCAL_MASTERKEY))]], + ]; + + return self::createTestClient(null, [], [ + 'autoEncryption' => $autoEncryptionOpts, + /* libmongocrypt caches results from listCollections. Use a new + * client in each test to ensure its encryptedFields is applied. */ + 'disableClientPersistence' => true, + ]); + } + + private function refreshCollections(Client $client): void + { + $encryptedDb = $client->getDatabase(self::getDatabaseName()); + $unencryptedDb = self::createTestClient()->getDatabase(self::getDatabaseName()); + + $optionsMap = [ + self::COLL_CSFLE => [ + 'validator' => [ + '$jsonSchema' => $this->decodeJson(file_get_contents(self::$dataDir . '/schema-csfle.json')), + ], + ], + self::COLL_CSFLE2 => [ + 'validator' => [ + '$jsonSchema' => $this->decodeJson(file_get_contents(self::$dataDir . '/schema-csfle2.json')), + ], + ], + self::COLL_QE => [ + 'encryptedFields' => $this->decodeJson(file_get_contents(self::$dataDir . '/schema-qe.json')), + ], + self::COLL_QE2 => [ + 'encryptedFields' => $this->decodeJson(file_get_contents(self::$dataDir . '/schema-qe2.json')), + ], + self::COLL_NO_SCHEMA => [], + self::COLL_NO_SCHEMA2 => [], + ]; + + foreach ($optionsMap as $collectionName => $options) { + $encryptedDb->dropCollection($collectionName); + $encryptedDb->createCollection($collectionName, $options); + + $collection = $unencryptedDb->getCollection($collectionName); + + $result = $encryptedDb->getCollection($collectionName)->insertOne([$collectionName => $collectionName]); + + if ($options) { + $document = $collection->findOne(['_id' => $result->getInsertedId()]); + $this->assertInstanceOf(Binary::class, $document->{$collectionName}); + } + } + } + + private function assertPipelineReturnsSingleDocument(string $collection, array $pipeline, array $expected): void + { + $this->skipIfServerVersion('<', '8.1.0', 'Lookup test case requires server version 8.1.0 or later'); + $this->skipIfClientSideEncryptionIsNotSupported(); + + $cursor = $this + ->getEncryptedClient() + ->getCollection(self::getDatabaseName(), $collection) + ->aggregate($pipeline); + + $cursor->rewind(); + $this->assertMatchesDocument( + $expected, + $cursor->current(), + ); + $this->assertNull($cursor->next()); + } + + public function testCase1_CsfleJoinsNoSchema(): void + { + $pipeline = [ + [ + '$match' => ['csfle' => 'csfle'], + ], + [ + '$lookup' => [ + 'from' => 'no_schema', + 'as' => 'matched', + 'pipeline' => [ + [ + '$match' => ['no_schema' => 'no_schema'], + ], + [ + '$project' => ['_id' => 0], + ], + ], + ], + ], + [ + '$project' => ['_id' => 0], + ], + ]; + $expected = [ + 'csfle' => 'csfle', + 'matched' => [ + ['no_schema' => 'no_schema'], + ], + ]; + + $this->assertPipelineReturnsSingleDocument(self::COLL_CSFLE, $pipeline, $expected); + } + + public function testCase2_QeJoinsNoSchema(): void + { + $pipeline = [ + [ + '$match' => ['qe' => 'qe'], + ], + [ + '$lookup' => [ + 'from' => 'no_schema', + 'as' => 'matched', + 'pipeline' => [ + [ + '$match' => ['no_schema' => 'no_schema'], + ], + [ + '$project' => [ + '_id' => 0, + '__safeContent__' => 0, + ], + ], + ], + ], + ], + [ + '$project' => [ + '_id' => 0, + '__safeContent__' => 0, + ], + ], + ]; + $expected = [ + 'qe' => 'qe', + 'matched' => [ + ['no_schema' => 'no_schema'], + ], + ]; + + $this->assertPipelineReturnsSingleDocument(self::COLL_QE, $pipeline, $expected); + } + + public function testCase3_NoSchemaJoinsCsfle(): void + { + $pipeline = [['$match' => ['no_schema' => 'no_schema']], + [ + '$lookup' => [ + 'from' => 'csfle', + 'as' => 'matched', + 'pipeline' => [ + [ + '$match' => ['csfle' => 'csfle'], + ], + [ + '$project' => ['_id' => 0], + ], + ], + ], + ], + ['$project' => ['_id' => 0]], + ]; + $expected = ['no_schema' => 'no_schema', 'matched' => [['csfle' => 'csfle']]]; + + $this->assertPipelineReturnsSingleDocument(self::COLL_NO_SCHEMA, $pipeline, $expected); + } + + public function testCase4_NoSchemaJoinsQe(): void + { + $pipeline = [ + [ + '$match' => ['no_schema' => 'no_schema'], + ], + [ + '$lookup' => [ + 'from' => 'qe', + 'as' => 'matched', + 'pipeline' => [ + [ + '$match' => ['qe' => 'qe'], + ], + [ + '$project' => [ + '_id' => 0, + '__safeContent__' => 0, + ], + ], + ], + ], + ], + [ + '$project' => ['_id' => 0], + ], + ]; + $expected = [ + 'no_schema' => 'no_schema', + 'matched' => [ + ['qe' => 'qe'], + ], + ]; + + $this->assertPipelineReturnsSingleDocument(self::COLL_NO_SCHEMA, $pipeline, $expected); + } + + public function testCase5_CsfleJoinsCsfle2(): void + { + $pipeline = [ + ['$match' => ['csfle' => 'csfle']], + [ + '$lookup' => [ + 'from' => 'csfle2', + 'as' => 'matched', + 'pipeline' => [ + [ + '$match' => ['csfle2' => 'csfle2'], + ], + [ + '$project' => ['_id' => 0], + ], + ], + ], + ], + ['$project' => ['_id' => 0]], + ]; + $expected = ['csfle' => 'csfle', 'matched' => [['csfle2' => 'csfle2']]]; + + $this->assertPipelineReturnsSingleDocument(self::COLL_CSFLE, $pipeline, $expected); + } + + public function testCase6_QeJoinsQe2(): void + { + $pipeline = [ + ['$match' => ['qe' => 'qe']], + [ + '$lookup' => [ + 'from' => 'qe2', + 'as' => 'matched', + 'pipeline' => [ + [ + '$match' => ['qe2' => 'qe2'], + ], + [ + '$project' => [ + '_id' => 0, + '__safeContent__' => 0, + ], + ], + ], + ], + ], + ['$project' => ['_id' => 0, '__safeContent__' => 0]], + ]; + $expected = ['qe' => 'qe', 'matched' => [['qe2' => 'qe2']]]; + + $this->assertPipelineReturnsSingleDocument(self::COLL_QE, $pipeline, $expected); + } + + public function testCase7_NoSchemaJoinsNoSchema2(): void + { + $pipeline = [ + ['$match' => ['no_schema' => 'no_schema']], + [ + '$lookup' => [ + 'from' => 'no_schema2', + 'as' => 'matched', + 'pipeline' => [ + ['$match' => ['no_schema2' => 'no_schema2']], + ['$project' => ['_id' => 0]], + ], + ], + ], + ['$project' => ['_id' => 0]], + ]; + $expected = ['no_schema' => 'no_schema', 'matched' => [['no_schema2' => 'no_schema2']]]; + + $this->assertPipelineReturnsSingleDocument(self::COLL_NO_SCHEMA, $pipeline, $expected); + } + + public function testCase8_CsfleJoinsQeFails(): void + { + $this->skipIfServerVersion('<', '8.1.0', 'Lookup test case requires server version 8.1.0 or later'); + $this->skipIfClientSideEncryptionIsNotSupported(); + + $this->expectExceptionMessage('not supported'); + + $this->getEncryptedClient() + ->getCollection(self::getDatabaseName(), self::COLL_CSFLE) + ->aggregate([ + [ + '$match' => ['csfle' => 'qe'], + ], + [ + '$lookup' => [ + 'from' => 'qe', + 'as' => 'matched', + 'pipeline' => [ + [ + '$match' => ['qe' => 'qe'], + ], + [ + '$project' => ['_id' => 0], + ], + ], + ], + ], + [ + '$project' => ['_id' => 0], + ], + ]); + } + + public function testCase9_TestErrorWithLessThan8_1(): void + { + $this->markTestSkipped('Depends on PHPC-2616 to determine crypt shared version.'); + } +} diff --git a/tests/drivers-evergreen-tools b/tests/drivers-evergreen-tools index a332144cf..92e5f9336 160000 --- a/tests/drivers-evergreen-tools +++ b/tests/drivers-evergreen-tools @@ -1 +1 @@ -Subproject commit a332144cfc785ab178be3d9d62e645cb79b5f81e +Subproject commit 92e5f93363ed12e91f1ef95a42575f47621365c8 diff --git a/tests/specifications b/tests/specifications index 5b15fd307..4e5d62456 160000 --- a/tests/specifications +++ b/tests/specifications @@ -1 +1 @@ -Subproject commit 5b15fd307b7ec06dbafc57e3c252f7ae3b8c4e46 +Subproject commit 4e5d6245655f30f13e42a15bd340f57f6729bb27