CSHARP-4106: Add ClientEncryption entity and Key Management API operations to Unified Test Format. (#826)

Author: DmitryLukyanov

build.cake

@@ -420,7 +420,7 @@ Task("TestCsfleWithMockedKmsNet472").IsDependentOn("TestCsfleWithMockedKms");
 Task("TestCsfleWithMockedKmsNetStandard20").IsDependentOn("TestCsfleWithMockedKms");
 Task("TestCsfleWithMockedKmsNetStandard21").IsDependentOn("TestCsfleWithMockedKms");
 
-Task("TestMongocryptd")
+Task("TestCsfleWithMongocryptd")
     .IsDependentOn("Build")
     .DoesForEach(
         items: GetFiles("./**/*.Tests.csproj"),
@@ -443,9 +443,9 @@ Task("TestMongocryptd")
         );
     });
 
-Task("TestMongocryptdNet472").IsDependentOn("TestMongocryptd");
-Task("TestMongocryptdNetStandard20").IsDependentOn("TestMongocryptd");
-Task("TestMongocryptdNetStandard21").IsDependentOn("TestMongocryptd");
+Task("TestCsfleWithMongocryptdNet472").IsDependentOn("TestCsfleWithMongocryptd");
+Task("TestCsfleWithMongocryptdNetStandard20").IsDependentOn("TestCsfleWithMongocryptd");
+Task("TestCsfleWithMongocryptdNetStandard21").IsDependentOn("TestCsfleWithMongocryptd");
 
 Task("Docs")
     .IsDependentOn("ApiDocs")

evergreen/evergreen.yml

@@ -47,9 +47,12 @@ functions:
 
            export DRIVERS_TOOLS="$(pwd)/../drivers-tools"
 
-           # Python has cygwin path problems on Windows. Detect prospective mongo-orchestration home directory
            if [ "Windows_NT" = "$OS" ]; then # Magic variable in cygwin
+              # Python has cygwin path problems on Windows. Detect prospective mongo-orchestration home directory
               export DRIVERS_TOOLS=$(cygpath -m $DRIVERS_TOOLS)
+           else
+              # non windows OSs don't have dotnet in the PATH
+              export PATH=$PATH:/usr/share/dotnet
            fi
 
            export MONGO_ORCHESTRATION_HOME="$DRIVERS_TOOLS/.evergreen/orchestration"
@@ -80,6 +83,21 @@ functions:
            # See what we've done
            cat expansion.yml
 
+           # Add CSFLE variables that shouldn't be output to the logs
+           cat <<EOT >> expansion.yml
+           PREPARE_CSFLE: |
+              set +o xtrace # Disable tracing.
+              export FLE_AWS_ACCESS_KEY_ID=${FLE_AWS_ACCESS_KEY_ID}
+              export FLE_AWS_SECRET_ACCESS_KEY=${FLE_AWS_SECRET_ACCESS_KEY}
+              export FLE_AZURE_TENANT_ID=${FLE_AZURE_TENANT_ID}
+              export FLE_AZURE_CLIENT_ID=${FLE_AZURE_CLIENT_ID}
+              export FLE_AZURE_CLIENT_SECRET=${FLE_AZURE_CLIENT_SECRET}
+              export FLE_GCP_EMAIL=${FLE_GCP_EMAIL}
+              export FLE_GCP_PRIVATE_KEY=${FLE_GCP_PRIVATE_KEY} 
+              set -o xtrace # Enable tracing.
+           EOT
+           # Do not output expansion.yml contents after this point
+
     # Load the expansion file to make an evergreen variable with the current unique version
     - command: expansions.update
       params:
@@ -282,13 +300,7 @@ functions:
         working_dir: mongo-csharp-driver
         script: |
           set +x
-          export FLE_AWS_ACCESS_KEY_ID=${FLE_AWS_ACCESS_KEY_ID}
-          export FLE_AWS_SECRET_ACCESS_KEY=${FLE_AWS_SECRET_ACCESS_KEY}
-          export FLE_AZURE_TENANT_ID=${FLE_AZURE_TENANT_ID}
-          export FLE_AZURE_CLIENT_ID=${FLE_AZURE_CLIENT_ID}
-          export FLE_AZURE_CLIENT_SECRET=${FLE_AZURE_CLIENT_SECRET}
-          export FLE_GCP_EMAIL=${FLE_GCP_EMAIL}
-          export FLE_GCP_PRIVATE_KEY=${FLE_GCP_PRIVATE_KEY}
+          ${PREPARE_CSFLE}
           . ./evergreen/set-virtualenv.sh
           . ./evergreen/set-temp-fle-aws-creds.sh
           ${PREPARE_SHELL}
@@ -310,23 +322,19 @@ functions:
           OS=${OS} \
             evergreen/cleanup-test-resources.sh
 
-  run-csfle-tests-with-mocked-kms:
+  run-csfle-with-mocked-kms-tests:
     - command: shell.exec
       type: test
       params:
         working_dir: "mongo-csharp-driver"
         script: |
           set +x
-          export FLE_AWS_ACCESS_KEY_ID=${FLE_AWS_ACCESS_KEY_ID}
-          export FLE_AWS_SECRET_ACCESS_KEY=${FLE_AWS_SECRET_ACCESS_KEY}
-          export FLE_AZURE_TENANT_ID=${FLE_AZURE_TENANT_ID}
-          export FLE_AZURE_CLIENT_ID=${FLE_AZURE_CLIENT_ID}
-          export FLE_AZURE_CLIENT_SECRET=${FLE_AZURE_CLIENT_SECRET}
-          export FLE_GCP_EMAIL=${FLE_GCP_EMAIL}
-          export FLE_GCP_PRIVATE_KEY=${FLE_GCP_PRIVATE_KEY}
+          ${PREPARE_CSFLE}
           export KMS_MOCK_SERVERS_ENABLED=true
           ${PREPARE_SHELL}
           set +o xtrace
+          OS=${OS} \
+            . ./evergreen/fetch-crypt_shared-library.sh          
           OS=${OS} \
             evergreen/add-ca-certs.sh
           AUTH=${AUTH} \
@@ -341,20 +349,14 @@ functions:
           OS=${OS} \
             evergreen/cleanup-test-resources.sh
 
-  run-mongocryptd-tests:
+  run-csfle-with-mongocryptd-tests:
     - command: shell.exec
       type: test
       params:
         working_dir: mongo-csharp-driver
         script: |
           set +x
-          export FLE_AWS_ACCESS_KEY_ID=${FLE_AWS_ACCESS_KEY_ID}
-          export FLE_AWS_SECRET_ACCESS_KEY=${FLE_AWS_SECRET_ACCESS_KEY}
-          export FLE_AZURE_TENANT_ID=${FLE_AZURE_TENANT_ID}
-          export FLE_AZURE_CLIENT_ID=${FLE_AZURE_CLIENT_ID}
-          export FLE_AZURE_CLIENT_SECRET=${FLE_AZURE_CLIENT_SECRET}
-          export FLE_GCP_EMAIL=${FLE_GCP_EMAIL}
-          export FLE_GCP_PRIVATE_KEY=${FLE_GCP_PRIVATE_KEY}
+          ${PREPARE_CSFLE}
           . ./evergreen/set-virtualenv.sh
           . ./evergreen/set-temp-fle-aws-creds.sh
           ${PREPARE_SHELL}
@@ -368,8 +370,7 @@ functions:
           COMPRESSOR=${COMPRESSOR} \
           CLIENT_PEM=${DRIVERS_TOOLS}/.evergreen/x509gen/client.pem \
           REQUIRE_API_VERSION=${REQUIRE_API_VERSION} \
-          TEST_MONGOCRYPTD="true" \
-          TARGET="TestMongocryptd" \
+          TARGET="TestCsfleWithMongocryptd" \
           FRAMEWORK=${FRAMEWORK} \
             evergreen/run-tests.sh
           echo "Skipping certificate removal..."
@@ -872,51 +873,51 @@ tasks:
           vars:
             FRAMEWORK: netstandard21
 
-    - name: test-mongocryptd-net472
+    - name: test-csfle-with-mongocryptd-net472
       commands:
         - func: bootstrap-mongo-orchestration
-        - func: run-mongocryptd-tests
+        - func: run-csfle-with-mongocryptd-tests
           vars:
             FRAMEWORK: net472
 
-    - name: test-mongocryptd-netstandard20
+    - name: test-csfle-with-mongocryptd-netstandard20
       commands:
         - func: bootstrap-mongo-orchestration
-        - func: run-mongocryptd-tests
+        - func: run-csfle-with-mongocryptd-tests
           vars:
             FRAMEWORK: netstandard20
 
-    - name: test-mongocryptd-netstandard21
+    - name: test-csfle-with-mongocryptd-netstandard21
       commands:
         - func: bootstrap-mongo-orchestration
-        - func: run-mongocryptd-tests
+        - func: run-csfle-with-mongocryptd-tests
           vars:
             FRAMEWORK: netstandard21
 
-    - name: test-kms-tls-mocked-net472
+    - name: test-csfle-with-mocked-kms-tls-net472
       commands:
         - func: start-kms-mock-servers
         - func: start-kms-kmip-server
         - func: bootstrap-mongo-orchestration
-        - func: run-csfle-tests-with-mocked-kms
+        - func: run-csfle-with-mocked-kms-tests
           vars:
             FRAMEWORK: net472
 
-    - name: test-kms-tls-mocked-netstandard20
+    - name: test-csfle-with-mocked-kms-tls-netstandard20
       commands:
         - func: start-kms-mock-servers
         - func: start-kms-kmip-server
         - func: bootstrap-mongo-orchestration
-        - func: run-csfle-tests-with-mocked-kms
+        - func: run-csfle-with-mocked-kms-tests
           vars:
             FRAMEWORK: netstandard20
 
-    - name: test-kms-tls-mocked-netstandard21
+    - name: test-csfle-with-mocked-kms-tls-netstandard21
       commands:
         - func: start-kms-mock-servers
         - func: start-kms-kmip-server
         - func: bootstrap-mongo-orchestration
-        - func: run-csfle-tests-with-mocked-kms
+        - func: run-csfle-with-mocked-kms-tests
           vars:
             FRAMEWORK: netstandard21
 
@@ -1721,40 +1722,40 @@ buildvariants:
   matrix_spec: { os: "windows-64", ssl: "nossl", version: [ "5.0", "6.0", "rapid", "latest" ], topology: ["standalone"] }
   display_name: "CSFLE Mocked KMS ${version} ${os}"
   tasks:
-    - name: test-kms-tls-mocked-net472
-    - name: test-kms-tls-mocked-netstandard20
-    - name: test-kms-tls-mocked-netstandard21
+    - name: test-csfle-with-mocked-kms-tls-net472
+    - name: test-csfle-with-mocked-kms-tls-netstandard20
+    - name: test-csfle-with-mocked-kms-tls-netstandard21
 
 - matrix_name: "csfle-with-mocked-kms-tests-linux"
   matrix_spec: { os: "ubuntu-1804", ssl: "nossl", version: [ "5.0", "6.0", "rapid", "latest" ], topology: ["standalone"] }
   display_name: "CSFLE Mocked KMS ${version} ${os}"
   tasks:
-    - name: test-kms-tls-mocked-netstandard20
-    - name: test-kms-tls-mocked-netstandard21
+    - name: test-csfle-with-mocked-kms-tls-netstandard20
+    - name: test-csfle-with-mocked-kms-tls-netstandard21
 
 - matrix_name: "csfle-with-mocked-kms-tests-macOS"
   matrix_spec: { os: "macos-1015", ssl: "nossl", version: [ "5.0", "6.0", "rapid", "latest" ], topology: ["standalone"] }
   display_name: "CSFLE Mocked KMS ${version} ${os}"
   tasks:
-    - name: test-kms-tls-mocked-netstandard21
+    - name: test-csfle-with-mocked-kms-tls-netstandard21
 
-- matrix_name: "csfle1-windows"
+- matrix_name: "csfle-with-mongocryptd-windows"
   matrix_spec: { os: "windows-64", ssl: "nossl", version: [ "4.2", "4.4", "5.0", "6.0", "latest" ], topology: ["replicaset"] }
-  display_name: "mongocryptd ${version} ${os}"
+  display_name: "CSFLE with mongocryptd ${version} ${os}"
   tasks:
-    - name: test-mongocryptd-net472
-    - name: test-mongocryptd-netstandard20
-    - name: test-mongocryptd-netstandard21
+    - name: test-csfle-with-mongocryptd-net472
+    - name: test-csfle-with-mongocryptd-netstandard20
+    - name: test-csfle-with-mongocryptd-netstandard21
 
-- matrix_name: "csfle1-linux"
+- matrix_name: "csfle-with-mongocryptd-linux"
   matrix_spec: { os: "ubuntu-1804", ssl: "nossl", version: [ "4.2", "4.4", "5.0", "6.0", "latest" ], topology: ["replicaset"] }
-  display_name: "mongocryptd ${version} ${os}"
+  display_name: "CSFLE with mongocryptd ${version} ${os}"
   tasks:
-    - name: test-mongocryptd-netstandard20
-    - name: test-mongocryptd-netstandard21
+    - name: test-csfle-with-mongocryptd-netstandard20
+    - name: test-csfle-with-mongocryptd-netstandard21
 
-- matrix_name: "csfle1-macOS"
+- matrix_name: "csfle-with-mongocryptd-macOS"
   matrix_spec: { os: "macos-1015", ssl: "nossl", version: [ "4.2", "4.4", "5.0", "6.0", "latest" ], topology: ["replicaset"] }
-  display_name: "mongocryptd ${version} ${os}"
+  display_name: "CSFLE with mongocryptd ${version} ${os}"
   tasks:
-    - name: test-mongocryptd-netstandard21
+    - name: test-csfle-with-mongocryptd-netstandard21

evergreen/fetch-crypt_shared-library.sh

@@ -14,7 +14,7 @@ set -o errexit  # Exit the script with an error if any of the commands fail
 
 
 PYTHON=$(OS=${OS} ${PROJECT_DIRECTORY}/evergreen/get-python-path.sh)
-$PYTHON -u ${DRIVERS_TOOLS}/.evergreen/mongodl.py --component crypt_shared --out ${DRIVERS_TOOLS}/evergreen/csfle --version 6.0.0-rc8
+$PYTHON -u ${DRIVERS_TOOLS}/.evergreen/mongodl.py --component crypt_shared --out ${DRIVERS_TOOLS}/evergreen/csfle --version 6.0.0-rc13
 
 if [[ "$OS" =~ Windows|windows ]]; then
     export CRYPT_SHARED_LIB_PATH="${DRIVERS_TOOLS}/evergreen/csfle/bin/mongo_crypt_v1.dll"

evergreen/run-tests.sh

@@ -69,7 +69,19 @@ provision_compressor () {
 #            Main Program                  #
 ############################################
 echo "CRYPT_SHARED_LIB_PATH:" $CRYPT_SHARED_LIB_PATH
-echo "TEST_MONGOCRYPTD:" $TEST_MONGOCRYPTD
+
+if [ "$TARGET" == "TestCsfleWithMongocryptd" ]; then
+    if [ ! -z "${CRYPT_SHARED_LIB_PATH}" ]; then
+          echo "CRYPT_SHARED_LIB_PATH must be unassigned for CSFLE tests with mongocryptd, but was ${CRYPT_SHARED_LIB_PATH}" 1>&2 # write to stderr
+          exit 1   
+    fi
+else
+    if [ -z "${CRYPT_SHARED_LIB_PATH}" ]; then
+          echo "CRYPT_SHARED_LIB_PATH must be assigned, but wasn't" 1>&2 # write to stderr"
+          exit 1   
+    fi
+fi
+
 echo "Initial MongoDB URI:" $MONGODB_URI
 echo "Framework: " $FRAMEWORK
 

src/MongoDB.Driver.Core/MongoDB.Driver.Core.csproj

@@ -177,7 +177,7 @@
   <ItemGroup>
     <PackageReference Include="DnsClient" Version="1.6.1" />
     <PackageReference Include="Microsoft.CodeAnalysis.FxCopAnalyzers" Version="2.6.2" PrivateAssets="All" />
-    <PackageReference Include="MongoDB.Libmongocrypt" Version="1.5.3" />
+    <PackageReference Include="MongoDB.Libmongocrypt" Version="1.5.4" />
     <PackageReference Include="SharpCompress" Version="0.30.1" />
     <PackageReference Include="System.Buffers" Version="4.5.1" />
     <PackageReference Include="Microsoft.SourceLink.GitHub" Version="1.1.1">

src/MongoDB.Driver.Core/ServerErrorCode.cs

@@ -28,6 +28,7 @@ internal enum ServerErrorCode
         FailedToSatisfyReadPreference = 133,
         HostNotFound = 7,
         HostUnreachable = 6,
+        DuplicateKey = 11000,
         Interrupted = 11601,
         InterruptedAtShutdown = 11600,
         InterruptedDueToReplStateChange = 11602,

src/MongoDB.Driver/Encryption/AutoEncryptionLibMongoController.cs

@@ -56,7 +56,7 @@ IMongoClient CreateInternalClient()
         // private fields
         private readonly IMongoClient _internalClient;
         private readonly IMongoClient _metadataClient;
-        private readonly IMongoClient _mongocryptdClient;
+        private readonly Lazy<IMongoClient> _mongocryptdClient;
         private readonly MongocryptdFactory _mongocryptdFactory;
 
         // constructors
@@ -75,15 +75,20 @@ private AutoEncryptionLibMongoCryptController(
             _internalClient = internalClient; // can be null
             _metadataClient = metadataClient; // can be null
             _mongocryptdFactory = new MongocryptdFactory(autoEncryptionOptions.ExtraOptions, autoEncryptionOptions.BypassQueryAnalysis);
-            _mongocryptdClient = _mongocryptdFactory.CreateMongocryptdClient();
+            _mongocryptdClient = new Lazy<IMongoClient>(() => _mongocryptdFactory.CreateMongocryptdClient(), isThreadSafe: true);
         }
 
         // internal properties
         /// <summary>
-        /// this property is used by DisposableMongoClient.Dispose to unregister the internal cluster.
+        /// This property is used by DisposableMongoClient.Dispose to unregister the internal cluster.
         /// </summary>
         internal IMongoClient InternalClient => _internalClient;
 
+        /// <summary>
+        /// This property is used by DisposableMongoClient.Dispose to unregister the mongocryptd cluster.
+        /// </summary>
+        internal IMongoClient MongoCryptdClient => _mongocryptdClient.IsValueCreated ? _mongocryptdClient.Value : null;
+
         // public methods
         public byte[] DecryptFields(byte[] encryptedDocumentBytes, CancellationToken cancellationToken)
         {
@@ -217,7 +222,7 @@ private async Task ProcessNeedCollectionInfoStateAsync(CryptContext context, str
 
         private void ProcessNeedMongoMarkingsState(CryptContext context, string databaseName, CancellationToken cancellationToken)
         {
-            var database = _mongocryptdClient.GetDatabase(databaseName);
+            var database = _mongocryptdClient.Value.GetDatabase(databaseName);
             var commandBytes = context.GetOperation().ToArray();
             var commandDocument = new RawBsonDocument(commandBytes);
             var command = new BsonDocumentCommand<BsonDocument>(commandDocument);
@@ -241,7 +246,7 @@ private void ProcessNeedMongoMarkingsState(CryptContext context, string database
 
         private async Task ProcessNeedMongoMarkingsStateAsync(CryptContext context, string databaseName, CancellationToken cancellationToken)
         {
-            var database = _mongocryptdClient.GetDatabase(databaseName);
+            var database = _mongocryptdClient.Value.GetDatabase(databaseName);
             var commandBytes = context.GetOperation().ToArray();
             var commandDocument = new RawBsonDocument(commandBytes);
             var command = new BsonDocumentCommand<BsonDocument>(commandDocument);
@@ -268,7 +273,7 @@ private void WaitForMongocryptdReady()
             var stopwatch = Stopwatch.StartNew();
             while (stopwatch.Elapsed < TimeSpan.FromSeconds(5))
             {
-                var clusterDescription = _mongocryptdClient.Cluster?.Description;
+                var clusterDescription = _mongocryptdClient.Value.Cluster?.Description;
                 var mongocryptdServer = clusterDescription?.Servers?.FirstOrDefault();
                 if (mongocryptdServer != null && mongocryptdServer.Type != ServerType.Unknown)
                 {
@@ -283,7 +288,7 @@ private async Task WaitForMongocryptdReadyAsync()
             var stopwatch = Stopwatch.StartNew();
             while (stopwatch.Elapsed < TimeSpan.FromSeconds(5))
             {
-                var clusterDescription = _mongocryptdClient.Cluster?.Description;
+                var clusterDescription = _mongocryptdClient.Value.Cluster?.Description;
                 var mongocryptdServer = clusterDescription?.Servers?.FirstOrDefault();
                 if (mongocryptdServer != null && mongocryptdServer.Type != ServerType.Unknown)
                 {

src/MongoDB.Driver/Encryption/AutoEncryptionOptions.cs

@@ -271,14 +271,14 @@ public override string ToString()
 
         // internal methods
         internal CryptClientSettings ToCryptClientSettings() =>
-             new CryptClientSettings(
-                    _bypassQueryAnalysis,
-                    ExtraOptions.GetValueOrDefault<string, string, object>("cryptSharedLibPath"),
-                    cryptSharedLibSearchPath: _bypassAutoEncryption ? null : "$SYSTEM",
-                    _encryptedFieldsMap,
-                    ExtraOptions.GetValueOrDefault<bool?, string, object>("cryptSharedLibRequired"),
-                    _kmsProviders,
-                    _schemaMap);
+            new CryptClientSettings(
+                _bypassQueryAnalysis,
+                ExtraOptions.GetValueOrDefault<string, string, object>("cryptSharedLibPath"),
+                cryptSharedLibSearchPath: _bypassAutoEncryption ? null : "$SYSTEM",
+                _encryptedFieldsMap,
+                ExtraOptions.GetValueOrDefault<bool?, string, object>("cryptSharedLibRequired"),
+                _kmsProviders,
+                _schemaMap);
 
         // private methods
         private bool ExtraOptionsEquals(IReadOnlyDictionary<string, object> x, IReadOnlyDictionary<string, object> y)