RUST-1386 Implement the first batch of FLE prose tests (#758)

Author: abr-egn

.evergreen/config.yml

@@ -55,6 +55,8 @@ functions:
            export MONGODB_BINARIES="$DRIVERS_TOOLS/mongodb/bin"
            export UPLOAD_BUCKET="${project}"
            export PROJECT_DIRECTORY="$(pwd)"
+           export MONGOCRYPT_LIB_DIR="$PROJECT_DIRECTORY/libmongocrypt/${LIBMONGOCRYPT_OS}/lib"
+           export LD_LIBRARY_PATH="$MONGOCRYPT_LIB_DIR:$LD_LIBRARY_PATH"
 
            cat <<EOT > expansion.yml
            CURRENT_VERSION: "$CURRENT_VERSION"
@@ -72,6 +74,8 @@ functions:
               export UPLOAD_BUCKET="$UPLOAD_BUCKET"
               export PROJECT_DIRECTORY="$PROJECT_DIRECTORY"
               export DRIVERS_TOOLS_X509="$DRIVERS_TOOLS/.evergreen/x509gen"
+              export MONGOCRYPT_LIB_DIR="$MONGOCRYPT_LIB_DIR"
+              export LD_LIBRARY_PATH="$LD_LIBRARY_PATH"
 
               export TMPDIR="$MONGO_ORCHESTRATION_HOME/db"
               export PATH="$MONGODB_BINARIES:$PATH"
@@ -81,6 +85,7 @@ functions:
               export SSL=${SSL}
               export TOPOLOGY=${TOPOLOGY}
               export MONGODB_VERSION=${MONGODB_VERSION}
+              export CRYPT_SHARED_VERSION=${CRYPT_SHARED_VERSION}
 
               if [ "Windows_NT" != "$OS" ]; then
                   ulimit -n 64000
@@ -344,6 +349,14 @@ functions:
         ${PREPARE_SHELL}
         .evergreen/install-dependencies.sh rust
   
+  "install libmongocrypt":
+    command: shell.exec
+    params:
+      working_dir: "src"
+      script: |
+        ${PREPARE_SHELL}
+        .evergreen/install-dependencies.sh libmongocrypt
+  
   "install junit dependencies":
     command: shell.exec
     params:
@@ -460,6 +473,52 @@ functions:
           MONGO_OCSP_TESTS=1 \
           .evergreen/run-ocsp-test.sh
 
+  "fetch crypt_shared":
+  - command: shell.exec
+    type: test
+    params:
+      shell: bash
+      working_dir: "src"
+      script: |
+        ${PREPARE_SHELL}
+        python3 ${DRIVERS_TOOLS}/.evergreen/mongodl.py --component=crypt_shared --version=${CRYPT_SHARED_VERSION} --out=./crypt_shared/
+        ls -R ${PROJECT_DIRECTORY}/crypt_shared
+
+  "run kmip server":
+  - command: shell.exec
+    type: test
+    params:
+      shell: bash
+      working_dir: "src"
+      background: true
+      script: |
+        ${PREPARE_SHELL}
+        cd ${DRIVERS_TOOLS}/.evergreen/csfle
+        . ./activate_venv.sh
+        # TMPDIR is required to avoid "AF_UNIX path too long" errors.
+        TMPDIR="$(dirname ${DRIVERS_TOOLS})" python3 kms_kmip_server.py
+
+  "run csfle tests":
+    - command: shell.exec
+      type: test
+      params:
+        shell: bash
+        working_dir: "src"
+        script: |
+          ${PREPARE_SHELL}
+          export ASYNC_RUNTIME=${ASYNC_RUNTIME}
+          export CSFLE_TLS_CERT_DIR=$DRIVERS_TOOLS_X509
+          export CSFLE_SHARED_LIB_PATH=$(.evergreen/find-crypt_shared.sh "$PROJECT_DIRECTORY/crypt_shared/lib/")
+          # Exported without xtrace to avoid leaking credentials
+          set +o xtrace
+          export KMS_PROVIDERS=$(cat << "EOF"
+          ${kms_providers}
+          EOF
+          )
+          set -o xtrace
+
+          .evergreen/run-csfle-tests.sh
+
   run-valid-ocsp-server:
     - command: shell.exec
       params:
@@ -713,6 +772,7 @@ pre:
   - func: "init test-results"
   - func: "make files executable"
   - func: "install rust"
+  - func: "install libmongocrypt"
 
 post:
   - func: "stop load balancer"
@@ -1111,6 +1171,17 @@ tasks:
       - func: "install junit dependencies"
       - func: "run serverless tests"
 
+  - name: "test-csfle"
+    tags: ["csfle"]
+    commands:
+      - func: "install junit dependencies"
+      - func: "fetch crypt_shared"
+      - func: "bootstrap mongo-orchestration"
+        vars:
+          TOPOLOGY: "server"
+      - func: "run kmip server"
+      - func: "run csfle tests"
+
   - name: "test-atlas-connectivity"
     tags: ["atlas-connect"]
     commands:
@@ -1506,6 +1577,7 @@ axes:
         display_name: "6.0"
         variables:
           MONGODB_VERSION: "6.0"
+          CRYPT_SHARED_VERSION: "6.0.0"
       - id: "5.0"
         display_name: "5.0"
         variables:
@@ -1608,31 +1680,36 @@ axes:
         variables:
           PYTHON: "/opt/mongodbtoolchain/v3/bin/python"
           VENV_BIN_DIR: "bin"
+          LIBMONGOCRYPT_OS: "ubuntu1804-64"
       - id: ubuntu-20.04
         display_name: "Ubuntu 20.04"
         run_on: ubuntu2004-test
         variables:
           PYTHON: "/opt/mongodbtoolchain/v3/bin/python"
           VENV_BIN_DIR: "bin"
+          LIBMONGOCRYPT_OS: "ubuntu2004-64"
       - id: ubuntu-18.04-arm64
         display_name: "ARM64 Ubuntu 18.04"
         run_on: ubuntu1804-arm64-test
         variables:
           PYTHON: "/opt/mongodbtoolchain/v3/bin/python"
           VENV_BIN_DIR: "bin"
+          LIBMONGOCRYPT_OS: "ubuntu1804-arm64"
       - id: macos-11.00
         display_name: "MacOS 11.00"
         run_on: macos-1100
         variables:
           SINGLE_THREAD: true
           PYTHON: "/opt/mongodbtoolchain/v3/bin/python"
           VENV_BIN_DIR: "bin"
+          LIBMONGOCRYPT_OS: "macos"
       - id: windows-64-vs2017
         display_name: "Windows (VS 2017)"
         run_on: windows-64-vs2017-test
         variables:
           PYTHON: "/cygdrive/c/python/Python36/python"
           VENV_BIN_DIR: "Scripts"
+          LIBMONGOCRYPT_OS: "windows-test"
 
   - id: "versioned-api"
     display_name: "Versioned API"
@@ -1791,6 +1868,17 @@ buildvariants:
   tasks:
     - "serverless_task_group"
 
+# TODO(RUST-1386): Expand the os and version listing
+- matrix_name: "csfle"
+  matrix_spec:
+    os:
+      - ubuntu-20.04
+    mongodb-version:
+      - "6.0"
+  display_name: "CSFLE on mongodb ${mongodb-version} / ${os}"
+  tasks:
+    - "test-csfle"
+
 - matrix_name: "atlas-connect"
   matrix_spec:
     os:

.evergreen/find-crypt_shared.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+if [ "$1" = "" ]; then
+    echo "crypt_shared library directory required"
+    exit 1
+fi
+
+crypt_shared_glob=("$1"/*)
+
+if [ "${#crypt_shared_glob[@]}" != "1" ]; then
+    echo "Wrong number of files found: ${crypt_shared_glob[@]}"
+    exit 1
+fi
+
+echo ${crypt_shared_glob[0]}

.evergreen/install-dependencies.sh

@@ -58,6 +58,14 @@ for arg; do
         if [ $RESULT -ne 0 ]; then
             exit $RESULT
         fi
+    elif [ $arg == "libmongocrypt" ]; then
+        mkdir ${PROJECT_DIRECTORY}/libmongocrypt
+        cd ${PROJECT_DIRECTORY}/libmongocrypt
+        curl -sSfO https://s3.amazonaws.com/mciuploads/libmongocrypt/all/master/latest/libmongocrypt-all.tar.gz
+        tar xzf libmongocrypt-all.tar.gz
+        if [ "Windows_NT" == "$OS" ]; then
+            chmod +x ${MONGOCRYPT_LIB_DIR}/../bin/*.dll
+        fi
     else
         echo Missing/unknown install option: "$arg"
         exit 1

.evergreen/run-csfle-tests.sh

@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+
+set -o errexit
+set -o pipefail
+
+source ./.evergreen/env.sh
+
+set -o xtrace
+
+FEATURE_FLAGS="openssl-tls,csfle"
+OPTIONS="-- -Z unstable-options --format json --report-time"
+
+if [ "$SINGLE_THREAD" = true ]; then
+	OPTIONS="$OPTIONS --test-threads=1"
+fi
+
+echo "cargo test options: --features ${FEATURE_FLAGS} ${OPTIONS}"
+
+CARGO_RESULT=0
+
+cargo_test() {
+    RUST_BACKTRACE=1 \
+        cargo test --features ${FEATURE_FLAGS} $1 ${OPTIONS} | cargo2junit
+    (( CARGO_RESULT = ${CARGO_RESULT} || $? ))
+}
+
+set +o errexit
+
+cargo_test test::csfle > results.xml
+
+exit ${CARGO_RESULT}

Cargo.toml

@@ -165,6 +165,7 @@ version = "1.1.2"
 features = ["v4"]
 
 [dev-dependencies]
+anyhow = { version = "1.0", features = ["backtrace"] }
 approx = "0.5.1"
 async_once = "0.2.6"
 ctrlc = "3.2.2"

src/client/csfle.rs

@@ -78,7 +78,11 @@ impl ClientState {
     }
 
     fn make_crypt(opts: &AutoEncryptionOptions) -> Result<Crypt> {
-        let mut builder = Crypt::builder();
+        let mut builder =
+            Crypt::builder().kms_providers(&bson::to_document(&opts.kms_providers)?)?;
+        if let Some(m) = &opts.schema_map {
+            builder = builder.schema_map(&bson::to_document(m)?)?;
+        }
         if Some(true) != opts.bypass_auto_encryption {
             builder = builder.append_crypt_shared_lib_search_path(Path::new("$SYSTEM"))?;
         }

src/client/csfle/client_encryption.rs

@@ -35,7 +35,9 @@ pub struct ClientEncryption {
 impl ClientEncryption {
     /// Create a new key vault handle with the given options.
     pub fn new(options: ClientEncryptionOptions) -> Result<Self> {
-        let crypt = Crypt::builder().build()?;
+        let crypt = Crypt::builder()
+            .kms_providers(&bson::to_document(&options.kms_providers)?)?
+            .build()?;
         let exec = CryptExecutor::new_explicit(
             options.key_vault_client.weak(),
             options.key_vault_namespace.clone(),
@@ -62,10 +64,10 @@ impl ClientEncryption {
     /// Returns the _id of the created document as a UUID (BSON binary subtype 0x04).
     pub async fn create_data_key(
         &self,
-        kms_provider: KmsProvider,
+        kms_provider: &KmsProvider,
         opts: impl Into<Option<DataKeyOptions>>,
     ) -> Result<Binary> {
-        let ctx = self.create_data_key_ctx(kms_provider, opts.into())?;
+        let ctx = self.create_data_key_ctx(kms_provider, opts.into().as_ref())?;
         let data_key = self.exec.run_ctx(ctx, None).await?;
         self.key_vault.insert_one(&data_key, None).await?;
         let bin_ref = data_key
@@ -76,8 +78,8 @@ impl ClientEncryption {
 
     fn create_data_key_ctx(
         &self,
-        kms_provider: KmsProvider,
-        opts: Option<DataKeyOptions>,
+        kms_provider: &KmsProvider,
+        opts: Option<&DataKeyOptions>,
     ) -> Result<Ctx> {
         let mut builder = self.crypt.ctx_builder();
         let mut key_doc = doc! { "provider": kms_provider.name() };
@@ -86,13 +88,13 @@ impl ClientEncryption {
                 let master_doc = bson::to_document(&opts.master_key)?;
                 key_doc.extend(master_doc);
             }
-            if let Some(alt_names) = opts.key_alt_names {
+            if let Some(alt_names) = &opts.key_alt_names {
                 for name in alt_names {
-                    builder = builder.key_alt_name(&name)?;
+                    builder = builder.key_alt_name(name)?;
                 }
             }
-            if let Some(material) = opts.key_material {
-                builder = builder.key_material(&material)?;
+            if let Some(material) = &opts.key_material {
+                builder = builder.key_material(material)?;
             }
         }
         builder = builder.key_encryption_key(&key_doc)?;
@@ -181,30 +183,30 @@ impl ClientEncryption {
     /// Encrypts a BsonValue with a given key and algorithm.
     /// Returns an encrypted value (BSON binary of subtype 6).
     pub async fn encrypt(&self, value: bson::RawBson, opts: EncryptOptions) -> Result<Binary> {
-        let ctx = self.encrypt_ctx(value, opts)?;
+        let ctx = self.encrypt_ctx(value, &opts)?;
         let result = self.exec.run_ctx(ctx, None).await?;
         let bin_ref = result
             .get_binary("v")
             .map_err(|e| Error::internal(format!("invalid encryption result: {}", e)))?;
         Ok(bin_ref.to_binary())
     }
 
-    fn encrypt_ctx(&self, value: bson::RawBson, opts: EncryptOptions) -> Result<Ctx> {
+    fn encrypt_ctx(&self, value: bson::RawBson, opts: &EncryptOptions) -> Result<Ctx> {
         let mut builder = self.crypt.ctx_builder();
-        match opts.key {
+        match &opts.key {
             EncryptKey::Id(id) => {
                 builder = builder.key_id(&id.bytes)?;
             }
             EncryptKey::AltName(name) => {
-                builder = builder.key_alt_name(&name)?;
+                builder = builder.key_alt_name(name)?;
             }
         }
         builder = builder.algorithm(opts.algorithm)?;
         if let Some(factor) = opts.contention_factor {
             builder = builder.contention_factor(factor)?;
         }
-        if let Some(qtype) = opts.query_type {
-            builder = builder.query_type(&qtype)?;
+        if let Some(qtype) = &opts.query_type {
+            builder = builder.query_type(qtype)?;
         }
         Ok(builder.build_explicit_encrypt(value)?)
     }
@@ -270,26 +272,30 @@ pub struct DataKeyOptions {
 }
 
 /// A KMS-specific key used to encrypt data keys.
+#[serde_with::skip_serializing_none]
 #[derive(Debug, Clone, Serialize)]
-#[serde(rename_all = "camelCase", untagged)]
+#[serde(untagged)]
 #[non_exhaustive]
 #[allow(missing_docs)]
 pub enum MasterKey {
+    #[serde(rename_all = "camelCase")]
     Aws {
         region: String,
         /// The Amazon Resource Name (ARN) to the AWS customer master key (CMK).
         key: String,
         /// An alternate host identifier to send KMS requests to. May include port number. Defaults
-        /// to "kms.<region>.amazonaws.com"
+        /// to "kms.REGION.amazonaws.com"
         endpoint: Option<String>,
     },
+    #[serde(rename_all = "camelCase")]
     Azure {
         /// Host with optional port. Example: "example.vault.azure.net".
         key_vault_endpoint: String,
         key_name: String,
         /// A specific version of the named key, defaults to using the key's primary version.
         key_version: Option<String>,
     },
+    #[serde(rename_all = "camelCase")]
     Gcp {
         project_id: String,
         location: String,
@@ -302,6 +308,7 @@ pub enum MasterKey {
     },
     /// Master keys are not applicable to `KmsProvider::Local`.
     Local,
+    #[serde(rename_all = "camelCase")]
     Kmip {
         /// keyId is the KMIP Unique Identifier to a 96 byte KMIP Secret Data managed object.  If
         /// keyId is omitted, the driver creates a random 96 byte KMIP Secret Data managed object.