RUST-1386 Implement the remainder of the FLE prose tests (#767)

Author: abr-egn

.evergreen/config.yml

@@ -475,28 +475,28 @@ functions:
 
   "fetch crypt_shared":
   - command: shell.exec
-    type: test
     params:
       shell: bash
       working_dir: "src"
       script: |
         ${PREPARE_SHELL}
-        python3 ${DRIVERS_TOOLS}/.evergreen/mongodl.py --component=crypt_shared --version=${CRYPT_SHARED_VERSION} --out=./crypt_shared/
-        ls -R ${PROJECT_DIRECTORY}/crypt_shared
+        if [ "${DISABLE_CRYPT_SHARED}" == "true" ]; then
+          echo "crypt_shared is disabled, not fetching"
+        else
+          ${PYTHON} ${DRIVERS_TOOLS}/.evergreen/mongodl.py --component=crypt_shared --version=${CRYPT_SHARED_VERSION} --out=./crypt_shared/
+          ls -R ${PROJECT_DIRECTORY}/crypt_shared
+        fi
 
   "run kmip server":
   - command: shell.exec
-    type: test
     params:
       shell: bash
       working_dir: "src"
       background: true
       script: |
         ${PREPARE_SHELL}
-        cd ${DRIVERS_TOOLS}/.evergreen/csfle
-        . ./activate_venv.sh
-        # TMPDIR is required to avoid "AF_UNIX path too long" errors.
-        TMPDIR="$(dirname ${DRIVERS_TOOLS})" python3 kms_kmip_server.py
+        export TLS_FEATURE=${TLS_FEATURE}
+        .evergreen/run-csfle-kmip-servers.sh
 
   "run csfle tests":
     - command: shell.exec
@@ -509,6 +509,8 @@ functions:
           export ASYNC_RUNTIME=${ASYNC_RUNTIME}
           export CSFLE_TLS_CERT_DIR=$DRIVERS_TOOLS_X509
           export CSFLE_SHARED_LIB_PATH=$(.evergreen/find-crypt_shared.sh "$PROJECT_DIRECTORY/crypt_shared/lib/")
+          export DISABLE_CRYPT_SHARED=${DISABLE_CRYPT_SHARED}
+          export TLS_FEATURE=${TLS_FEATURE}
           # Exported without xtrace to avoid leaking credentials
           set +o xtrace
           export KMS_PROVIDERS=$(cat << "EOF"
@@ -1177,8 +1179,6 @@ tasks:
       - func: "install junit dependencies"
       - func: "fetch crypt_shared"
       - func: "bootstrap mongo-orchestration"
-        vars:
-          TOPOLOGY: "server"
       - func: "run kmip server"
       - func: "run csfle tests"
 
@@ -1569,6 +1569,7 @@ axes:
         display_name: "latest"
         variables:
            MONGODB_VERSION: "latest"
+           CRYPT_SHARED_VERSION: "latest"
       - id: "rapid"
         display_name: "rapid"
         variables:
@@ -1653,6 +1654,18 @@ axes:
            SSL: "nossl"
            TLS_FEATURE: ""
 
+  - id: "tls-feature"
+    display_name: "TLS Feature"
+    values:
+      - id: "default"
+        display_name: "rustls TLS"
+        variables:
+          TLS_FEATURE: ""
+      - id: "openssl"
+        display_name: "OpenSSL TLS"
+        variables:
+          TLS_FEATURE: "openssl-tls"
+
   - id: "compressor"
     display_name: "Compressor"
     values:
@@ -1678,29 +1691,29 @@ axes:
         display_name: "Ubuntu 18.04"
         run_on: ubuntu1804-test
         variables:
-          PYTHON: "/opt/mongodbtoolchain/v3/bin/python"
+          PYTHON: "/opt/mongodbtoolchain/v3/bin/python3"
           VENV_BIN_DIR: "bin"
           LIBMONGOCRYPT_OS: "ubuntu1804-64"
       - id: ubuntu-20.04
         display_name: "Ubuntu 20.04"
         run_on: ubuntu2004-test
         variables:
-          PYTHON: "/opt/mongodbtoolchain/v3/bin/python"
+          PYTHON: "/opt/mongodbtoolchain/v3/bin/python3"
           VENV_BIN_DIR: "bin"
           LIBMONGOCRYPT_OS: "ubuntu2004-64"
       - id: ubuntu-18.04-arm64
         display_name: "ARM64 Ubuntu 18.04"
         run_on: ubuntu1804-arm64-test
         variables:
-          PYTHON: "/opt/mongodbtoolchain/v3/bin/python"
+          PYTHON: "/opt/mongodbtoolchain/v3/bin/python3"
           VENV_BIN_DIR: "bin"
           LIBMONGOCRYPT_OS: "ubuntu1804-arm64"
       - id: macos-11.00
         display_name: "MacOS 11.00"
         run_on: macos-1100
         variables:
           SINGLE_THREAD: true
-          PYTHON: "/opt/mongodbtoolchain/v3/bin/python"
+          PYTHON: "/opt/mongodbtoolchain/v3/bin/python3"
           VENV_BIN_DIR: "bin"
           LIBMONGOCRYPT_OS: "macos"
       - id: windows-64-vs2017
@@ -1720,6 +1733,18 @@ axes:
           REQUIRE_API_VERSION: "1"
           MONGODB_API_VERSION: "1"
 
+  - id: "crypt-shared"
+    display_name: "crypt_shared"
+    values:
+      - id: enabled
+        display_name: "crypt_shared enabled"
+        variables:
+          DISABLE_CRYPT_SHARED: ""
+      - id: disabled
+        display_name: "crypt_shared disabled"
+        variables:
+          DISABLE_CRYPT_SHARED: "true"
+
 task_groups:
   - name: serverless_task_group
     setup_group_can_fail_task: true
@@ -1868,14 +1893,72 @@ buildvariants:
   tasks:
     - "serverless_task_group"
 
-# TODO(RUST-1386): Expand the os and version listing
-- matrix_name: "csfle"
+- matrix_name: "csfle-topology"
   matrix_spec:
     os:
       - ubuntu-20.04
     mongodb-version:
       - "6.0"
-  display_name: "CSFLE on mongodb ${mongodb-version} / ${os}"
+    topology:
+      - "standalone"
+      - "replica-set"
+    crypt-shared:
+      - "enabled"
+    tls-feature:
+      - "default"
+  display_name: "CSFLE (${crypt-shared}, ${tls-feature}) on mongodb ${mongodb-version} ${topology} / ${os}"
+  tasks:
+    - "test-csfle"
+
+- matrix_name: "csfle-os"
+  matrix_spec:
+    os:
+      - macos-11.00
+      - windows-64-vs2017
+    mongodb-version:
+      - "6.0"
+    topology:
+      - "standalone"
+    crypt-shared:
+      - "enabled"
+    tls-feature:
+      - "default"
+  display_name: "CSFLE (${crypt-shared}, ${tls-feature}) on mongodb ${mongodb-version} ${topology} / ${os}"
+  tasks:
+    - "test-csfle"
+
+- matrix_name: "csfle-crypt-shared"
+  matrix_spec:
+    os:
+      - ubuntu-20.04
+    mongodb-version:
+      - "5.0"
+      - "6.0"
+    topology:
+      - "standalone"
+    crypt-shared:
+      - "disabled"
+    tls-feature:
+      - "default"
+  display_name: "CSFLE (${crypt-shared}, ${tls-feature}) on mongodb ${mongodb-version} ${topology} / ${os}"
+  tasks:
+    - "test-csfle"
+
+- matrix_name: "csfle-tls"
+  matrix_spec:
+    os:
+      - ubuntu-20.04
+      - macos-11.00
+      - windows-64-vs2017
+    mongodb-version:
+      - "6.0"
+    topology:
+      - "standalone"
+    crypt-shared:
+      - "enabled"
+    tls-feature:
+      - "openssl"
+  display_name: "CSFLE (${crypt-shared}, ${tls-feature}) on mongodb ${mongodb-version} ${topology} / ${os}"
   tasks:
     - "test-csfle"
 

.evergreen/configure-rust.sh

@@ -6,9 +6,13 @@ export CARGO_HOME="${PROJECT_DIRECTORY}/.cargo"
 export PATH="${CARGO_HOME}/bin:$PATH"
 
 if [[ "Windows_NT" == "$OS" ]]; then
+    # Update path for DLLs
+    export PATH="${MONGOCRYPT_LIB_DIR}/../bin:$PATH"
+
     # rustup/cargo need the native Windows paths; $PROJECT_DIRECTORY is a cygwin path
     export RUSTUP_HOME=$(cygpath ${RUSTUP_HOME} --windows)
     export CARGO_HOME=$(cygpath ${CARGO_HOME} --windows)
+    export MONGOCRYPT_LIB_DIR=$(cygpath ${MONGOCRYPT_LIB_DIR} --windows)
 fi
 
 . ${CARGO_HOME}/env

.evergreen/find-crypt_shared.sh

@@ -1,15 +1,15 @@
 #!/usr/bin/env bash
 
-if [ "$1" = "" ]; then
-    echo "crypt_shared library directory required"
-    exit 1
-fi
-
-crypt_shared_glob=("$1"/*)
-
-if [ "${#crypt_shared_glob[@]}" != "1" ]; then
-    echo "Wrong number of files found: ${crypt_shared_glob[@]}"
-    exit 1
-fi
-
-echo ${crypt_shared_glob[0]}
+if [[ "Windows_NT" == "$OS" ]]; then
+    cygpath ${PROJECT_DIRECTORY}/crypt_shared/bin/mongo_crypt_v1.dll --windows
+else
+    CS_PATH=${PROJECT_DIRECTORY}/crypt_shared/lib
+    crypt_shared_glob=("$CS_PATH"/*)
+
+    if [ "${#crypt_shared_glob[@]}" != "1" ]; then
+        echo "Wrong number of files found: ${crypt_shared_glob[@]}"
+        exit 1
+    fi
+
+    echo ${crypt_shared_glob[0]}
+fi

.evergreen/run-csfle-kmip-servers.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+if [ "$TLS_FEATURE" != "openssl-tls" ]; then
+    echo "Skipping kms servers: openssl-tls not enabled"
+    exit
+fi
+
+cd ${DRIVERS_TOOLS}/.evergreen/csfle
+. ./activate_venv.sh
+# TMPDIR is required to avoid "AF_UNIX path too long" errors.
+export TMPDIR="$(dirname ${DRIVERS_TOOLS})"
+
+python kms_kmip_server.py &
+python -u kms_http_server.py --ca_file ../x509gen/ca.pem --cert_file ../x509gen/expired.pem --port 9000 &
+python -u kms_http_server.py --ca_file ../x509gen/ca.pem --cert_file ../x509gen/wrong-host.pem --port 9001 &
+python -u kms_http_server.py --ca_file ../x509gen/ca.pem --cert_file ../x509gen/server.pem --port 9002 --require_client_cert

.evergreen/run-csfle-tests.sh

@@ -7,13 +7,19 @@ source ./.evergreen/env.sh
 
 set -o xtrace
 
-FEATURE_FLAGS="openssl-tls,csfle"
+FEATURE_FLAGS="csfle,${TLS_FEATURE}"
 OPTIONS="-- -Z unstable-options --format json --report-time"
 
 if [ "$SINGLE_THREAD" = true ]; then
 	OPTIONS="$OPTIONS --test-threads=1"
 fi
 
+if [ "$OS" = "Windows_NT" ]; then
+    export CSFLE_TLS_CERT_DIR=$(cygpath ${CSFLE_TLS_CERT_DIR} --windows)
+    export SSL_CERT_FILE=$(cygpath /etc/ssl/certs/ca-bundle.crt --windows)
+    export SSL_CERT_DIR=$(cygpath /etc/ssl/certs --windows)
+fi
+
 echo "cargo test options: --features ${FEATURE_FLAGS} ${OPTIONS}"
 
 CARGO_RESULT=0

.gitignore

@@ -12,3 +12,4 @@ Cargo.lock
 # we install cargo and rustup in the project directory on Evergreen.
 .cargo
 .rustup
+mongocryptd.pid

src/client/csfle.rs

@@ -2,7 +2,7 @@ pub mod client_encryption;
 pub mod options;
 mod state_machine;
 
-use std::path::Path;
+use std::{path::Path, time::Duration};
 
 use derivative::Derivative;
 use mongocrypt::Crypt;
@@ -44,15 +44,33 @@ struct AuxClients {
 }
 
 impl ClientState {
+    const MONGOCRYPTD_DEFAULT_URI: &'static str = "mongodb://localhost:27020";
+    const MONGOCRYPTD_SERVER_SELECTION_TIMEOUT: Duration = Duration::from_millis(10_000);
+
     pub(super) async fn new(client: &Client, mut opts: AutoEncryptionOptions) -> Result<Self> {
         let crypt = Self::make_crypt(&opts)?;
         let mongocryptd_opts = Self::make_mongocryptd_opts(&opts, &crypt)?;
         let aux_clients = Self::make_aux_clients(client, &opts)?;
+        let mongocryptd_connect = opts.bypass_auto_encryption != Some(true)
+            && opts.bypass_query_analysis != Some(true)
+            && crypt.shared_lib_version().is_none()
+            && opts.extra_option(&EO_CRYPT_SHARED_REQUIRED)? != Some(true);
+        let mongocryptd_client = if mongocryptd_connect {
+            let uri = opts
+                .extra_option(&EO_MONGOCRYPTD_URI)?
+                .unwrap_or(Self::MONGOCRYPTD_DEFAULT_URI);
+            let mut options = crate::options::ClientOptions::parse_uri(uri, None).await?;
+            options.server_selection_timeout = Some(Self::MONGOCRYPTD_SERVER_SELECTION_TIMEOUT);
+            Some(Client::with_options(options)?)
+        } else {
+            None
+        };
         let exec = CryptExecutor::new_implicit(
             aux_clients.key_vault_client,
             opts.key_vault_namespace.clone(),
             opts.tls_options.take(),
             mongocryptd_opts,
+            mongocryptd_client,
             aux_clients.metadata_client,
         )
         .await?;
@@ -83,11 +101,20 @@ impl ClientState {
         if let Some(m) = &opts.schema_map {
             builder = builder.schema_map(&bson::to_document(m)?)?;
         }
-        if Some(true) != opts.bypass_auto_encryption {
-            builder = builder.append_crypt_shared_lib_search_path(Path::new("$SYSTEM"))?;
+        #[cfg(not(test))]
+        let disable_crypt_shared = false;
+        #[cfg(test)]
+        let disable_crypt_shared = opts.disable_crypt_shared.unwrap_or(false);
+        if !disable_crypt_shared {
+            if Some(true) != opts.bypass_auto_encryption {
+                builder = builder.append_crypt_shared_lib_search_path(Path::new("$SYSTEM"))?;
+            }
+            if let Some(p) = opts.extra_option(&EO_CRYPT_SHARED_LIB_PATH)? {
+                builder = builder.set_crypt_shared_lib_path_override(Path::new(p))?;
+            }
         }
-        if let Some(p) = opts.extra_option(&EO_CRYPT_SHARED_LIB_PATH)? {
-            builder = builder.set_crypt_shared_lib_path_override(Path::new(p))?;
+        if opts.bypass_query_analysis == Some(true) {
+            builder = builder.bypass_query_analysis();
         }
         let crypt = builder.build()?;
         if opts.extra_option(&EO_CRYPT_SHARED_REQUIRED)? == Some(true)
@@ -105,6 +132,7 @@ impl ClientState {
         crypt: &Crypt,
     ) -> Result<Option<MongocryptdOptions>> {
         if opts.bypass_auto_encryption == Some(true)
+            || opts.bypass_query_analysis == Some(true)
             || opts.extra_option(&EO_MONGOCRYPTD_BYPASS_SPAWN)? == Some(true)
             || crypt.shared_lib_version().is_some()
             || opts.extra_option(&EO_CRYPT_SHARED_REQUIRED)? == Some(true)
@@ -123,13 +151,9 @@ impl ClientState {
                 spawn_args.push(str_arg.to_string());
             }
         }
-        let uri = opts
-            .extra_option(&EO_MONGOCRYPTD_URI)?
-            .map(|s| s.to_string());
         Ok(Some(MongocryptdOptions {
             spawn_path,
             spawn_args,
-            uri,
         }))
     }