RUST-1605 Update to use libmongocrypt fle2v2 (#863)

Author: abr-egn

.evergreen/config.yml

@@ -1983,6 +1983,7 @@ buildvariants:
     os:
       - ubuntu-20.04
     mongodb-version:
+      - "latest"
       - "rapid"
       - "6.0"
     topology:

src/client/csfle.rs

@@ -97,7 +97,9 @@ impl ClientState {
     }
 
     fn make_crypt(opts: &AutoEncryptionOptions) -> Result<Crypt> {
-        let mut builder = Crypt::builder().kms_providers(&opts.kms_providers.credentials_doc()?)?;
+        let mut builder = Crypt::builder()
+            .kms_providers(&opts.kms_providers.credentials_doc()?)?
+            .fle2v2(true)?;
         if let Some(m) = &opts.schema_map {
             builder = builder.schema_map(&bson::to_document(m)?)?;
         }
@@ -204,7 +206,7 @@ pub(crate) fn aux_collections(
     enc_fields: &bson::Document,
 ) -> Result<Vec<Namespace>> {
     let mut out = vec![];
-    for &key in &["esc", "ecc", "ecoc"] {
+    for &key in &["esc", "ecoc"] {
         let coll = match enc_fields.get_str(format!("{}Collection", key)) {
             Ok(s) => s.to_string(),
             Err(_) => format!("enxcol_.{}.{}", base_ns.coll, key),

src/client/csfle/client_encryption.rs

@@ -74,6 +74,7 @@ impl ClientEncryption {
         let crypt = Crypt::builder()
             .kms_providers(&kms_providers.credentials_doc()?)?
             .use_need_kms_credentials_state()
+            .fle2v2(true)?
             .build()?;
         let exec = CryptExecutor::new_explicit(
             key_vault_client.weak(),

src/client/mod.rs

@@ -565,6 +565,29 @@ impl Client {
         &self.inner.topology
     }
 
+    #[cfg(feature = "in-use-encryption-unstable")]
+    pub(crate) async fn primary_description(&self) -> Option<crate::sdam::ServerDescription> {
+        let start_time = Instant::now();
+        let timeout = self
+            .inner
+            .options
+            .server_selection_timeout
+            .unwrap_or(DEFAULT_SERVER_SELECTION_TIMEOUT);
+        let mut watcher = self.inner.topology.watch();
+        loop {
+            let topology = watcher.observe_latest();
+            if let Some(desc) = topology.description.primary() {
+                return Some(desc.clone());
+            }
+            if !watcher
+                .wait_for_update(timeout - start_time.elapsed())
+                .await
+            {
+                return None;
+            }
+        }
+    }
+
     #[cfg(feature = "in-use-encryption-unstable")]
     pub(crate) fn weak(&self) -> WeakClient {
         WeakClient {

src/db/mod.rs

@@ -379,6 +379,22 @@ impl Database {
             Some(f) => f,
             None => return Ok(()),
         };
+        let max_wire = match self.client().primary_description().await {
+            Some(p) => p.max_wire_version()?,
+            None => None,
+        };
+        const SERVER_7_0_0_WIRE_VERSION: i32 = 21;
+        match max_wire {
+            Some(v) if v >= SERVER_7_0_0_WIRE_VERSION => (),
+            _ => {
+                return Err(ErrorKind::IncompatibleServer {
+                    message: "Driver support of Queryable Encryption is incompatible with server. \
+                              Upgrade server to use Queryable Encryption."
+                        .to_string(),
+                }
+                .into())
+            }
+        }
         for ns in crate::client::csfle::aux_collections(base_ns, enc_fields)? {
             let mut sub_opts = opts.clone();
             sub_opts.clustered_index = Some(self::options::ClusteredIndex {

src/sdam/description/server.rs

@@ -15,7 +15,7 @@ use crate::{
 
 const DRIVER_MIN_DB_VERSION: &str = "3.6";
 const DRIVER_MIN_WIRE_VERSION: i32 = 6;
-const DRIVER_MAX_WIRE_VERSION: i32 = 17;
+const DRIVER_MAX_WIRE_VERSION: i32 = 21;
 
 /// Enum representing the possible types of servers that the driver can connect to.
 #[derive(Debug, Deserialize, Clone, Copy, Eq, PartialEq, Serialize)]

src/sdam/description/topology/server_selection/mod.rs

@@ -186,7 +186,7 @@ impl TopologyDescription {
             .filter(move |server| types.contains(&server.server_type))
     }
 
-    #[cfg(test)]
+    #[cfg(any(test, feature = "in-use-encryption-unstable"))]
     pub(crate) fn primary(&self) -> Option<&ServerDescription> {
         self.servers_with_type(&[ServerType::RsPrimary]).next()
     }

src/test/csfle.rs

@@ -68,6 +68,7 @@ use super::{
     TestClient,
     CLIENT_OPTIONS,
     LOCK,
+    SERVERLESS,
 };
 
 type Result<T> = anyhow::Result<T>;
@@ -2062,15 +2063,34 @@ async fn kms_tls_options() -> Result<()> {
     Ok(())
 }
 
+async fn fle2v2_ok(name: &str) -> bool {
+    if *SERVERLESS {
+        log_uncaptured(format!("Skipping {}: not supported on serverless", name));
+        return false;
+    }
+    let setup_client = Client::test_builder().build().await;
+    if setup_client.server_version_lt(7, 0) {
+        log_uncaptured(format!("Skipping {}: not supported on server < 7.0", name));
+        return false;
+    }
+    if setup_client.is_standalone() {
+        log_uncaptured(format!("Skipping {}: not supported on standalone", name));
+        return false;
+    }
+    true
+}
+
 // Prose test 12. Explicit Encryption (Case 1: can insert encrypted indexed and find)
 #[cfg_attr(feature = "tokio-runtime", tokio::test)]
 #[cfg_attr(feature = "async-std-runtime", async_std::test)]
 async fn explicit_encryption_case_1() -> Result<()> {
     if !check_env("explicit_encryption_case_1", false) {
         return Ok(());
     }
+    if !fle2v2_ok("explicit_encryption_case_1").await {
+        return Ok(());
+    }
     let _guard = LOCK.run_exclusively().await;
-
     let testdata = match explicit_encryption_setup().await? {
         Some(t) => t,
         None => return Ok(()),
@@ -2127,6 +2147,9 @@ async fn explicit_encryption_case_2() -> Result<()> {
     if !check_env("explicit_encryption_case_2", false) {
         return Ok(());
     }
+    if !fle2v2_ok("explicit_encryption_case_2").await {
+        return Ok(());
+    }
     let _guard = LOCK.run_exclusively().await;
 
     let testdata = match explicit_encryption_setup().await? {
@@ -2206,6 +2229,9 @@ async fn explicit_encryption_case_3() -> Result<()> {
     if !check_env("explicit_encryption_case_3", false) {
         return Ok(());
     }
+    if !fle2v2_ok("explicit_encryption_case_3").await {
+        return Ok(());
+    }
     let _guard = LOCK.run_exclusively().await;
 
     let testdata = match explicit_encryption_setup().await? {
@@ -2254,6 +2280,9 @@ async fn explicit_encryption_case_4() -> Result<()> {
     if !check_env("explicit_encryption_case_4", false) {
         return Ok(());
     }
+    if !fle2v2_ok("explicit_encryption_case_4").await {
+        return Ok(());
+    }
     let _guard = LOCK.run_exclusively().await;
 
     let testdata = match explicit_encryption_setup().await? {
@@ -2288,6 +2317,9 @@ async fn explicit_encryption_case_5() -> Result<()> {
     if !check_env("explicit_encryption_case_5", false) {
         return Ok(());
     }
+    if !fle2v2_ok("explicit_encryption_case_5").await {
+        return Ok(());
+    }
     let _guard = LOCK.run_exclusively().await;
 
     let testdata = match explicit_encryption_setup().await? {
@@ -2904,6 +2936,9 @@ async fn auto_encryption_keys(master_key: MasterKey) -> Result<()> {
     if !check_env("custom_key_material", false) {
         return Ok(());
     }
+    if !fle2v2_ok("auto_encryption_keys").await {
+        return Ok(());
+    }
     let _guard = LOCK.run_exclusively().await;
 
     let client = Client::test_builder().build().await;
@@ -3022,6 +3057,9 @@ async fn auto_encryption_keys(master_key: MasterKey) -> Result<()> {
 #[cfg_attr(feature = "tokio-runtime", tokio::test)]
 #[cfg_attr(feature = "async-std-runtime", async_std::test)]
 async fn range_explicit_encryption() -> Result<()> {
+    if !fle2v2_ok("range_explicit_encryption").await {
+        return Ok(());
+    }
     let client = TestClient::new().await;
     if client.server_version_lt(6, 2) || client.is_standalone() {
         log_uncaptured("Skipping range_explicit_encryption due to unsupported topology");
@@ -3410,8 +3448,8 @@ async fn fle2_example() -> Result<()> {
 
     // FLE 2 is not supported on Standalone topology.
     let test_client = Client::test_builder().build().await;
-    if test_client.server_version_lt(6, 0) {
-        log_uncaptured("skipping fle2 example: server below 6.0");
+    if test_client.server_version_lt(7, 0) {
+        log_uncaptured("skipping fle2 example: server below 7.0");
         return Ok(());
     }
     if test_client.is_standalone() {

src/test/spec/json/client-side-encryption/README.rst

@@ -1679,7 +1679,14 @@ Expect no error on construction.
 12. Explicit Encryption
 ~~~~~~~~~~~~~~~~~~~~~~~
 
-The Explicit Encryption tests require MongoDB server 6.0+. The tests must not run against a standalone.
+The Explicit Encryption tests require MongoDB server 7.0+. The tests must not run against a standalone.
+
+.. note::
+   MongoDB Server 7.0 introduced a backwards breaking change to the Queryable Encryption (QE) protocol: QEv2.
+	libmongocrypt 1.8.0 is configured to use the QEv2 protocol.
+
+.. note::
+   Skip this test on Serverless until MongoDB Serverless enables the QEv2 protocol. Refer: `DRIVERS-2589 <https://jira.mongodb.org/browse/DRIVERS-2589>`_
 
 Before running each of the following test cases, perform the following Test Setup.
 
@@ -2495,7 +2502,14 @@ The following tests that a mongocryptd client is not created when shared library
 21. Automatic Data Encryption Keys
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-The Automatic Data Encryption Keys tests require MongoDB server 6.0+. The tests must not run against a standalone.
+The Automatic Data Encryption Keys tests require MongoDB server 7.0+. The tests must not run against a standalone.
+
+.. note::
+   MongoDB Server 7.0 introduced a backwards breaking change to the Queryable Encryption (QE) protocol: QEv2.
+	libmongocrypt 1.8.0 is configured to use the QEv2 protocol.
+
+.. note::
+   Skip this test on Serverless until MongoDB Serverless enables the QEv2 protocol. Refer: `DRIVERS-2589 <https://jira.mongodb.org/browse/DRIVERS-2589>`_
 
 For each of the following test cases, assume `DB` is a valid open database
 handle, and assume a ClientEncryption_ object `CE` created using the following
@@ -2644,7 +2658,14 @@ with encrypted value.
 
 22. Range Explicit Encryption
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-The Range Explicit Encryption tests require MongoDB server 6.2+. The tests must not run against a standalone.
+The Range Explicit Encryption tests require MongoDB server 7.0+. The tests must not run against a standalone.
+
+.. note::
+   MongoDB Server 7.0 introduced a backwards breaking change to the Queryable Encryption (QE) protocol: QEv2.
+	libmongocrypt 1.8.0 is configured to use the QEv2 protocol.
+
+.. note::
+   Skip this test on Serverless until MongoDB Serverless enables the QEv2 protocol. Refer: `DRIVERS-2589 <https://jira.mongodb.org/browse/DRIVERS-2589>`_
 
 Each of the following test cases must pass for each of the supported types (``DecimalNoPrecision``, ``DecimalPrecision``, ``DoublePrecision``, ``DoubleNoPrecision``, ``Date``, ``Int``, and ``Long``), unless it is stated the type should be skipped.
 
@@ -2654,7 +2675,7 @@ Before running each of the following test cases, perform the following Test Setu
 
 Test Setup
 ``````````
-Load the file for the specific data type being tested ``encryptedFields-<type>.json``. For example, for ``Int`` load `range-encryptedFields-Int.json <https://github.com/mongodb/specifications/blob/master/source/client-side-encryption/etc/data/range-encryptedFields-Int.json>`_ as ``encryptedFields``.
+Load the file for the specific data type being tested ``range-encryptedFields-<type>.json``. For example, for ``Int`` load `range-encryptedFields-Int.json <https://github.com/mongodb/specifications/blob/master/source/client-side-encryption/etc/data/range-encryptedFields-Int.json>`_ as ``encryptedFields``.
 
 Load the file `key1-document.json <https://github.com/mongodb/specifications/tree/master/source/client-side-encryption/etc/data/keys/key1-document.json>`_ as ``key1Document``.