RUST-1665 Run OIDC unified tests (#1329)

Author: isabelatkinson

.evergreen/build-static-test-tarball.sh

@@ -5,11 +5,14 @@ set -o pipefail
 
 source ./.evergreen/env.sh
 
+rm -rf test_files && mkdir test_files
+cp ${TEST_FILES}/* test_files
+
 export RUSTFLAGS="-C target-feature=+crt-static"
 cargo test ${BUILD_FEATURES} --target x86_64-unknown-linux-gnu get_exe_name
 TEST_BINARY=$(cat exe_name.txt)
 TEST_TARBALL="/tmp/mongo-rust-driver.tar.gz"
-tar czvf ${TEST_TARBALL} ${TEST_BINARY} ./.evergreen
+tar czvf ${TEST_TARBALL} ${TEST_BINARY} ./.evergreen test_files
 
 cat <<EOT > static-test-tarball-expansion.yml
 STATIC_TEST_BINARY: ${TEST_BINARY}

.evergreen/config.yml

@@ -1169,6 +1169,7 @@ tasks:
     - func: "build static test tarball"
       vars:
         BUILD_FEATURES: "--features azure-oidc"
+        TEST_FILES: "${PROJECT_DIRECTORY}/src/test/spec/json/auth/unified"
     - command: subprocess.exec
       type: test
       params:
@@ -1185,6 +1186,7 @@ tasks:
     - func: "build static test tarball"
       vars:
         BUILD_FEATURES: "--features gcp-oidc"
+        TEST_FILES: "${PROJECT_DIRECTORY}/src/test/spec/json/auth/unified"
     - command: subprocess.exec
       type: test
       params:
@@ -1199,6 +1201,8 @@ tasks:
   - name: "oidc-auth-test-k8s-latest"
     commands:
     - func: "build static test tarball"
+      vars:
+        TEST_FILES: "${PROJECT_DIRECTORY}/src/test/spec/json/auth/unified"
     - command: ec2.assume_role
       params:
           role_arn: ${aws_test_secrets_role}
@@ -1927,6 +1931,7 @@ functions:
         include_expansions_in_env:
           - PROJECT_DIRECTORY
           - BUILD_FEATURES
+          - TEST_FILES
     - command: expansions.update
       params:
         file: src/static-test-tarball-expansion.yml

src/client/auth.rs

@@ -90,6 +90,7 @@ pub enum AuthMechanism {
     MongoDbAws,
 
     /// MONGODB-OIDC authenticates using [OpenID Connect](https://openid.net/developers/specs/) access tokens.
+    #[serde(alias = "MONGODB-OIDC")]
     MongoDbOidc,
 }
 

src/test.rs

@@ -48,6 +48,7 @@ pub(crate) use self::{
     },
 };
 
+use futures::FutureExt;
 use home::home_dir;
 use once_cell::sync::Lazy;
 use tokio::sync::OnceCell;
@@ -61,7 +62,13 @@ use crate::{
         options::{ServerApi, ServerApiVersion},
     },
     hello::HelloCommandResponse,
-    options::{ClientOptions, ServerAddress},
+    options::{
+        oidc::{Callback, IdpServerResponse},
+        AuthMechanism,
+        ClientOptions,
+        ServerAddress,
+    },
+    test::spec::oidc::get_access_token_test_user_1,
     Client,
 };
 use std::{fs::read_to_string, str::FromStr};
@@ -89,7 +96,20 @@ async fn get_test_client_metadata() -> &'static TestClientMetadata {
     static TEST_CLIENT_METADATA: OnceCell<TestClientMetadata> = OnceCell::const_new();
     TEST_CLIENT_METADATA
         .get_or_init(|| async {
-            let client = Client::for_test().await;
+            let mut client_options = get_client_options().await.clone();
+            // OIDC admin credentials are required to call getParameter when running with OIDC
+            // authentication.
+            if let (Ok(username), Ok(password)) = (
+                std::env::var("OIDC_ADMIN_USER"),
+                std::env::var("OIDC_ADMIN_PWD"),
+            ) {
+                let credential = Credential::builder()
+                    .username(username)
+                    .password(password)
+                    .build();
+                client_options.credential = Some(credential);
+            }
+            let client = Client::for_test().options(client_options).await;
 
             let build_info = client
                 .database("test")
@@ -265,6 +285,8 @@ pub(crate) static LOAD_BALANCED_SINGLE_URI: Lazy<Option<String>> =
     Lazy::new(|| std::env::var("SINGLE_MONGOS_LB_URI").ok());
 pub(crate) static LOAD_BALANCED_MULTIPLE_URI: Lazy<Option<String>> =
     Lazy::new(|| std::env::var("MULTI_MONGOS_LB_URI").ok());
+pub(crate) static OIDC_URI: Lazy<Option<String>> =
+    Lazy::new(|| std::env::var("MONGODB_URI_SINGLE").ok());
 pub(crate) static SERVERLESS_ATLAS_USER: Lazy<Option<String>> =
     Lazy::new(|| std::env::var("SERVERLESS_ATLAS_USER").ok());
 pub(crate) static SERVERLESS_ATLAS_PASSWORD: Lazy<Option<String>> =
@@ -308,6 +330,25 @@ pub(crate) fn update_options_for_testing(options: &mut ClientOptions) {
                 .build(),
         );
     }
+
+    if let Some(ref mut credential) = options.credential {
+        if credential.mechanism == Some(AuthMechanism::MongoDbOidc)
+            && credential
+                .mechanism_properties
+                .as_ref()
+                .map(|properties| properties.get("ENVIRONMENT").is_none())
+                .unwrap_or(true)
+        {
+            credential.oidc_callback = Callback::machine(move |_| {
+                async move {
+                    Ok(IdpServerResponse::builder()
+                        .access_token(get_access_token_test_user_1().await)
+                        .build())
+                }
+                .boxed()
+            });
+        }
+    }
 }
 
 fn get_default_uri() -> String {
@@ -316,6 +357,9 @@ fn get_default_uri() -> String {
             return uri;
         }
     }
+    if let Some(uri) = &*OIDC_URI {
+        return uri.clone();
+    }
     if let Ok(uri) = std::env::var("MONGODB_URI") {
         return uri;
     }

src/test/spec.rs

@@ -11,7 +11,7 @@ mod handshake;
 mod initial_dns_seedlist_discovery;
 mod load_balancers;
 #[path = "spec/oidc.rs"]
-mod oidc_skip_ci;
+pub(crate) mod oidc_skip_ci;
 mod read_write_concern;
 mod retryable_reads;
 mod retryable_writes;
@@ -37,25 +37,39 @@ use std::{
 use serde::{de::DeserializeOwned, Deserialize};
 
 pub(crate) use self::{
+    oidc_skip_ci as oidc,
     unified_runner::{merge_uri_options, ExpectedEventType, Topology},
     v2_runner::{operation::Operation, test_file::RunOn},
 };
 use crate::{bson::Bson, test::SERVERLESS};
 
 use super::log_uncaptured;
 
+pub(crate) fn deserialize_spec_tests_from_exact_path<T: DeserializeOwned>(
+    path: &[&str],
+    skipped_files: Option<&[&str]>,
+) -> Vec<(T, PathBuf)> {
+    deserialize_spec_tests_common(path.iter().collect(), skipped_files)
+}
+
 pub(crate) fn deserialize_spec_tests<T: DeserializeOwned>(
     spec: &[&str],
     skipped_files: Option<&[&str]>,
 ) -> Vec<(T, PathBuf)> {
-    let dir_path: PathBuf = [env!("CARGO_MANIFEST_DIR"), "src", "test", "spec", "json"]
+    let mut path: PathBuf = [env!("CARGO_MANIFEST_DIR"), "src", "test", "spec", "json"]
         .iter()
-        .chain(spec.iter())
         .collect();
+    path.extend(spec);
+    deserialize_spec_tests_common(path, skipped_files)
+}
 
+fn deserialize_spec_tests_common<T: DeserializeOwned>(
+    path: PathBuf,
+    skipped_files: Option<&[&str]>,
+) -> Vec<(T, PathBuf)> {
     let mut tests = vec![];
-    for entry in read_dir(&dir_path)
-        .unwrap_or_else(|e| panic!("Failed to read directory at {:?}: {}", &dir_path, e))
+    for entry in
+        read_dir(&path).unwrap_or_else(|e| panic!("Failed to read directory at {:?}: {}", &path, e))
     {
         let path = entry.unwrap().path();
         let Some(filename) = path

src/test/spec/auth.rs

@@ -109,5 +109,3 @@ async fn run_auth_test(test_file: TestFile) {
 async fn run_legacy() {
     run_spec_test(&["auth", "legacy"], run_auth_test).await;
 }
-
-// TODO RUST-1665: run unified tests

src/test/spec/oidc.rs

@@ -3,6 +3,11 @@ use std::path::PathBuf;
 use once_cell::sync::Lazy;
 use tokio::sync::OnceCell;
 
+use crate::{
+    bson::Bson,
+    test::spec::unified_runner::{TestFile, TestFileEntity},
+};
+
 static MONGODB_URI: Lazy<String> = Lazy::new(|| get_env_var("MONGODB_URI"));
 static MONGODB_URI_SINGLE: Lazy<String> = Lazy::new(|| get_env_var("MONGODB_URI_SINGLE"));
 #[cfg(target_os = "linux")]
@@ -30,7 +35,7 @@ async fn get_access_token_test_user(once_cell: &'static OnceCell<String>, user_n
         .await
         .to_string()
 }
-async fn get_access_token_test_user_1() -> String {
+pub(crate) async fn get_access_token_test_user_1() -> String {
     static ACCESS_TOKEN_TEST_USER_1: OnceCell<String> = OnceCell::const_new();
     get_access_token_test_user(&ACCESS_TOKEN_TEST_USER_1, 1).await
 }
@@ -44,11 +49,37 @@ fn get_env_var(var: &str) -> String {
     std::env::var(var).expect(var)
 }
 
+fn remove_mechanism_properties_placeholder(test_file: &mut TestFile) {
+    if let Some(ref mut create_entities) = test_file.create_entities {
+        for ref mut entity in create_entities {
+            if let TestFileEntity::Client(ref mut client) = entity {
+                if let Some(ref mut uri_options) = client.uri_options {
+                    if let Some(mut mechanism_properties) = uri_options
+                        .remove("authMechanismProperties")
+                        .and_then(|bson| match bson {
+                            Bson::Document(document) => Some(document),
+                            _ => None,
+                        })
+                    {
+                        mechanism_properties.remove("$$placeholder");
+                        if !mechanism_properties.is_empty() {
+                            uri_options.insert("authMechanismProperties", mechanism_properties);
+                        }
+                    }
+                }
+            }
+        }
+    }
+}
+
 mod basic {
     use crate::{
         client::auth::{oidc, AuthMechanism, Credential},
         options::ClientOptions,
-        test::util::fail_point::{FailPoint, FailPointMode},
+        test::{
+            spec::unified_runner::run_unified_tests,
+            util::fail_point::{FailPoint, FailPointMode},
+        },
         Client,
     };
     use bson::{doc, Document};
@@ -61,6 +92,7 @@ mod basic {
 
     use super::{
         get_access_token_test_user_1,
+        remove_mechanism_properties_placeholder,
         MONGODB_URI,
         MONGODB_URI_SINGLE,
         TEST_USER_1_USERNAME,
@@ -74,6 +106,13 @@ mod basic {
         TEST_USER_2_USERNAME,
     };
 
+    #[tokio::test(flavor = "multi_thread")]
+    async fn run_unified() {
+        run_unified_tests(&["auth", "unified"])
+            .transform_files(remove_mechanism_properties_placeholder)
+            .await;
+    }
+
     // Machine Callback tests
     #[tokio::test]
     async fn machine_1_1_callback_is_called() -> anyhow::Result<()> {
@@ -1281,10 +1320,25 @@ mod basic {
 }
 
 mod azure {
-    use crate::client::{options::ClientOptions, Client};
-    use bson::{doc, Document};
+    use crate::{
+        bson::{doc, Document},
+        client::{
+            auth::oidc::{AZURE_ENVIRONMENT_VALUE_STR, ENVIRONMENT_PROP_STR},
+            options::ClientOptions,
+            Client,
+        },
+        test::spec::unified_runner::run_unified_tests,
+    };
 
-    use super::MONGODB_URI_SINGLE;
+    use super::{remove_mechanism_properties_placeholder, MONGODB_URI_SINGLE};
+
+    #[tokio::test(flavor = "multi_thread")]
+    async fn run_unified() {
+        run_unified_tests(&["test_files"])
+            .transform_files(remove_mechanism_properties_placeholder)
+            .use_exact_path()
+            .await;
+    }
 
     #[tokio::test]
     async fn machine_5_1_azure_with_no_username() -> anyhow::Result<()> {
@@ -1318,8 +1372,6 @@ mod azure {
 
     #[tokio::test]
     async fn machine_5_3_token_resource_must_be_set_for_azure() -> anyhow::Result<()> {
-        use crate::client::auth::oidc::{AZURE_ENVIRONMENT_VALUE_STR, ENVIRONMENT_PROP_STR};
-
         let mut opts = ClientOptions::parse(&*MONGODB_URI_SINGLE).await?;
         opts.credential.as_mut().unwrap().mechanism_properties = Some(doc! {
             ENVIRONMENT_PROP_STR: AZURE_ENVIRONMENT_VALUE_STR,
@@ -1341,10 +1393,21 @@ mod azure {
 }
 
 mod gcp {
-    use crate::client::{options::ClientOptions, Client};
-    use bson::{doc, Document};
+    use crate::{
+        bson::{doc, Document},
+        client::{options::ClientOptions, Client},
+        test::spec::unified_runner::run_unified_tests,
+    };
 
-    use super::MONGODB_URI_SINGLE;
+    use super::{remove_mechanism_properties_placeholder, MONGODB_URI_SINGLE};
+
+    #[tokio::test(flavor = "multi_thread")]
+    async fn run_unified() {
+        run_unified_tests(&["test_files"])
+            .transform_files(remove_mechanism_properties_placeholder)
+            .use_exact_path()
+            .await;
+    }
 
     #[tokio::test]
     async fn machine_5_4_gcp_with_no_username() -> anyhow::Result<()> {
@@ -1385,23 +1448,15 @@ mod gcp {
 }
 
 mod k8s {
-    use crate::{
-        bson::{doc, Document},
-        Client,
-    };
+    use crate::test::spec::unified_runner::run_unified_tests;
 
-    use super::MONGODB_URI_SINGLE;
+    use super::remove_mechanism_properties_placeholder;
 
-    // There's no spec test for K8s, so we run this simple sanity check.
-    #[tokio::test]
-    async fn successfully_authenticates() -> anyhow::Result<()> {
-        let client = Client::with_uri_str(&*MONGODB_URI_SINGLE).await?;
-        client
-            .database("test")
-            .collection::<Document>("test")
-            .find_one(doc! {})
-            .await?;
-
-        Ok(())
+    #[tokio::test(flavor = "multi_thread")]
+    async fn run_unified() {
+        run_unified_tests(&["test_files"])
+            .transform_files(remove_mechanism_properties_placeholder)
+            .use_exact_path()
+            .await;
     }
 }