Support random number generation for recordings (Azure#2148)

* Support random number generation for recordings

Resolves Azure#2143

* Resolve PR feedback

Author: heaths

Cargo.lock

@@ -95,6 +95,7 @@ proc-macro2 = "1.0.86"
 quick-xml = { version = "0.31", features = ["serialize", "serde-types"] }
 quote = "1.0.37"
 rand = "0.8"
+rand_chacha = "0.3"
 reqwest = { version = "0.12", features = [
   "json",
   "stream",

Cargo.toml

@@ -19,6 +19,7 @@ paste
 pin-project
 quick-xml
 rand
+rand_chacha
 reqwest
 rustc_version
 serde
@@ -33,4 +34,4 @@ tracing
 tracing-subscriber
 tz-rs
 url
-uuid
+uuid

eng/dict/crates.txt

@@ -24,6 +24,8 @@ azure_core = { workspace = true, features = ["test"] }
 azure_core_test_macros.workspace = true
 azure_identity.workspace = true
 futures.workspace = true
+rand.workspace = true
+rand_chacha.workspace = true
 serde.workspace = true
 serde_json.workspace = true
 tracing.workspace = true
@@ -44,6 +46,8 @@ tokio = { workspace = true, features = [
 ] }
 
 [dev-dependencies]
+# Crate used in README.md example.
+azure_security_keyvault_secrets = { path = "../../keyvault/azure_security_keyvault_secrets" }
 clap.workspace = true
 tracing-subscriber = { workspace = true, features = ["env-filter", "fmt"] }
 

sdk/core/azure_core_test/Cargo.toml

@@ -7,12 +7,26 @@ The types and functions in this crate help test client libraries built on `azure
 To test client methods using our [Test Proxy] or run against live resources, you can attribute asynchronous tests
 using the `#[recorded::test]` attribute:
 
-```rust
+```rust no_run
 use azure_core::Result;
 use azure_core_test::{recorded, TestContext};
+use azure_security_keyvault_secrets::{SecretClient, SecretClientOptions};
 
 #[recorded::test]
 async fn get_secret(ctx: TestContext) -> Result<()> {
+    let recording = ctx.recording();
+
+    // Instrument your client options to hook up the test-proxy pipeline policy.
+    let mut options = SecretClientOptions::default();
+    recording.instrument(&mut options.client_options);
+
+    // Get variables, credentials, and pass the instrumented client options to the client to begin.
+    let client = SecretClient::new(
+        recording.var("AZURE_KEYVAULT_URL", None).as_str(),
+        recording.credential(),
+        Some(options),
+    )?;
+
     todo!()
 }
 ```
@@ -24,4 +38,61 @@ and provides other information to test functions that may be useful.
 
 These tests must also return a `std::result::Result<T, E>`, which can be redefined e.g., `azure_core::Result<T>`.
 
+### Recording features
+
+Besides instrumenting your client and getting credentials - real credentials when running live or recording,
+but mock credentials when playing back - there are a number of other helpful features on the `Recording` object returned above:
+
+* `add_sanitizer` will add custom sanitizers. There are many pre-configured by the [Test Proxy] as well.
+* `remove_sanitizers` will remove named sanitizers, like `AZSDK3430` that sanitizes all `$..id` fields and may cause playback to fail.
+* `add_matcher` adds a custom matcher to match headers, path segments, and or body content.
+* `random` gets random data (numbers, arrays, etc.) that is initialized from the OS when running live or recording,
+  but the seed is saved with the recording and used during play back so that sequential generation of random data is deterministic.
+  ChaCha20 is used to provide a deterministic, portable sequence of seeded random data.
+* `var` gets a required variable with optional `ValueOptions` you can use to sanitize values.
+  This function will err if the variable is not set in the environment when running live or recording, or available when playing back.
+* `var_opt` gets optional variables and will not err in the aforementioned cases.
+
+## Record tests
+
+Like with all our other Azure SDK languages, we use a common system for provisioning resources named [Test Resources].
+This uses a `test-resources.json` or `test-resources.bicep` file in the crate or service directory.
+
+When you run this, it will output some environment variables you can set in your shell, or pass on the command line if your shell supports it.
+
+```bash
+pwsh ./eng/common/TestResources/New-TestResources.ps1 -ServiceDirectory keyvault
+```
+
+In this example, it will output a number of environment variables including `AZURE_KEYVAULT_URL`. We'll pass that in the command line for bash below.
+
+Before you can record, though, you need to make sure you're authenticated. The script above will authenticate you in PowerShell, but the
+`AzurePowerShellCredential` is not yet implemented; however, the `AzureCliCredential` is so log in with `az`:
+
+```bash
+az login
+```
+
+No you can record your tests and, after they pass successfully, check in the test recordings. You need to pass `AZURE_TEST_MODE=record` to record,
+along with any other environment variables output by the [Test Resources] scripts.
+
+```bash
+AZURE_TEST_MODE=record AZURE_KEYVAULT_URL=https://my-vault.vault.azure.net cargo test -p azure_security_keyvault_secrets
+test-proxy push -a sdk/keyvault/assets.json
+```
+
+Once your assets are checked in by [Test Proxy], you can commit your local changes and create a PR.
+
+## Playing back tests
+
+To play back tests, just run `cargo test`:
+
+```bash
+cargo test
+```
+
+If you get errors, they could indicate regressions in your tests or perhaps variables or random data wasn't saved correctly.
+Review any data you generate or use not coming from the service.
+
 [Test Proxy]: https://github.com/Azure/azure-sdk-tools/blob/main/tools/test-proxy/Azure.Sdk.Tools.TestProxy/README.md
+[Test Resources]: https://github.com/Azure/azure-sdk-tools/blob/main/eng/common/TestResources/README.md

sdk/core/azure_core_test/README.md

@@ -3,6 +3,7 @@
 
 //! The [`Recording`] and other types used in recorded tests.
 
+// cspell:ignore csprng seedable
 use crate::{
     proxy::{
         client::{
@@ -16,19 +17,25 @@ use crate::{
     Matcher, MockCredential, Sanitizer,
 };
 use azure_core::{
+    base64,
     credentials::TokenCredential,
     error::ErrorKind,
     headers::{AsHeaders, HeaderName, HeaderValue},
     test::TestMode,
     ClientOptions, Header,
 };
 use azure_identity::DefaultAzureCredential;
+use rand::{
+    distributions::{Distribution, Standard},
+    Rng, SeedableRng,
+};
+use rand_chacha::ChaCha20Rng;
 use std::{
     borrow::Cow,
     cell::OnceCell,
     collections::HashMap,
     env,
-    sync::{Arc, RwLock},
+    sync::{Arc, Mutex, OnceLock, RwLock},
 };
 use tracing::span::EnteredSpan;
 
@@ -47,6 +54,7 @@ pub struct Recording {
     recording_assets_file: Option<String>,
     id: Option<RecordingId>,
     variables: RwLock<HashMap<String, Value>>,
+    rand: OnceLock<Mutex<ChaCha20Rng>>,
 }
 
 impl Recording {
@@ -127,6 +135,75 @@ impl Recording {
         options.per_try_policies.push(policy);
     }
 
+    /// Get random data from the OS or recording.
+    ///
+    /// This will always be the OS cryptographically secure pseudo-random number generator (CSPRNG) when running live.
+    /// When recording, it will initialize from the OS CSPRNG but save the seed value to the recording file.
+    /// When playing back, the saved seed value is read from the recording to reproduce the same sequence of random data.
+    ///
+    /// # Examples
+    ///
+    /// Generate a symmetric data encryption key (DEK).
+    ///
+    /// ```no_compile
+    /// let dek: [u8; 32] = recording.random();
+    /// ```
+    ///
+    /// # Panics
+    ///
+    /// Panics if the recording variables cannot be locked for reading or writing,
+    /// or if the random seed cannot be encoded or decoded properly.
+    ///
+    pub fn random<T>(&self) -> T
+    where
+        Standard: Distribution<T>,
+    {
+        const NAME: &str = "RandomSeed";
+
+        // Use ChaCha20 for a deterministic, portable CSPRNG.
+        let rng = self.rand.get_or_init(|| match self.test_mode {
+            TestMode::Live => ChaCha20Rng::from_entropy().into(),
+            TestMode::Playback => {
+                let variables = self
+                    .variables
+                    .read()
+                    .map_err(read_lock_error)
+                    .unwrap_or_else(|err| panic!("{err}"));
+                let seed: String = variables
+                    .get(NAME)
+                    .map(Into::into)
+                    .unwrap_or_else(|| panic!("random seed variable not set"));
+                let seed = base64::decode(seed)
+                    .unwrap_or_else(|err| panic!("failed to decode random seed: {err}"));
+                let seed = seed
+                    .first_chunk::<32>()
+                    .unwrap_or_else(|| panic!("insufficient random seed variable"));
+
+                ChaCha20Rng::from_seed(*seed).into()
+            }
+            TestMode::Record => {
+                let rng = ChaCha20Rng::from_entropy();
+                let seed = rng.get_seed();
+                let seed = base64::encode(seed);
+
+                let mut variables = self
+                    .variables
+                    .write()
+                    .map_err(write_lock_error)
+                    .unwrap_or_else(|err| panic!("{err}"));
+                variables.insert(NAME.to_string(), Value::from(Some(seed), None));
+
+                rng.into()
+            }
+        });
+
+        let Ok(mut rng) = rng.lock() else {
+            panic!("failed to lock RNG");
+        };
+
+        rng.gen()
+    }
+
     /// Removes the list of sanitizers from the recording.
     ///
     /// You can find a list of default sanitizers in [source code](https://github.com/Azure/azure-sdk-tools/blob/main/tools/test-proxy/Azure.Sdk.Tools.TestProxy/Common/SanitizerDictionary.cs).
@@ -229,6 +306,7 @@ impl Recording {
             recording_assets_file,
             id: None,
             variables: RwLock::new(HashMap::new()),
+            rand: OnceLock::new(),
         }
     }
 

sdk/core/azure_core_test/src/recording.rs

@@ -380,8 +380,7 @@ async fn wrap_key_unwrap_key(ctx: TestContext) -> Result<()> {
     let version = key.resource_id()?.version.unwrap_or_default();
 
     // Generate a data encryption key.
-    // TODO: Replace with recorded randomness similar to .NET's: https://github.com/Azure/azure-sdk-for-net/blob/fefa057116832364695ca010216f66f198182647/sdk/core/Azure.Core.TestFramework/src/TestRecording.cs#L165
-    let dek = b"17cf8194356442099ef7482b0f22340e".to_vec();
+    let dek = recording.random::<[u8; 32]>().to_vec();
 
     // Wrap the DEK.
     let mut parameters = KeyOperationsParameters {