diff --git a/doc/distributed-tracing-for-rust-service-clients.md b/doc/distributed-tracing-for-rust-service-clients.md index 25d8415923..9341fd2ba0 100644 --- a/doc/distributed-tracing-for-rust-service-clients.md +++ b/doc/distributed-tracing-for-rust-service-clients.md @@ -1,6 +1,3 @@ - - - # Distributed tracing options in Rust service clients ## Distributed tracing fundamentals @@ -149,7 +146,7 @@ to ```diff pub struct MyServiceClient { endpoint: Url, -+ tracer: std::sync::Arc, ++ tracer: std::sync::Arc, } ``` @@ -192,7 +189,7 @@ pub fn new( Vec::default(), vec![auth_policy], ), - }) + }) } ``` @@ -217,33 +214,33 @@ pub fn new( credential, vec!["https://vault.azure.net/.default"], )); -+ let tracer = -+ if let Some(tracer_options) = &options.client_options.instrumentation { -+ tracer_options -+ .tracer_provider -+ .as_ref() -+ .map(|tracer_provider| { -+ tracer_provider.get_tracer( -+ Some(#client_namespace), -+ option_env!("CARGO_PKG_NAME").unwrap_or("UNKNOWN"), -+ option_env!("CARGO_PKG_VERSION").unwrap_or("UNKNOWN"), -+ ) -+ }) -+ } else { -+ None -+ }; ++ let tracer = ++ if let Some(tracer_options) = &options.client_options.instrumentation { ++ tracer_options ++ .tracer_provider ++ .as_ref() ++ .map(|tracer_provider| { ++ tracer_provider.get_tracer( ++ Some(#client_namespace), ++ option_env!("CARGO_PKG_NAME").unwrap_or("UNKNOWN"), ++ option_env!("CARGO_PKG_VERSION").unwrap_or("UNKNOWN"), ++ ) ++ }) ++ } else { ++ None ++ }; Ok(Self { -+ tracer, - endpoint, - api_version: options.api_version, - pipeline: Pipeline::new( - option_env!("CARGO_PKG_NAME"), - option_env!("CARGO_PKG_VERSION"), - options.client_options, - Vec::default(), - vec![auth_policy], - ), - }) ++ tracer, + endpoint, + api_version: options.api_version, + pipeline: Pipeline::new( + option_env!("CARGO_PKG_NAME"), + option_env!("CARGO_PKG_VERSION"), + options.client_options, + Vec::default(), + vec![auth_policy], + ), + }) } ``` @@ -297,19 +294,19 @@ pub async fn get( path: &str, options: Option>, ) -> Result { -+ let options = { -+ let mut options = options.unwrap_or_default(); -+ let public_api_info = azure_core::tracing::PublicApiInstrumentationInformation { -+ api_name: "TestFunction", -+ attributes: Vec::new(), -+ }; -+ let mut ctx = options.method_options.context.with_value(public_api_info); -+ if let Some(tracer) = &self.tracer { -+ ctx = ctx.with_value(tracer.clone()); -+ } -+ options.method_options.context = ctx; -+ Some(options) -+ }; ++ let options = { ++ let mut options = options.unwrap_or_default(); ++ let public_api_info = azure_core::tracing::PublicApiInstrumentationInformation { ++ api_name: "TestFunction", ++ attributes: Vec::new(), ++ }; ++ let mut ctx = options.method_options.context.with_value(public_api_info); ++ if let Some(tracer) = &self.tracer { ++ ctx = ctx.with_value(tracer.clone()); ++ } ++ options.method_options.context = ctx; ++ Some(options) ++ }; let mut url = self.endpoint.clone(); url.set_path(path); url.query_pairs_mut() @@ -319,8 +316,8 @@ pub async fn get( let response = self .pipeline -- .send(&options.method_options.context, &mut request) -+ .send(&ctx, &mut request) +- .send(&options.method_options.context, &mut request) ++ .send(&ctx, &mut request) .await?; if !response.status().is_success() { return Err(azure_core::Error::message( @@ -340,7 +337,7 @@ implement per-service-client distributed tracing attributes. The parameters for the `tracing::function` roughly follow the following BNF: ```bnf -tracing_parameters = quoted_string [ ',' '( attribute_list ')'] +tracing_parameters = quoted_string [ ',' 'attributes' '=' '( attribute_list ')'] quoted_string = `"` `"` attribute_list = attribute | attribute [`,`] attribute_list attribute = key '=' value @@ -353,9 +350,9 @@ value = This means that the following are valid parameters for `tracing::function`: * `#[tracing::function("MyServiceClient.MyApi")]` - specifies a public API name. -* `#[tracing::function("Name", (az.namespace="namespace"))]` - specifies a public API name and creates a span with an attribute named "az.namespace" and a value of "namespace". -* `#[tracing::function("Name", (api_count=23, "my_attribute" = "Abc"))]` - specifies a public API name and creates a span with two attributes, one named "api_count" with a value of "23" and the other with the name "my_attribute" and a value of "Abc" -* `#[tracing::function("Name", ("API path"=path))]` - specifies a public API name and creates a span with an attribute named "API path" and the value of the parameter named "path". +* `#[tracing::function("Name", attributes = (az.namespace="namespace"))]` - specifies a public API name and creates a span with an attribute named "az.namespace" and a value of "namespace". +* `#[tracing::function("Name", attributes = (api_count=23, "my_attribute" = "Abc"))]` - specifies a public API name and creates a span with two attributes, one named "api_count" with a value of "23" and the other with the name "my_attribute" and a value of "Abc" +* `#[tracing::function("Name", attributes = ("API path"=path))]` - specifies a public API name and creates a span with an attribute named "API path" and the value of the parameter named "path". This allows a generator to pass in simple attribute annotations to the public API spans created by the pipeline. @@ -379,6 +376,7 @@ The client can then add whatever attributes to the span it needs, and after the Note that in this model, the client is responsible for ending the span. + ### Service implementations with "subclients" Service clients can sometimes contain "subclients" - clients which have their own pipelines and endpoint which contain subclient specific functionality. @@ -386,7 +384,6 @@ Service clients can sometimes contain "subclients" - clients which have their ow Such subclients often have an accessor function to create a new subclient instance which looks like this: ```rust - pub fn get_operation_templates_lro_client(&self) -> OperationTemplatesLroClient { OperationTemplatesLroClient { api_version: self.api_version.clone(), @@ -412,3 +409,83 @@ pub fn get_operation_templates_lro_client(&self) -> OperationTemplatesLroClient ``` This adds a clone of the parent client's `tracer` to the subclient - it functions similarly to `#[tracing::new]` but for subclient instantiation. + +## Verifying distributed tracing support + +The `azure_core_test::tracing` package provides functionality to allow developers to verify that their distributed tracing functionality is generating the expected tracing information. + +This functionality is driven from the `azure_core_test::tracing::assert_instrumentation_information` API. + +This function takes two closures and a structure which describes the expected API shape. The first closure is used to create an instance of the public API, the second actually executes the public API. + +As an example, here is a test for the `keyvault_secrets` package: + +```rust +assert_instrumentation_information( + // Create an instance of a SecretClient adding the instrumentation options to ensure distributed tracing is enabled. + |tracer_provider| { + let mut options = SecretClientOptions::default(); + recording.instrument(&mut options.client_options); + options.client_options.instrumentation = Some(InstrumentationOptions { + tracer_provider: Some(tracer_provider), + }); + SecretClient::new( + recording.var("AZURE_KEYVAULT_URL", None).as_str(), + recording.credential(), + Some(options), + ) + }, + // Perform a series of tests against the newly created SecretClient. + // In this case, there are two APIs called - set_secret and get_secret. + |client: SecretClient| { + Box::pin(async move { + // Set a secret. + let body = SetSecretParameters { + value: Some("secret-value-instrument".into()), + ..Default::default() + }; + let secret = client + .set_secret("secret-roundtrip-instrument", body.try_into()?, None) + .await? + .into_body() + .await?; + assert_eq!(secret.value, Some("secret-value-instrument".into())); + + // Get a specific version of a secret. + let version = secret.resource_id()?.version.unwrap_or_default(); + let secret = client + .get_secret("secret-roundtrip-instrument", version.as_ref(), None) + .await? + .into_body() + .await?; + assert_eq!(secret.value, Some("secret-value-instrument".into())); + Ok(()) + }) + }, + // Verify that the tests above generated appropriate distributed traces. + // First verify the package information and service client namespace. + // Next verify that two service APIs were called - one with the language + // independent name of "KeyVault.setSecret" (using the "PUT" HTTP verb), + // and the other with a language independent name of "KeyVault.getSecret" + // using the "get" verb. + ExpectedInstrumentation { + package_name: recording.var("CARGO_PKG_NAME", None), + package_version: recording.var("CARGO_PKG_VERSION", None), + package_namespace: Some("azure_security_keyvault_secrets"), + api_calls: vec![ + ExpectedApiInformation { + api_name: Some("KeyVault.setSecret"), + api_verb: azure_core::http::Method::Put, + ..Default::default() + }, + ExpectedApiInformation { + api_name: Some("KeyVault.getSecret"), + ..Default::default() + }, + ], + }, +) +.await?; +``` + +The first closure creates an instance of a KeyVault client. The second one executes a keyvault secrets round trip test setting a secret and retrieving the secret. In this case, the instrumentation information describes the name of the package and the namespace for the service client. It says that there will be two instrumented APIs being called - the first named `KeyVault.setSecret` and the second named `KeyVault.getSecret`. diff --git a/sdk/core/azure_core/src/http/pipeline.rs b/sdk/core/azure_core/src/http/pipeline.rs index 5fc0bc852f..8254aec073 100644 --- a/sdk/core/azure_core/src/http/pipeline.rs +++ b/sdk/core/azure_core/src/http/pipeline.rs @@ -57,8 +57,19 @@ impl Pipeline { let tracer = core_client_options .instrumentation .tracer_provider - .as_ref() .map(|provider| { + // Note that the choice to use "None" as the namespace here + // is intentional. + // The `azure_namespace` parameter is used to populate the `az.namespace` + // span attribute, however that information is only known by the author of the + // client library, not the core library. + // It is also *not* a constant that can be derived from the crate information - + // it is a value that is determined from the list of resource providers + // listed [here](https://learn.microsoft.com/azure/azure-resource-manager/management/azure-services-resource-providers). + // + // This information can only come from the package owner. It doesn't make sense + // to burden all users of the azure_core pipeline with determining this + // information, so we use `None` here. provider.get_tracer(None, crate_name.unwrap_or("Unknown"), crate_version) }); @@ -75,18 +86,6 @@ impl Pipeline { let mut per_try_policies = per_try_policies.clone(); if let Some(ref tracer) = tracer { - // Note that the choice to use "None" as the namespace here - // is intentional. - // The `azure_namespace` parameter is used to populate the `az.namespace` - // span attribute, however that information is only known by the author of the - // client library, not the core library. - // It is also *not* a constant that can be derived from the crate information - - // it is a value that is determined from the list of resource providers - // listed [here](https://learn.microsoft.com/azure/azure-resource-manager/management/azure-services-resource-providers). - // - // This information can only come from the package owner. It doesn't make sense - // to burden all users of the azure_core pipeline with determining this - // information, so we use `None` here. let request_instrumentation_policy = RequestInstrumentationPolicy::new(Some(tracer.clone())); push_unique(&mut per_try_policies, request_instrumentation_policy); diff --git a/sdk/core/azure_core_opentelemetry/assets.json b/sdk/core/azure_core_opentelemetry/assets.json index f0a1e04890..37db6277c0 100644 --- a/sdk/core/azure_core_opentelemetry/assets.json +++ b/sdk/core/azure_core_opentelemetry/assets.json @@ -1,6 +1,6 @@ { "AssetsRepo": "Azure/azure-sdk-assets", "AssetsRepoPrefixPath": "rust", - "Tag": "rust/azure_core_opentelemetry_2bc5efc404", + "Tag": "rust/azure_core_opentelemetry_341a93c45d", "TagPrefix": "rust/azure_core_opentelemetry" } \ No newline at end of file diff --git a/sdk/core/azure_core_opentelemetry/tests/telemetry_service_macros.rs b/sdk/core/azure_core_opentelemetry/tests/telemetry_service_macros.rs index 4762d6e411..30a0a40858 100644 --- a/sdk/core/azure_core_opentelemetry/tests/telemetry_service_macros.rs +++ b/sdk/core/azure_core_opentelemetry/tests/telemetry_service_macros.rs @@ -174,14 +174,21 @@ impl TestServiceClientWithMacros { mod tests { use super::*; use ::tracing::{info, trace}; - use azure_core::http::{ExponentialRetryOptions, RetryOptions}; - use azure_core::tracing::TracerProvider; - use azure_core::Result; - use azure_core_test::{recorded, TestContext}; + use azure_core::{ + http::{ExponentialRetryOptions, RetryOptions}, + tracing::TracerProvider, + Result, + }; + use azure_core_test::{ + recorded, + tracing::{ExpectedApiInformation, ExpectedInstrumentation}, + TestContext, + }; use opentelemetry::trace::{ SpanKind as OpenTelemetrySpanKind, Status as OpenTelemetrySpanStatus, }; use opentelemetry::Value as OpenTelemetryAttributeValue; + use typespec_client_core::http; fn create_exportable_tracer_provider() -> (Arc, InMemorySpanExporter) { let otel_exporter = InMemorySpanExporter::default(); @@ -665,4 +672,92 @@ mod tests { Ok(()) } + + #[recorded::test()] + async fn test_http_tracing_tests(ctx: TestContext) -> Result<()> { + let recording = ctx.recording(); + let package_name = recording.var("CARGO_PKG_NAME", None); + let package_version = recording.var("CARGO_PKG_VERSION", None); + azure_core_test::tracing::assert_instrumentation_information( + |tracer_provider| Ok(create_service_client(&ctx, tracer_provider)), + |client| { + let client = client; + Box::pin(async move { client.get("get", None).await }) + }, + ExpectedInstrumentation { + package_name, + package_version, + package_namespace: Some("Az.TestServiceClient"), + api_calls: vec![ExpectedApiInformation { + api_name: None, + ..Default::default() + }], + }, + ) + .await?; + + Ok(()) + } + + #[recorded::test()] + async fn test_function_tracing_tests(ctx: TestContext) -> Result<()> { + let recording = ctx.recording(); + let package_name = recording.var("CARGO_PKG_NAME", None); + let package_version = recording.var("CARGO_PKG_VERSION", None); + azure_core_test::tracing::assert_instrumentation_information( + |tracer_provider| Ok(create_service_client(&ctx, tracer_provider)), + |client| { + let client = client; + Box::pin(async move { client.get_with_function_tracing("get", None).await }) + }, + ExpectedInstrumentation { + package_name, + package_version, + package_namespace: Some("Az.TestServiceClient"), + api_calls: vec![ExpectedApiInformation { + api_name: Some("macros_get_with_tracing"), + additional_api_attributes: vec![ + ("a.b", 1.into()), + ("az.telemetry", "Abc".into()), + ("string attribute", "get".into()), + ], + ..Default::default() + }], + }, + ) + .await?; + + Ok(()) + } + #[recorded::test()] + async fn test_function_tracing_tests_error(ctx: TestContext) -> Result<()> { + let recording = ctx.recording(); + let package_name = recording.var("CARGO_PKG_NAME", None); + let package_version = recording.var("CARGO_PKG_VERSION", None); + azure_core_test::tracing::assert_instrumentation_information( + |tracer_provider| Ok(create_service_client(&ctx, tracer_provider)), + |client| { + let client = client; + Box::pin(async move { client.get_with_function_tracing("index.htm", None).await }) + }, + ExpectedInstrumentation { + package_name, + package_version, + package_namespace: Some("Az.TestServiceClient"), + api_calls: vec![ExpectedApiInformation { + api_name: Some("macros_get_with_tracing"), + expected_status_code: http::StatusCode::NotFound, + additional_api_attributes: vec![ + ("a.b", 1.into()), + ("az.telemetry", "Abc".into()), + ("string attribute", "index.htm".into()), + ], + ..Default::default() + }], + }, + ) + .await?; + + Ok(()) + } } diff --git a/sdk/core/azure_core_test/src/tracing.rs b/sdk/core/azure_core_test/src/tracing.rs index f2cfca01e6..33bd325238 100644 --- a/sdk/core/azure_core_test/src/tracing.rs +++ b/sdk/core/azure_core_test/src/tracing.rs @@ -2,8 +2,12 @@ // Licensed under the MIT License. // cspell: ignore traceparent -use std::sync::{Arc, Mutex}; -use tracing::trace; + +//! This module contains a set of tests to help verify correctness of the Distributed Tracing implementation, and correctness of service client implementations of Distributed Tracing. +use std::{ + pin::Pin, + sync::{Arc, Mutex}, +}; use typespec_client_core::{ http::{headers::HeaderName, Context, Request}, tracing::{ @@ -11,12 +15,14 @@ use typespec_client_core::{ }, }; +/// Mock Tracing Provider - used for testing distributed tracing without involving a specific tracing implementation. #[derive(Debug)] pub struct MockTracingProvider { tracers: Mutex>>, } impl MockTracingProvider { + /// Instantiate a new instance of a Mock Tracing Provider. pub fn new() -> Self { Self { tracers: Mutex::new(Vec::new()), @@ -49,7 +55,7 @@ impl TracerProvider for MockTracingProvider { tracer } } - +/// Mock Tracer - used for testing distributed tracing without involving a specific tracing implementation. #[derive(Debug)] pub struct MockTracer { pub namespace: Option<&'static str>, @@ -94,6 +100,7 @@ impl Tracer for MockTracer { } } +/// Mock span for testing purposes. #[derive(Debug)] pub struct MockSpan { pub name: String, @@ -104,9 +111,8 @@ pub struct MockSpan { } impl MockSpan { fn new(name: &str, kind: SpanKind, attributes: Vec) -> Self { - println!("Creating MockSpan: {}", name); - println!("Attributes: {:?}", attributes); - println!("Converted attributes: {:?}", attributes); + eprintln!("Creating MockSpan: {}", name); + eprintln!("Attributes: {:?}", attributes); Self { name: name.to_string(), kind, @@ -119,7 +125,7 @@ impl MockSpan { impl Span for MockSpan { fn set_attribute(&self, key: &'static str, value: AttributeValue) { - println!("{}: Setting attribute {}: {:?}", self.name, key, value); + eprintln!("{}: Setting attribute {}: {:?}", self.name, key, value); let mut attributes = self.attributes.lock().unwrap(); attributes.push(Attribute { key: key.into(), @@ -128,13 +134,13 @@ impl Span for MockSpan { } fn set_status(&self, status: crate::tracing::SpanStatus) { - println!("{}: Setting span status: {:?}", self.name, status); + eprintln!("{}: Setting span status: {:?}", self.name, status); let mut state = self.state.lock().unwrap(); *state = status; } fn end(&self) { - println!("Ending span: {}", self.name); + eprintln!("Ending span: {}", self.name); let mut is_open = self.is_open.lock().unwrap(); *is_open = false; } @@ -174,34 +180,48 @@ impl AsAny for MockSpan { } } +/// Expected information about a tracer. #[derive(Debug)] pub struct ExpectedTracerInformation<'a> { + /// Expected name for the tracer. pub name: &'a str, + + /// Expected version for the tracer. pub version: Option<&'a str>, + + /// Expected namespace for the tracer. pub namespace: Option<&'a str>, - pub spans: Vec>, -} -#[derive(Debug)] -pub struct ExpectedSpanInformation<'a> { - pub span_name: &'a str, - pub status: SpanStatus, - pub kind: SpanKind, - pub attributes: Vec<(&'a str, AttributeValue)>, + /// A set of spans which should have been generated. + pub spans: Vec>, } +/// Checks the instrumentation result against the expected tracers. +/// +/// Used to verify that the mock tracer has recorded the expected spans and attributes. Primarily +/// intended for use in unit tests of the distributed tracing functionality. +/// +/// # Arguments +/// - `mock_tracer`: The mock tracer instance that contains the recorded spans. +/// - `expected_tracers`: The expected tracer information to compare against. pub fn check_instrumentation_result( mock_tracer: Arc, expected_tracers: Vec>, ) { + let tracers = mock_tracer.tracers.lock().unwrap(); + if tracers.len() != expected_tracers.len() { + eprintln!("Expected tracers: {:?}", expected_tracers); + eprintln!("Found tracers: {:?}", tracers); + } assert_eq!( - mock_tracer.tracers.lock().unwrap().len(), + tracers.len(), expected_tracers.len(), - "Unexpected number of tracers", + "Unexpected number of tracers, expected: {}, found: {}", + expected_tracers.len(), + tracers.len() ); - let tracers = mock_tracer.tracers.lock().unwrap(); for (index, expected) in expected_tracers.iter().enumerate() { - trace!("Checking tracer {}: {}", index, expected.name); + eprintln!("Checking tracer {}: {}", index, expected.name); let tracer = &tracers[index]; assert_eq!(tracer.package_name, expected.name); assert_eq!(tracer.package_version, expected.version); @@ -216,7 +236,7 @@ pub fn check_instrumentation_result( ); for (span_index, span_expected) in expected.spans.iter().enumerate() { - println!( + eprintln!( "Checking span {} of tracer {}: {}", span_index, expected.name, span_expected.span_name ); @@ -225,17 +245,37 @@ pub fn check_instrumentation_result( } } +/// Information about an expected span. Used to assert span properties. +#[derive(Debug)] +pub struct ExpectedSpanInformation<'a> { + /// The expected name of the span. + pub span_name: &'a str, + /// The expected status of the span. + pub status: SpanStatus, + + /// The expected kind of the span. + pub kind: SpanKind, + + /// The expected attributes associated with the span. + pub attributes: Vec<(&'a str, AttributeValue)>, +} + fn check_span_information(span: &Arc, expected: &ExpectedSpanInformation<'_>) { assert_eq!(span.name, expected.span_name); assert_eq!(span.kind, expected.kind); assert_eq!(*span.state.lock().unwrap(), expected.status); let attributes = span.attributes.lock().unwrap(); + eprintln!("Expected attributes: {:?}", expected.attributes); + eprintln!("Found attributes: {:?}", attributes); for (index, attr) in attributes.iter().enumerate() { - println!("Attribute {}: {} = {:?}", index, attr.key, attr.value); + eprintln!("Attribute {}: {} = {:?}", index, attr.key, attr.value); let mut found = false; for (key, value) in &expected.attributes { if attr.key == *key { - assert_eq!(attr.value, *value, "Attribute mismatch for key: {}", key); + // Skip checking the value for "" as it is a placeholder + if *value != AttributeValue::String("".into()) { + assert_eq!(attr.value, *value, "Attribute mismatch for key: {}", *key); + } found = true; break; } @@ -245,11 +285,200 @@ fn check_span_information(span: &Arc, expected: &ExpectedSpanInformati } } for (key, value) in expected.attributes.iter() { - if !attributes - .iter() - .any(|attr| attr.key == *key && attr.value == *value) - { + if !attributes.iter().any(|attr| attr.key == *key) { panic!("Expected attribute not found: {} = {:?}", key, value); } } } + +/// Information about an instrumented API call. +/// +/// This structure is used to collect information about a specific API call that is being instrumented for tracing. +/// +/// It provides hooks which can be use to verify the expected HTTP result type for the API call, and provides the ability +/// to register any service client specific public API attributes which will be generated during the API call. +#[derive(Debug, Clone)] +pub struct ExpectedApiInformation { + /// The name of the API being called. + /// + /// This is the name of the API as it appears in the service documentation. If `None`, it means that + /// public API instrumentation is not enabled for this API, and the test will only look for request + /// instrumentation spans. + pub api_name: Option<&'static str>, + + /// The HTTP verb used in the API request. + pub api_verb: azure_core::http::Method, + + /// Expected status code returned by the service. + pub expected_status_code: azure_core::http::StatusCode, + + /// A set of optional additional attributes attached to the public API span for service clients which require them. + /// If the attribute value has the `` placeholder, it means that the test should accept any value for that attribute. + pub additional_api_attributes: Vec<(&'static str, AttributeValue)>, +} + +impl Default for ExpectedApiInformation { + fn default() -> Self { + Self { + api_name: None, + api_verb: azure_core::http::Method::Get, + expected_status_code: azure_core::http::StatusCode::Ok, + additional_api_attributes: Vec::new(), + } + } +} + +/// Information about an instrumented package calling the `test_instrumentation_for_api` test. +#[derive(Debug, Default, Clone)] +pub struct ExpectedInstrumentation { + /// The package name for the service client. + pub package_name: String, + /// The package version for the service client. + pub package_version: String, + /// The namespace for the service client. + pub package_namespace: Option<&'static str>, + /// Individual instrumented API calls from the test function. + pub api_calls: Vec, +} + +/// Tests the instrumentation of a service client API call. +/// +/// Asserts that the generated distributed tracing information for a particular API matches the expected shape of the API. +/// +/// # Arguments +/// - `create_client`: A function to create the service client. +/// - `test_api`: A function to test the API call. +/// - `api_information`: Information about the API call being tested. +/// +/// This function will call the `create_client` function to create a new instance of the service client. It is the responsibility of the `create_client` callback to +/// add the provided distributed tracing `TracerProvider` to the newly created service client. +/// +/// Once the client has been created, it will call the `test_api` function to test the API call(s). +/// +/// After the APIs have been tested, this function will verify that the expected tracing spans were created. +/// +/// To do that, it uses the `[ExpectedInstrumentation]` structure to collect and compare the actual spans generated during the test. +/// +/// The code does not verify the actual client URLs or server ports, it only verifies that the relevant attributes are created. +/// +/// The `test_api` call may issue multiple service client calls, if it does, this function will verify that all expected spans were created. The caller of the `test_instrumentation_for_api` call +/// should make sure to include all expected APIs in the call. +/// +pub async fn assert_instrumentation_information( + create_client: FnInit, + test_api: FnTest, + api_information: ExpectedInstrumentation, +) -> azure_core::Result<()> +where + FnInit: FnOnce(Arc) -> azure_core::Result, + FnTest: FnOnce(C) -> Pin>>>, +{ + // Initialize the mock tracer provider + let mock_tracer = Arc::new(MockTracingProvider::new()); + + // Create a client with the mock tracer + let client = create_client(mock_tracer.clone())?; + + // We don't actually care about the result of the API call - just that it was made. + let _ = test_api(client).await; + + // There will be two tracers generated - one for public APIs, the second for HTTP calls. + // + // If there are public API instrumentation spans, we will see the public API and HTTP API traces on + // the public API tracer. + let mut public_api_tracer = ExpectedTracerInformation { + name: api_information.package_name.as_str(), + version: Some(api_information.package_version.as_str()), + namespace: api_information.package_namespace, + spans: Vec::new(), + }; + + // If there are no public API spans in the API call, they will appear on the Request Activity Tracer. + let mut request_activity_tracer = ExpectedTracerInformation { + name: api_information.package_name.as_str(), + version: Some(api_information.package_version.as_str()), + namespace: None, + spans: Vec::new(), + }; + + // Iterate over the expected API calls calculating the expected spans which will be created. + for api_call in api_information.api_calls.iter() { + let mut expected_spans = Vec::new(); + + let mut public_api_attributes = api_call.additional_api_attributes.clone(); + // Add additional attributes as needed. + if let Some(namespace) = api_information.package_namespace { + public_api_attributes.push(("az.namespace", namespace.into())); + } + if !api_call.expected_status_code.is_success() { + public_api_attributes.push(( + "error.type", + api_call.expected_status_code.to_string().into(), + )); + } + + if let Some(api_name) = api_call.api_name { + // Public API spans only enter the Error state if the status code is a server error. + expected_spans.push(ExpectedSpanInformation { + span_name: api_name, + status: if api_call.expected_status_code.is_server_error() { + SpanStatus::Error { + description: "".into(), + } + } else { + SpanStatus::Unset + }, + kind: SpanKind::Internal, + attributes: public_api_attributes, + }); + } + + // Add the HTTP API span after creating the expected set of attributes. + let mut http_request_attributes = vec![ + ("http.request.method", api_call.api_verb.as_str().into()), + ("url.full", "".into()), + ("server.address", "".into()), + ("server.port", "".into()), + ("az.client_request_id", "".into()), + ( + "http.response.status_code", + (*api_call.expected_status_code).into(), + ), + ]; + if !api_call.expected_status_code.is_success() { + http_request_attributes.push(( + "error.type", + api_call.expected_status_code.to_string().into(), + )); + } + // If we have no public API information, we won't have a namespace in the HTTP attributes. + if api_call.api_name.is_some() && api_information.package_namespace.is_some() { + http_request_attributes.push(( + "az.namespace", + api_information.package_namespace.unwrap().into(), + )); + } + expected_spans.push(ExpectedSpanInformation { + span_name: api_call.api_verb.as_str(), + status: if !api_call.expected_status_code.is_success() { + SpanStatus::Error { + description: "".into(), + } + } else { + SpanStatus::Unset + }, + kind: SpanKind::Client, + attributes: http_request_attributes, + }); + if api_call.api_name.is_some() { + public_api_tracer.spans.extend(expected_spans); + } else { + request_activity_tracer.spans.extend(expected_spans); + } + } + let expected_tracers = vec![public_api_tracer, request_activity_tracer]; + + check_instrumentation_result(mock_tracer, expected_tracers); + + Ok(()) +} diff --git a/sdk/keyvault/assets.json b/sdk/keyvault/assets.json index 14eff2710b..4bbed3eee0 100644 --- a/sdk/keyvault/assets.json +++ b/sdk/keyvault/assets.json @@ -2,5 +2,5 @@ "AssetsRepo": "Azure/azure-sdk-assets", "AssetsRepoPrefixPath": "rust", "TagPrefix": "rust/keyvault", - "Tag": "rust/keyvault_6c9f540438" + "Tag": "rust/keyvault_c904be268a" } diff --git a/sdk/keyvault/azure_security_keyvault_secrets/tests/secret_client.rs b/sdk/keyvault/azure_security_keyvault_secrets/tests/secret_client.rs index 554317fc24..9ffcf610f3 100644 --- a/sdk/keyvault/azure_security_keyvault_secrets/tests/secret_client.rs +++ b/sdk/keyvault/azure_security_keyvault_secrets/tests/secret_client.rs @@ -3,8 +3,15 @@ #![cfg_attr(target_arch = "wasm32", allow(unused_imports))] -use azure_core::{http::StatusCode, Result}; -use azure_core_test::{recorded, TestContext, TestMode}; +use azure_core::{ + http::{InstrumentationOptions, StatusCode}, + Result, +}; +use azure_core_test::{ + recorded, + tracing::{ExpectedApiInformation, ExpectedInstrumentation}, + TestContext, TestMode, +}; use azure_security_keyvault_secrets::{ models::{SetSecretParameters, UpdateSecretPropertiesParameters}, ResourceExt as _, SecretClient, SecretClientOptions, @@ -200,3 +207,68 @@ async fn purge_secret(ctx: TestContext) -> Result<()> { Ok(()) } + +#[recorded::test] +async fn round_trip_secret_verify_telemetry(ctx: TestContext) -> Result<()> { + let recording = ctx.recording(); + + // Verify that the distributed tracing traces generated from the API call below match the expected traces. + azure_core_test::tracing::assert_instrumentation_information( + |tracer_provider| { + let mut options = SecretClientOptions::default(); + recording.instrument(&mut options.client_options); + options.client_options.instrumentation = Some(InstrumentationOptions { + tracer_provider: Some(tracer_provider), + }); + SecretClient::new( + recording.var("AZURE_KEYVAULT_URL", None).as_str(), + recording.credential(), + Some(options), + ) + }, + |client: SecretClient| { + Box::pin(async move { + // Set a secret. + let body = SetSecretParameters { + value: Some("secret-value-instrument".into()), + ..Default::default() + }; + let secret = client + .set_secret("secret-roundtrip-instrument", body.try_into()?, None) + .await? + .into_body() + .await?; + assert_eq!(secret.value, Some("secret-value-instrument".into())); + + // Get a specific version of a secret. + let version = secret.resource_id()?.version.unwrap_or_default(); + let secret = client + .get_secret("secret-roundtrip-instrument", version.as_ref(), None) + .await? + .into_body() + .await?; + assert_eq!(secret.value, Some("secret-value-instrument".into())); + Ok(()) + }) + }, + ExpectedInstrumentation { + package_name: recording.var("CARGO_PKG_NAME", None), + package_version: recording.var("CARGO_PKG_VERSION", None), + package_namespace: Some("azure_security_keyvault_secrets"), + api_calls: vec![ + ExpectedApiInformation { + api_name: Some("KeyVault.setSecret"), + api_verb: azure_core::http::Method::Put, + ..Default::default() + }, + ExpectedApiInformation { + api_name: Some("KeyVault.getSecret"), + ..Default::default() + }, + ], + }, + ) + .await?; + + Ok(()) +}