Added ability to test distributed tracing functionality.

doc/distributed-tracing-for-rust-service-clients.md

@@ -340,7 +340,7 @@ implement per-service-client distributed tracing attributes.
 The parameters for the `tracing::function` roughly follow the following BNF:
 
 ```bnf
-tracing_parameters = quoted_string [ ',' '( attribute_list ')']
+tracing_parameters = quoted_string [ ',' 'attributes' '=' '( attribute_list ')']
 quoted_string = `"` <characters> `"`
 attribute_list = attribute | attribute [`,`] attribute_list
 attribute = key '=' value
@@ -353,9 +353,9 @@ value = <any Rust expression>
 This means that the following are valid parameters for `tracing::function`:
 
 * `#[tracing::function("MyServiceClient.MyApi")]` - specifies a public API name.
-* `#[tracing::function("Name", (az.namespace="namespace"))]` - specifies a public API name and creates a span with an attribute named "az.namespace" and a value of "namespace".
-* `#[tracing::function("Name", (api_count=23, "my_attribute" = "Abc"))]` - specifies a public API name and creates a span with two attributes, one named  "api_count" with a value of "23" and the other with the name "my_attribute" and a value of "Abc"
-* `#[tracing::function("Name", ("API path"=path))]` - specifies a public API name and creates a span with an attribute named "API path" and the value of the parameter named "path".
+* `#[tracing::function("Name", attributes = (az.namespace="namespace"))]` - specifies a public API name and creates a span with an attribute named "az.namespace" and a value of "namespace".
+* `#[tracing::function("Name", attributes = (api_count=23, "my_attribute" = "Abc"))]` - specifies a public API name and creates a span with two attributes, one named  "api_count" with a value of "23" and the other with the name "my_attribute" and a value of "Abc"
+* `#[tracing::function("Name", attributes = ("API path"=path))]` - specifies a public API name and creates a span with an attribute named "API path" and the value of the parameter named "path".
 
 This allows a generator to pass in simple attribute annotations to the public API spans created by the pipeline.
 
@@ -412,3 +412,83 @@ pub fn get_operation_templates_lro_client(&self) -> OperationTemplatesLroClient
 ```
 
 This adds a clone of the parent client's `tracer` to the subclient - it functions similarly to `#[tracing::new]` but for subclient instantiation.
+
+## Verifying distributed tracing support
+
+The `azure_core_test::tracing` package provides functionality to allow developers to verify that their distributed tracing functionality is generating the expected tracing information.
+
+This functionality is driven from the `azure_core_test::tracing::assert_instrumentation_information` API.
+
+This function takes two closures and a structure which describes the expected API shape. The first closure is used to create an instance of the public API, the second actually executes the public API.
+
+As an example, here is a test for the `keyvault_secrets` package:
+
+```rust
+assert_instrumentation_information(
+    // Create an instance of a SecretClient adding the instrumentation options to ensure distributed tracing is enabled.
+    |tracer_provider| {
+        let mut options = SecretClientOptions::default();
+        recording.instrument(&mut options.client_options);
+        options.client_options.instrumentation = Some(InstrumentationOptions {
+            tracer_provider: Some(tracer_provider),
+        });
+        SecretClient::new(
+            recording.var("AZURE_KEYVAULT_URL", None).as_str(),
+            recording.credential(),
+            Some(options),
+        )
+    },
+    // Perform a series of tests against the newly created SecretClient.
+    // In this case, there are two APIs called - set_secret and get_secret.
+    |client: SecretClient| {
+        Box::pin(async move {
+            // Set a secret.
+            let body = SetSecretParameters {
+                value: Some("secret-value-instrument".into()),
+                ..Default::default()
+            };
+            let secret = client
+                .set_secret("secret-roundtrip-instrument", body.try_into()?, None)
+                .await?
+                .into_body()
+                .await?;
+            assert_eq!(secret.value, Some("secret-value-instrument".into()));
+
+            // Get a specific version of a secret.
+            let version = secret.resource_id()?.version.unwrap_or_default();
+            let secret = client
+                .get_secret("secret-roundtrip-instrument", version.as_ref(), None)
+                .await?
+                .into_body()
+                .await?;
+            assert_eq!(secret.value, Some("secret-value-instrument".into()));
+            Ok(())
+        })
+    },
+    // Verify that the tests above generated appropriate distributed traces.
+    // First verify the package information and service client namespace.
+    // Next verify that two service APIs were called - one with the language
+    // independent name of "KeyVault.setSecret" (using the "PUT" HTTP verb),
+    // and the other with a language independent name of "KeyVault.getSecret"
+    // using the "get" verb.
+    ExpectedInstrumentation {
+        package_name: recording.var("CARGO_PKG_NAME", None),
+        package_version: recording.var("CARGO_PKG_VERSION", None),
+        package_namespace: Some("azure_security_keyvault_secrets"),
+        api_calls: vec![
+            ExpectedApiInformation {
+                api_name: Some("KeyVault.setSecret"),
+                api_verb: azure_core::http::Method::Put,
+                ..Default::default()
+            },
+            ExpectedApiInformation {
+                api_name: Some("KeyVault.getSecret"),
+                ..Default::default()
+            },
+        ],
+    },
+)
+.await?;
+```
+
+The first closure creates an instance of a KeyVault client. The second one executes a keyvault secrets round trip test setting a secret and retrieving the secret. In this case, the instrumentation information describes the name of the package and the namespace for the service client. It says that there will be two instrumented APIs being called - the first named `KeyVault.setSecret` and the second named `KeyVault.getSecret`.

sdk/core/azure_core/src/http/pipeline.rs

@@ -57,8 +57,19 @@ impl Pipeline {
         let tracer = core_client_options
             .instrumentation
             .tracer_provider
-            .as_ref()
             .map(|provider| {
+                // Note that the choice to use "None" as the namespace here
+                // is intentional.
+                // The `azure_namespace` parameter is used to populate the `az.namespace`
+                // span attribute, however that information is only known by the author of the
+                // client library, not the core library.
+                // It is also *not* a constant that can be derived from the crate information -
+                // it is a value that is determined from the list of resource providers
+                // listed [here](https://learn.microsoft.com/azure/azure-resource-manager/management/azure-services-resource-providers).
+                //
+                // This information can only come from the package owner. It doesn't make sense
+                // to burden all users of the azure_core pipeline with determining this
+                // information, so we use `None` here.
                 provider.get_tracer(None, crate_name.unwrap_or("Unknown"), crate_version)
             });
 
@@ -75,18 +86,6 @@ impl Pipeline {
 
         let mut per_try_policies = per_try_policies.clone();
         if let Some(ref tracer) = tracer {
-            // Note that the choice to use "None" as the namespace here
-            // is intentional.
-            // The `azure_namespace` parameter is used to populate the `az.namespace`
-            // span attribute, however that information is only known by the author of the
-            // client library, not the core library.
-            // It is also *not* a constant that can be derived from the crate information -
-            // it is a value that is determined from the list of resource providers
-            // listed [here](https://learn.microsoft.com/azure/azure-resource-manager/management/azure-services-resource-providers).
-            //
-            // This information can only come from the package owner. It doesn't make sense
-            // to burden all users of the azure_core pipeline with determining this
-            // information, so we use `None` here.
             let request_instrumentation_policy =
                 RequestInstrumentationPolicy::new(Some(tracer.clone()));
             push_unique(&mut per_try_policies, request_instrumentation_policy);

sdk/core/azure_core_opentelemetry/assets.json

@@ -1,6 +1,6 @@
 {
   "AssetsRepo": "Azure/azure-sdk-assets",
   "AssetsRepoPrefixPath": "rust",
-  "Tag": "rust/azure_core_opentelemetry_2bc5efc404",
+  "Tag": "rust/azure_core_opentelemetry_341a93c45d",
   "TagPrefix": "rust/azure_core_opentelemetry"
 }

sdk/core/azure_core_opentelemetry/tests/telemetry_service_macros.rs

@@ -174,14 +174,21 @@ impl TestServiceClientWithMacros {
 mod tests {
     use super::*;
     use ::tracing::{info, trace};
-    use azure_core::http::{ExponentialRetryOptions, RetryOptions};
-    use azure_core::tracing::TracerProvider;
-    use azure_core::Result;
-    use azure_core_test::{recorded, TestContext};
+    use azure_core::{
+        http::{ExponentialRetryOptions, RetryOptions},
+        tracing::TracerProvider,
+        Result,
+    };
+    use azure_core_test::{
+        recorded,
+        tracing::{ExpectedApiInformation, ExpectedInstrumentation},
+        TestContext,
+    };
     use opentelemetry::trace::{
         SpanKind as OpenTelemetrySpanKind, Status as OpenTelemetrySpanStatus,
     };
     use opentelemetry::Value as OpenTelemetryAttributeValue;
+    use typespec_client_core::http;
 
     fn create_exportable_tracer_provider() -> (Arc<SdkTracerProvider>, InMemorySpanExporter) {
         let otel_exporter = InMemorySpanExporter::default();
@@ -514,6 +521,7 @@ mod tests {
             },
             ..Default::default()
         };
+        //        recording.instrument(&mut options.client_options);
 
         let client = TestServiceClientWithMacros::new(endpoint, credential, Some(options)).unwrap();
         let response = client.get_with_function_tracing("failing_url", None).await;
@@ -665,4 +673,92 @@ mod tests {
 
         Ok(())
     }
+
+    #[recorded::test()]
+    async fn test_http_tracing_tests(ctx: TestContext) -> Result<()> {
+        let recording = ctx.recording();
+        let package_name = recording.var("CARGO_PKG_NAME", None);
+        let package_version = recording.var("CARGO_PKG_VERSION", None);
+        azure_core_test::tracing::assert_instrumentation_information(
+            |tracer_provider| Ok(create_service_client(&ctx, tracer_provider)),
+            |client| {
+                let client = client;
+                Box::pin(async move { client.get("get", None).await })
+            },
+            ExpectedInstrumentation {
+                package_name,
+                package_version,
+                package_namespace: Some("Az.TestServiceClient"),
+                api_calls: vec![ExpectedApiInformation {
+                    api_name: None,
+                    ..Default::default()
+                }],
+            },
+        )
+        .await?;
+
+        Ok(())
+    }
+
+    #[recorded::test()]
+    async fn test_function_tracing_tests(ctx: TestContext) -> Result<()> {
+        let recording = ctx.recording();
+        let package_name = recording.var("CARGO_PKG_NAME", None);
+        let package_version = recording.var("CARGO_PKG_VERSION", None);
+        azure_core_test::tracing::assert_instrumentation_information(
+            |tracer_provider| Ok(create_service_client(&ctx, tracer_provider)),
+            |client| {
+                let client = client;
+                Box::pin(async move { client.get_with_function_tracing("get", None).await })
+            },
+            ExpectedInstrumentation {
+                package_name,
+                package_version,
+                package_namespace: Some("Az.TestServiceClient"),
+                api_calls: vec![ExpectedApiInformation {
+                    api_name: Some("macros_get_with_tracing"),
+                    additional_api_attributes: vec![
+                        ("a.b", 1.into()),
+                        ("az.telemetry", "Abc".into()),
+                        ("string attribute", "get".into()),
+                    ],
+                    ..Default::default()
+                }],
+            },
+        )
+        .await?;
+
+        Ok(())
+    }
+    #[recorded::test()]
+    async fn test_function_tracing_tests_error(ctx: TestContext) -> Result<()> {
+        let recording = ctx.recording();
+        let package_name = recording.var("CARGO_PKG_NAME", None);
+        let package_version = recording.var("CARGO_PKG_VERSION", None);
+        azure_core_test::tracing::assert_instrumentation_information(
+            |tracer_provider| Ok(create_service_client(&ctx, tracer_provider)),
+            |client| {
+                let client = client;
+                Box::pin(async move { client.get_with_function_tracing("index.htm", None).await })
+            },
+            ExpectedInstrumentation {
+                package_name,
+                package_version,
+                package_namespace: Some("Az.TestServiceClient"),
+                api_calls: vec![ExpectedApiInformation {
+                    api_name: Some("macros_get_with_tracing"),
+                    expected_status_code: http::StatusCode::NotFound,
+                    additional_api_attributes: vec![
+                        ("a.b", 1.into()),
+                        ("az.telemetry", "Abc".into()),
+                        ("string attribute", "index.htm".into()),
+                    ],
+                    ..Default::default()
+                }],
+            },
+        )
+        .await?;
+
+        Ok(())
+    }
 }