Align spans with SRA and add tests (#4052)

## Motivation and Context
<!--- Why is this change required? What problem does it solve? -->
<!--- If it fixes an open issue, please link to the issue here -->
Update our tracing spans to better align with the SRA.

## Description
<!--- Describe your changes in detail -->
* Add top level `{ServiceName}.{OperationName}` span with various SRA
mandated attributes
* Make all spans debug level (was previously mix of debug and info)
* Add some useful fields to spans (attempt_number, etc.)
* Add tests for span behavior (checking nesting structure, number of
times span appears, etc.)

## Testing
<!--- Please describe in detail how you tested your changes -->
<!--- Include details of your testing environment, and the tests you ran
to -->
<!--- see how your change affects other areas of the code, etc. -->
Added tests for span emission logic in the `integration-tests/metrics`
test suite

## Checklist
<!--- If a checkbox below is not applicable, then please DELETE it
rather than leaving it unchecked -->
- [x] For changes to the smithy-rs codegen or runtime crates, I have
created a changelog entry Markdown file in the `.changelog` directory,
specifying "client," "server," or both in the `applies_to` key.
- [x] For changes to the AWS SDK, generated SDK code, or SDK runtime
crates, I have created a changelog entry Markdown file in the
`.changelog` directory, specifying "aws-sdk-rust" in the `applies_to`
key.

----

_By submitting this pull request, I confirm that you can use, modify,
copy, and redistribute this contribution, under the terms of your
choice._

Author: landonxjames

.changelog/spans.md

@@ -0,0 +1,10 @@
+---
+applies_to: ["client", "aws-sdk-rust"]
+authors: ["landonxjames"]
+references: ["smithy-rs#4052"]
+breaking: false
+new_feature: false
+bug_fix: false
+---
+
+Update spans to better align with spec.

aws/rust-runtime/aws-config/src/default_provider/credentials.rs

@@ -72,7 +72,7 @@ impl DefaultCredentialsChain {
     async fn credentials(&self) -> provider::Result {
         self.provider_chain
             .provide_credentials()
-            .instrument(tracing::debug_span!("provide_credentials", provider = %"default_chain"))
+            .instrument(tracing::debug_span!("default_credentials_chain"))
             .await
     }
 }

aws/rust-runtime/aws-config/src/meta/credentials/chain.rs

@@ -92,7 +92,7 @@ impl CredentialsProviderChain {
 
     async fn credentials(&self) -> provider::Result {
         for (name, provider) in &self.providers {
-            let span = tracing::debug_span!("load_credentials", provider = %name);
+            let span = tracing::debug_span!("credentials_provider_chain", provider = %name);
             match provider.provide_credentials().instrument(span).await {
                 Ok(credentials) => {
                     tracing::debug!(provider = %name, "loaded credentials");

aws/rust-runtime/aws-config/src/meta/region.rs

@@ -40,7 +40,7 @@ impl RegionProviderChain {
         for provider in &self.providers {
             if let Some(region) = provider
                 .region()
-                .instrument(tracing::info_span!("load_region", provider = ?provider))
+                .instrument(tracing::debug_span!("region_provider_chain", provider = ?provider))
                 .await
             {
                 return Some(region);

aws/rust-runtime/aws-config/src/meta/token.rs

@@ -83,7 +83,7 @@ impl TokenProviderChain {
 
     async fn token(&self) -> Result {
         for (name, provider) in &self.providers {
-            let span = tracing::debug_span!("load_token", provider = %name);
+            let span = tracing::debug_span!("token_provider_chain", provider = %name);
             match provider.provide_token().instrument(span).await {
                 Ok(credentials) => {
                     tracing::debug!(provider = %name, "loaded access token");

aws/rust-runtime/aws-config/src/profile/credentials.rs

@@ -452,7 +452,7 @@ impl Builder {
 
     /// Builds a [`ProfileFileCredentialsProvider`]
     pub fn build(self) -> ProfileFileCredentialsProvider {
-        let build_span = tracing::debug_span!("build_profile_provider");
+        let build_span = tracing::debug_span!("build_profile_file_credentials_provider");
         let _enter = build_span.enter();
         let conf = self
             .provider_config

aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/AwsCodegenDecorator.kt

@@ -67,6 +67,7 @@ val DECORATORS: List<ClientCodegenDecorator> =
             HttpRequestCompressionDecorator(),
             DisablePayloadSigningDecorator(),
             AwsDisableStalledStreamProtection(),
+            SpanDecorator(),
             // TODO(https://github.com/smithy-lang/smithy-rs/issues/3863): Comment in once the issue has been resolved
             // SmokeTestsDecorator(),
         ),

aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/SpanDecorator.kt

@@ -0,0 +1,46 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package software.amazon.smithy.rustsdk
+
+import software.amazon.smithy.model.shapes.OperationShape
+import software.amazon.smithy.rust.codegen.client.smithy.ClientCodegenContext
+import software.amazon.smithy.rust.codegen.client.smithy.customize.ClientCodegenDecorator
+import software.amazon.smithy.rust.codegen.client.smithy.generators.OperationCustomization
+import software.amazon.smithy.rust.codegen.client.smithy.generators.OperationSection
+import software.amazon.smithy.rust.codegen.core.rustlang.Writable
+import software.amazon.smithy.rust.codegen.core.rustlang.writable
+import software.amazon.smithy.rust.codegen.core.util.dq
+
+/**
+ * Adds the AWS SDK specific `rpc.system = "aws-api"` field to the operation's tracing span
+ */
+class SpanDecorator : ClientCodegenDecorator {
+    override val name: String = "SpanDecorator"
+    override val order: Byte = 0
+
+    override fun operationCustomizations(
+        codegenContext: ClientCodegenContext,
+        operation: OperationShape,
+        baseCustomizations: List<OperationCustomization>,
+    ): List<OperationCustomization> {
+        return baseCustomizations +
+            object : OperationCustomization() {
+                private val runtimeConfig = codegenContext.runtimeConfig
+
+                override fun section(section: OperationSection): Writable {
+                    return writable {
+                        when (section) {
+                            is OperationSection.AdditionalOperationSpanFields -> {
+                                section.addField(this, "rpc.system", writable("aws-api".dq()))
+                            }
+
+                            else -> {}
+                        }
+                    }
+                }
+            }
+    }
+}

aws/sdk/integration-tests/Cargo.toml

@@ -11,13 +11,13 @@ members = [
     "iam",
     "kms",
     "lambda",
-    "metrics",
     "no-default-features",
     "polly",
     "qldbsession",
     "s3",
     "s3control",
     "sts",
+    "telemetry",
     "timestreamquery",
     "transcribestreaming",
     "webassembly",

aws/sdk/integration-tests/metrics/Cargo.toml renamed to aws/sdk/integration-tests/telemetry/Cargo.toml

@@ -1,6 +1,6 @@
 # This Cargo.toml is unused in generated code. It exists solely to enable these tests to compile in-situ
 [package]
-name = "metrics-tests"
+name = "telemetry-tests"
 version = "0.1.0"
 authors = ["AWS Rust SDK Team <[email protected]>"]
 edition = "2021"
@@ -23,3 +23,6 @@ opentelemetry = { version = "0.26.0", features = ["metrics"] }
 opentelemetry_sdk = { version = "0.26.0", features = ["metrics", "testing"] }
 serial_test = "3.1.1"
 tokio = { version = "1.23.1", features = ["full", "test-util"] }
+tracing = "0.1.40"
+tracing-subscriber = "0.3.18"
+tracing-fluent-assertions = "0.3.0"