feat(baggage): adding baggage to span tags (#7020)

## Summary of changes

This PR introduces automatic propagation of baggage key-value pairs as
span tags, prefixed with baggage. The new configuration option
`DD_TRACE_BAGGAGE_TAG_KEYS` allows customization of this feature.

## Reason for change

See the related
[RFC](https://docs.google.com/document/d/1ZsgAcQHKjaBxDmhHup0zqFz07WYiOrbBksHPoXJe6u4/edit?tab=t.0#heading=h.q69qzmye8dfb)
for more details.

## Implementation details

## Test coverage

## Other details
<!-- Fixes #{issue} -->

<!-- ⚠️ Note: where possible, please obtain 2 approvals prior to
merging. Unless CODEOWNERS specifies otherwise, for external teams it is
typically best to have one review from a team member, and one review
from apm-dotnet. Trivial changes do not require 2 reviews. -->

---------

Co-authored-by: Zach Montoya <zach.montoya@datadoghq.com>

Author: rachelyangdog

tracer/src/Datadog.Trace/AspNet/TracingHttpModule.cs

@@ -125,6 +125,21 @@ internal static void AddHeaderTagsFromHttpResponse(HttpContext httpContext, Scop
             }
         }
 
+        private static void AddBaggageTagsToSpan(ISpan span, Baggage baggage, Tracer tracer)
+        {
+            if (baggage != null)
+            {
+                try
+                {
+                    tracer.TracerManager.SpanContextPropagator.AddBaggageToSpanAsTags(span, baggage, tracer.Settings.BaggageTagKeys);
+                }
+                catch (Exception ex)
+                {
+                    Log.Error(ex, "Error adding baggage tags to span.");
+                }
+            }
+        }
+
         private void OnBeginRequest(object sender, EventArgs eventArgs)
         {
             Scope scope = null;
@@ -198,6 +213,8 @@ private void OnBeginRequest(object sender, EventArgs eventArgs)
                 scope.Span.DecorateWebServerSpan(resourceName: resourceName, httpMethod, host, url, userAgent, tags);
                 tracer.TracerManager.SpanContextPropagator.AddHeadersToSpanAsTags(scope.Span, headers, tracer.Settings.HeaderTags, defaultTagPrefix: SpanContextPropagator.HttpRequestHeadersTagPrefix);
 
+                AddBaggageTagsToSpan(scope.Span, extractedContext.Baggage, tracer);
+
                 if (tracer.Settings.IpHeaderEnabled || Security.Instance.AppsecEnabled)
                 {
                     Headers.Ip.RequestIpExtractor.AddIpToTags(httpRequest.UserHostAddress, httpRequest.IsSecureConnection, key => requestHeaders[key], tracer.Settings.IpHeader, tags);

tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/AspNet/AspNetMvcIntegration.cs

@@ -40,6 +40,21 @@ internal static class AspNetMvcIntegration
         private const IntegrationId IntegrationId = Configuration.IntegrationId.AspNetMvc;
         private static readonly IDatadogLogger Log = DatadogLogging.GetLoggerFor(typeof(AspNetMvcIntegration));
 
+        private static void AddBaggageTagsToSpan(ISpan span, Baggage baggage, Tracer tracer)
+        {
+            if (baggage != null)
+            {
+                try
+                {
+                    tracer.TracerManager.SpanContextPropagator.AddBaggageToSpanAsTags(span, baggage, tracer.Settings.BaggageTagKeys);
+                }
+                catch (Exception ex)
+                {
+                    Log.Error(ex, "Error adding baggage tags to span.");
+                }
+            }
+        }
+
         /// <summary>
         /// Creates a scope used to instrument an MVC action and populates some common details.
         /// </summary>
@@ -183,6 +198,8 @@ internal static Scope CreateScope(ControllerContextStruct controllerContext)
                         tracer.TracerManager.SpanContextPropagator.AddHeadersToSpanAsTags(span, headers.Value, tracer.Settings.HeaderTags, SpanContextPropagator.HttpRequestHeadersTagPrefix);
                     }
 
+                    AddBaggageTagsToSpan(span, extractedContext.Baggage, tracer);
+
                     if (tracer.Settings.IpHeaderEnabled || Security.Instance.AppsecEnabled)
                     {
                         Headers.Ip.RequestIpExtractor.AddIpToTags(httpContext.Request.UserHostAddress, httpContext.Request.IsSecureConnection, key => httpContext.Request.Headers[key], tracer.Settings.IpHeader, tags);

tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/AspNet/AspNetWebApi2Integration.cs

@@ -33,6 +33,21 @@ internal static class AspNetWebApi2Integration
         private const IntegrationId IntegrationId = Configuration.IntegrationId.AspNetWebApi2;
         private static readonly IDatadogLogger Log = DatadogLogging.GetLoggerFor(typeof(AspNetWebApi2Integration));
 
+        private static void AddBaggageTagsToSpan(ISpan span, Baggage baggage, Tracer tracer)
+        {
+            if (baggage != null)
+            {
+                try
+                {
+                    tracer.TracerManager.SpanContextPropagator.AddBaggageToSpanAsTags(span, baggage, tracer.Settings.BaggageTagKeys);
+                }
+                catch (Exception ex)
+                {
+                    Log.Error(ex, "Error adding baggage tags to span.");
+                }
+            }
+        }
+
         internal static Scope CreateScope(IHttpControllerContext controllerContext, out AspNetTags tags)
         {
             Scope scope = null;
@@ -104,6 +119,8 @@ internal static Scope CreateScope(IHttpControllerContext controllerContext, out
                     tracer.TracerManager.SpanContextPropagator.AddHeadersToSpanAsTags(scope.Span, headersCollection.Value, tracer.Settings.HeaderTags, SpanContextPropagator.HttpRequestHeadersTagPrefix, request.Headers.UserAgent.ToString());
                 }
 
+                AddBaggageTagsToSpan(scope.Span, extractedContext.Baggage, tracer);
+
                 tags.SetAnalyticsSampleRate(IntegrationId, tracer.Settings, enabledWithGlobalSetting: true);
                 tracer.TracerManager.Telemetry.IntegrationGeneratedSpan(IntegrationId);
             }

tracer/src/Datadog.Trace/Configuration/ConfigurationKeys.cs

@@ -467,6 +467,18 @@ internal static partial class ConfigurationKeys
         /// <seealso cref="TracerSettings.BaggageMaximumBytes"/>
         public const string BaggageMaximumBytes = "DD_TRACE_BAGGAGE_MAX_BYTES";
 
+        /// <summary>
+        /// Configuration key for controlling which baggage keys are converted into span tags.
+        /// Default value is "user.id,session.id,account.id".
+        ///
+        /// Behavior options:
+        /// - Empty string: No baggage keys are converted into span tags (feature disabled)
+        /// - Comma-separated list: Only baggage keys matching exact, case-sensitive names in the list are added as span tags
+        /// - Wildcard (*): All baggage keys are converted into span tags
+        /// </summary>
+        /// <seealso cref="TracerSettings.BaggageTagKeys"/>
+        public const string BaggageTagKeys = "DD_TRACE_BAGGAGE_TAG_KEYS";
+
         /// <summary>
         /// Configuration key for enabling automatic instrumentation on specified methods.
         /// Default value is "" (disabled).

tracer/src/Datadog.Trace/Configuration/TracerSettings.cs

@@ -637,6 +637,11 @@ _ when x.ToBoolean() is { } boolean => boolean,
                                  .WithKeys(ConfigurationKeys.BaggageMaximumBytes)
                                  .AsInt32(defaultValue: W3CBaggagePropagator.DefaultMaximumBaggageBytes);
 
+            BaggageTagKeys = config
+                            .WithKeys(ConfigurationKeys.BaggageTagKeys)
+                            .AsString(defaultValue: "user.id,session.id,account.id")
+                            ?.Split([','], StringSplitOptions.RemoveEmptyEntries) ?? [];
+
             LogSubmissionSettings = new DirectLogSubmissionSettings(source, _telemetry);
 
             TraceMethods = config
@@ -1146,6 +1151,13 @@ public bool DiagnosticSourceEnabled
         /// <seealso cref="ConfigurationKeys.BaggageMaximumBytes"/>
         internal int BaggageMaximumBytes { get; }
 
+        /// <summary>
+        /// Gets the configuration for which baggage keys are converted into span tags.
+        /// Default value is "user.id,session.id,account.id".
+        /// </summary>
+        /// <seealso cref="ConfigurationKeys.BaggageTagKeys"/>
+        internal string[] BaggageTagKeys { get; }
+
         /// <summary>
         /// Gets a value indicating whether runtime metrics
         /// are enabled and sent to DogStatsd.

tracer/src/Datadog.Trace/PlatformHelpers/AspNetCoreHttpRequestHandler.cs

@@ -98,6 +98,21 @@ private void AddHeaderTagsToSpan(ISpan span, HttpRequest request, Tracer tracer)
             }
         }
 
+        private void AddBaggageTagsToSpan(ISpan span, Baggage baggage, Tracer tracer)
+        {
+            if (baggage != null)
+            {
+                try
+                {
+                    tracer.TracerManager.SpanContextPropagator.AddBaggageToSpanAsTags(span, baggage, tracer.Settings.BaggageTagKeys);
+                }
+                catch (Exception ex)
+                {
+                    _log.Error(ex, "Error adding baggage tags to span.");
+                }
+            }
+        }
+
         public Scope StartAspNetCorePipelineScope(Tracer tracer, Security security, HttpContext httpContext, string resourceName)
         {
             var request = httpContext.Request;
@@ -125,6 +140,7 @@ public Scope StartAspNetCorePipelineScope(Tracer tracer, Security security, Http
             var scope = tracer.StartActiveInternal(_requestInOperationName, extractedContext.SpanContext, tags: tags, links: extractedContext.Links);
             scope.Span.DecorateWebServerSpan(resourceName, httpMethod, host, url, userAgent, tags);
             AddHeaderTagsToSpan(scope.Span, request, tracer);
+            AddBaggageTagsToSpan(scope.Span, extractedContext.Baggage, tracer);
 
             var originalPath = request.PathBase.HasValue ? request.PathBase.Add(request.Path) : request.Path;
             var requestTrackingFeature = new RequestTrackingFeature(originalPath, scope);

tracer/src/Datadog.Trace/Propagators/SpanContextPropagator.cs

@@ -310,6 +310,40 @@ internal void ExtractHeaderTags<THeaders, TProcessor>(ref TProcessor processor,
             }
         }
 
+        internal void AddBaggageToSpanAsTags(ISpan span, Baggage? baggage, string[] baggageTagKeys)
+        {
+            if (baggage is null or { Count: 0 })
+            {
+                return;
+            }
+
+            if (baggageTagKeys.Length == 0)
+            {
+                // feature disabled
+                return;
+            }
+
+            if (baggageTagKeys.Length == 1 && baggageTagKeys[0] == "*")
+            {
+                // add all baggage items as tags
+                foreach (var item in baggage)
+                {
+                    span.SetTag("baggage." + item.Key, item.Value);
+                }
+
+                return;
+            }
+
+            // add only specified baggage items as tags
+            foreach (var key in baggageTagKeys)
+            {
+                if (baggage.TryGetValue(key, out var value))
+                {
+                    span.SetTag("baggage." + key, value);
+                }
+            }
+        }
+
 #pragma warning disable SA1201
         public interface IHeaderTagProcessor
 #pragma warning restore SA1201

tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/AspNet/AspNetMvc5Tests.cs

@@ -8,6 +8,7 @@
 #pragma warning disable SA1649 // File name must match first type name
 
 using System;
+using System.Collections.Generic;
 using System.Globalization;
 using System.Linq;
 using System.Net;
@@ -173,6 +174,113 @@ public AspNetMvc5TestsInferredProxySpansDisabled(IisFixture iisFixture, ITestOut
         }
     }
 
+    [UsesVerify]
+    public abstract class AspNetMvc5TestsWithBaggage : TracingIntegrationTest, IClassFixture<IisFixture>, IAsyncLifetime
+    {
+        private readonly IisFixture _iisFixture;
+        private readonly string _testName;
+        private readonly bool _classicMode;
+        private readonly bool _enableInferredProxySpans;
+
+        protected AspNetMvc5TestsWithBaggage(IisFixture iisFixture, ITestOutputHelper output)
+            : base("AspNetMvc5", @"test\test-applications\aspnet", output)
+        {
+            SetServiceVersion("1.0.0");
+            SetEnvironmentVariable(ConfigurationKeys.FeatureFlags.RouteTemplateResourceNamesEnabled, true.ToString());
+            SetEnvironmentVariable(ConfigurationKeys.ExpandRouteTemplatesEnabled, false.ToString());
+            SetEnvironmentVariable(ConfigurationKeys.FeatureFlags.TraceId128BitGenerationEnabled, false.ToString());
+            SetEnvironmentVariable(ConfigurationKeys.FeatureFlags.InferredProxySpansEnabled, false.ToString());
+
+            _classicMode = false;
+            _enableInferredProxySpans = false;
+            _iisFixture = iisFixture;
+            _iisFixture.ShutdownPath = "/home/shutdown";
+
+            _testName = nameof(AspNetMvc5Tests)
+                      + ".Integrated"  // _classicMode = false
+                      + ".WithFF"      // enableRouteTemplateResourceNames = true, enableRouteTemplateExpansion = false
+                      + ".WithBaggage";
+        }
+
+        public static TheoryData<string, int> Data => new()
+        {
+            { "/", 200 },
+            { "/Home/Index", 200 },
+            { "/badrequest", 500 },
+        };
+
+        protected virtual string ExpectedServiceName => "sample";
+
+        public override Result ValidateIntegrationSpan(MockSpan span, string metadataSchemaVersion) =>
+            span.Name switch
+            {
+                "aspnet.request" => span.IsAspNet(metadataSchemaVersion, excludeTags: new HashSet<string> { "baggage.user.id" }),
+                "aspnet-mvc.request" => span.IsAspNetMvc(metadataSchemaVersion, excludeTags: new HashSet<string> { "baggage.user.id" }),
+                _ => Result.DefaultSuccess,
+            };
+
+        [SkippableTheory]
+        [Trait("Category", "EndToEnd")]
+        [Trait("RunOnWindows", "True")]
+        [Trait("LoadFromGAC", "True")]
+        [MemberData(nameof(Data))]
+        public async Task BaggageInSpanTags(string path, HttpStatusCode statusCode)
+        {
+            // TransferRequest cannot be called in the classic mode, so we expect a 500 when this happens
+            var toLowerPath = path.ToLower();
+            if (_testName.Contains(".Classic") && toLowerPath.Contains("badrequest") && toLowerPath.Contains("transferrequest"))
+            {
+                statusCode = (HttpStatusCode)500;
+            }
+
+            var expectedSpanCount = _enableInferredProxySpans ? 3 : 2;
+
+            var spans = await GetWebServerSpans(
+                path: _iisFixture.VirtualApplicationPath + path, // Append virtual directory to the actual request
+                agent: _iisFixture.Agent,
+                httpPort: _iisFixture.HttpPort,
+                expectedHttpStatusCode: statusCode,
+                expectedSpanCount: expectedSpanCount,
+                filterServerSpans: !_enableInferredProxySpans);
+
+            var serverSpans = spans.Where(s => s.Tags.GetValueOrDefault(Tags.SpanKind) == SpanKinds.Server);
+            ValidateIntegrationSpans(serverSpans, metadataSchemaVersion: "v0", expectedServiceName: ExpectedServiceName, isExternalSpan: false);
+
+            var sanitisedPath = VerifyHelper.SanitisePathsForVerify(path);
+            var settings = VerifyHelper.GetSpanVerifierSettings(sanitisedPath, (int)statusCode);
+
+            await Verifier.Verify(spans, settings)
+                          .UseMethodName("_withBaggage")
+                          .UseTypeName(_testName);
+        }
+
+        public async Task InitializeAsync()
+        {
+            await _iisFixture.TryStartIis(this, _classicMode ? IisAppType.AspNetClassic : IisAppType.AspNetIntegrated);
+        }
+
+        public Task DisposeAsync() => Task.CompletedTask;
+
+        /// <summary>
+        /// Override <see cref="CreateHttpRequestMessage"/> to add baggage headers to the request.
+        /// </summary>
+        protected override HttpRequestMessage CreateHttpRequestMessage(HttpMethod method, string path, DateTimeOffset testStart)
+        {
+            var request = base.CreateHttpRequestMessage(method, path, testStart);
+            request.Headers.Add("baggage", "user.id=doggo");
+            return request;
+        }
+    }
+
+    [Collection("IisTests")]
+    public class AspNetMvc5TestsWithBaggageEnabled : AspNetMvc5TestsWithBaggage
+    {
+        public AspNetMvc5TestsWithBaggageEnabled(IisFixture iisFixture, ITestOutputHelper output)
+            : base(iisFixture, output)
+        {
+        }
+    }
+
     [UsesVerify]
     public abstract class AspNetMvc5Tests : TracingIntegrationTest, IClassFixture<IisFixture>, IAsyncLifetime
     {

tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/AspNet/OwinWebApi2Tests.cs

@@ -8,6 +8,7 @@
 #pragma warning disable SA1649 // File name must match first type name
 
 using System;
+using System.Collections.Generic;
 using System.Collections.Immutable;
 using System.Diagnostics;
 using System.Linq;
@@ -112,7 +113,7 @@ public OwinWebApi2Tests(OwinFixture fixture, ITestOutputHelper output, bool enab
         public override Result ValidateIntegrationSpan(MockSpan span, string metadataSchemaVersion) =>
             span.Name switch
             {
-                "aspnet-webapi.request" => span.IsAspNetWebApi2(metadataSchemaVersion),
+                "aspnet-webapi.request" => span.IsAspNetWebApi2(metadataSchemaVersion, excludeTags: new HashSet<string> { "baggage.user.id" }),
                 _ => Result.DefaultSuccess,
             };
 
@@ -147,6 +148,7 @@ public OwinFixture()
                 _httpClient = new HttpClient();
                 _httpClient.DefaultRequestHeaders.Add(HttpHeaderNames.TracingEnabled, "false");
                 _httpClient.DefaultRequestHeaders.Add(HttpHeaderNames.UserAgent, "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.69 Safari/537.36");
+                _httpClient.DefaultRequestHeaders.Add("baggage", "user.id=doggo");
             }
 
             public MockTracerAgent.TcpUdpAgent Agent { get; private set; }