[Tracer] Fix resource-based sampling for ASP.NET spans in IIS (#6936)

bouwkast · web-flow · commit 8b1a9ff9e986 · 2025-05-23T13:55:02.000-04:00
## Summary of changes Set's a preliminary `Span.ResourceName` for `ASP.NET` spans in `OnBeginRequest` as well as in `OnEndRequest` to ensure that resource-based sampling will work. ## Reason for change Within `TracingHttpModule` we do not set a `ResourceName` for the span in `OnBeginRequest`, this is to try and ensure that the resource names from the parent `ASP.NET` span and it's child span will be equivalent. We instead set it in the `OnEndRequest`. This poses an issue though in IIS as we then (commonly) call `Inject` which ends up making a sampling decision _without_ a `Span.ResourceName`. What this ends up causing is that resource-based sampling decisions / rules will never be applied. Once a sampling decision has been made it is not changed at a later point. The following code path gets hit for `ASP.NET` in IIS which causes the behavior. ```csharp // Decorate the incoming HTTP Request with distributed tracing headers // in case the next processor cannot access the stored Scope // (e.g. WCF being hosted in IIS) if (HttpRuntime.UsingIntegratedPipeline) { var injectedContext = new PropagationContext(scope.Span.Context, Baggage.Current); tracer.TracerManager.SpanContextPropagator.Inject(injectedContext, requestHeaders.Wrap()); } ``` ## Implementation details I've extracted _a part_ of the logic to build the resource name in the `OnEndRequest` and am using it in the `OnBeginRequest` phase. This will allow us to have some semblance of a relatively accurate resource name at the time that we are making the sampling decision. The part that is different is that we do not in the `OnBeginRequest` have access to the child span resource name, which is the path -> ```csharp if (app.Context.Items[SharedItems.HttpContextPropagatedResourceNameKey] is string resourceName && !string.IsNullOrEmpty(resourceName)) { currentSpan.ResourceName = resourceName; } ``` So while I _think_ that the resource name computed now will be relatively accurate there may be instances where the child HTTP span from it has a different resource name that overrides it. ## Test coverage Updated MVC4 ASP net tests to have a resource that should be sampled out (see the snapshot sampling value) based on its `aspnet` span (not the `MVC`) span resource name. ## Other details It may be beneficial to tag the original resource name as it may look a bit different to help determine what resource was used in the sampling decision (if any), but we can come back to that later.
diff --git a/tracer/src/Datadog.Trace/AspNet/TracingHttpModule.cs b/tracer/src/Datadog.Trace/AspNet/TracingHttpModule.cs
@@ -85,6 +85,22 @@ public void Dispose()
         {
         }
 
+        private static string BuildResourceName(Tracer tracer, HttpRequest httpRequest)
+        {
+            var url = tracer.Settings.BypassHttpRequestUrlCachingEnabled
+                               ? RequestDataHelper.BuildUrl(httpRequest)
+                               : RequestDataHelper.GetUrl(httpRequest);
+            if (url is not null)
+            {
+                var path = UriHelpers.GetCleanUriPath(url, httpRequest.ApplicationPath);
+                return $"{httpRequest.HttpMethod.ToUpperInvariant()} {path.ToLowerInvariant()}";
+            }
+            else
+            {
+                return $"{httpRequest.HttpMethod.ToUpperInvariant()}";
+            }
+        }
+
         internal static void AddHeaderTagsFromHttpResponse(HttpContext httpContext, Scope scope)
         {
             if (!Tracer.Instance.Settings.HeaderTags.IsNullOrEmpty() &&
@@ -174,7 +190,12 @@ private void OnBeginRequest(object sender, EventArgs eventArgs)
                 var tags = new WebTags();
                 scope = tracer.StartActiveInternal(_requestOperationName, extractedContext.SpanContext, tags: tags);
                 // Leave resourceName blank for now - we'll update it in OnEndRequest
-                scope.Span.DecorateWebServerSpan(resourceName: null, httpMethod, host, url, userAgent, tags);
+
+                // Attempt to set Resource Name to something that will be close to what is expected
+                // Note: we will go and re-do it in OnEndRequest, but doing it here will allow for resource-based sampling
+                // this likely won't be perfect - but we need something to try and allow resource-based sampling to function
+                var resourceName = BuildResourceName(tracer, httpRequest);
+                scope.Span.DecorateWebServerSpan(resourceName: resourceName, httpMethod, host, url, userAgent, tags);
                 tracer.TracerManager.SpanContextPropagator.AddHeadersToSpanAsTags(scope.Span, headers, tracer.Settings.HeaderTags, defaultTagPrefix: SpanContextPropagator.HttpRequestHeadersTagPrefix);
 
                 if (tracer.Settings.IpHeaderEnabled || Security.Instance.AppsecEnabled)
@@ -386,18 +407,7 @@ private void OnEndRequest(object sender, EventArgs eventArgs)
                         }
                         else
                         {
-                            var url = tracer.Settings.BypassHttpRequestUrlCachingEnabled
-                                ? RequestDataHelper.BuildUrl(app.Request)
-                                : RequestDataHelper.GetUrl(app.Request);
-                            if (url is not null)
-                            {
-                                string path = UriHelpers.GetCleanUriPath(url, app.Request.ApplicationPath);
-                                currentSpan.ResourceName = $"{app.Request.HttpMethod.ToUpperInvariant()} {path.ToLowerInvariant()}";
-                            }
-                            else
-                            {
-                                currentSpan.ResourceName = $"{app.Request.HttpMethod.ToUpperInvariant()}";
-                            }
+                            currentSpan.ResourceName = BuildResourceName(tracer, app.Request);
                         }
                     }
                     finally
diff --git a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/AspNet/AspNetMvc4Tests.cs b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/AspNet/AspNetMvc4Tests.cs
@@ -13,6 +13,7 @@
 using System.Text;
 using System.Threading.Tasks;
 using Datadog.Trace.Configuration;
+using Datadog.Trace.Sampling;
 using Datadog.Trace.TestHelpers;
 using FluentAssertions;
 using VerifyXunit;
@@ -80,6 +81,11 @@ public AspNetMvc4Tests(IisFixture iisFixture, ITestOutputHelper output, bool cla
             SetEnvironmentVariable(ConfigurationKeys.FeatureFlags.RouteTemplateResourceNamesEnabled, enableRouteTemplateResourceNames.ToString());
             SetEnvironmentVariable(ConfigurationKeys.ExpandRouteTemplatesEnabled, enableRouteTemplateExpansion.ToString());
 
+            // these two are to test resource-based sampling on the parent ASP.NET span (non-MVC) one.
+            // it should not show up at all
+            SetEnvironmentVariable(ConfigurationKeys.CustomSamplingRules, """[{"sample_rate":0.0, "service":"*", "resource":"GET /home/pleasesamplemeout/?"}]""");
+            SetEnvironmentVariable(ConfigurationKeys.CustomSamplingRulesFormat, SamplingRulesFormat.Glob.ToString()); // for ease of use
+
             _classicMode = classicMode;
             _iisFixture = iisFixture;
             _iisFixture.ShutdownPath = "/home/shutdown";
@@ -110,6 +116,7 @@ public AspNetMvc4Tests(IisFixture iisFixture, ITestOutputHelper output, bool cla
               { "/Home/BadRequestWithStatusCode?statuscode=401&TransferRequest=true", 401 },
               { "/Home/BadRequestWithStatusCode?statuscode=503&TransferRequest=true", 503 },
               { "/graphql/GetAllFoo", 200 }, // Slug in route template
+              { "/Home/PleaseSampleMeOut/5555", 200 }, // test resource-based sampling for parent ASP.NET span (non-MVC) "GET /home/pleaseSampleMeOut/?" <- note the ? it isn't {id}
         };
 
         public override Result ValidateIntegrationSpan(MockSpan span, string metadataSchemaVersion) =>
diff --git a/tracer/test/snapshots/AspNetMvc4Tests.Classic.NoFF.__path=_Home_PleaseSampleMeOut_5555_statusCode=200.verified.txt b/tracer/test/snapshots/AspNetMvc4Tests.Classic.NoFF.__path=_Home_PleaseSampleMeOut_5555_statusCode=200.verified.txt
@@ -0,0 +1,53 @@
+[
+  {
+    TraceId: Id_1,
+    SpanId: Id_2,
+    Name: aspnet-mvc.request,
+    Resource: GET /home/pleasesamplemeout/?,
+    Service: sample,
+    Type: web,
+    ParentId: Id_3,
+    Tags: {
+      aspnet.action: pleasesamplemeout,
+      aspnet.controller: home,
+      aspnet.route: {controller}/{action}/{id},
+      env: integration_tests,
+      http.method: GET,
+      http.request.headers.host: localhost:00000,
+      http.status_code: 200,
+      http.url: http://localhost:00000/Home/PleaseSampleMeOut/5555,
+      http.useragent: testhelper,
+      language: dotnet,
+      span.kind: server,
+      version: 1.0.0
+    }
+  },
+  {
+    TraceId: Id_1,
+    SpanId: Id_3,
+    Name: aspnet.request,
+    Resource: GET /home/pleasesamplemeout/?,
+    Service: sample,
+    Type: web,
+    Tags: {
+      env: integration_tests,
+      http.method: GET,
+      http.request.headers.host: localhost:00000,
+      http.route: {controller}/{action}/{id},
+      http.status_code: 200,
+      http.url: http://localhost:00000/Home/PleaseSampleMeOut/5555,
+      http.useragent: testhelper,
+      language: dotnet,
+      runtime-id: Guid_1,
+      span.kind: server,
+      version: 1.0.0
+    },
+    Metrics: {
+      process_id: 0,
+      _dd.rule_psr: 0.0,
+      _dd.top_level: 1.0,
+      _dd.tracer_kr: 1.0,
+      _sampling_priority_v1: -1.0
+    }
+  }
+]
diff --git a/tracer/test/snapshots/AspNetMvc4Tests.Classic.WithFF.__path=_Home_PleaseSampleMeOut_5555_statusCode=200.verified.txt b/tracer/test/snapshots/AspNetMvc4Tests.Classic.WithFF.__path=_Home_PleaseSampleMeOut_5555_statusCode=200.verified.txt
@@ -0,0 +1,52 @@
+[
+  {
+    TraceId: Id_1,
+    SpanId: Id_2,
+    Name: aspnet-mvc.request,
+    Resource: GET /home/pleasesamplemeout/{id},
+    Service: sample,
+    Type: web,
+    ParentId: Id_3,
+    Tags: {
+      aspnet.action: pleasesamplemeout,
+      aspnet.controller: home,
+      aspnet.route: {controller}/{action}/{id},
+      env: integration_tests,
+      http.method: GET,
+      http.request.headers.host: localhost:00000,
+      http.status_code: 200,
+      http.url: http://localhost:00000/Home/PleaseSampleMeOut/5555,
+      http.useragent: testhelper,
+      language: dotnet,
+      span.kind: server,
+      version: 1.0.0
+    }
+  },
+  {
+    TraceId: Id_1,
+    SpanId: Id_3,
+    Name: aspnet.request,
+    Resource: GET /home/pleasesamplemeout/{id},
+    Service: sample,
+    Type: web,
+    Tags: {
+      env: integration_tests,
+      http.method: GET,
+      http.request.headers.host: localhost:00000,
+      http.route: {controller}/{action}/{id},
+      http.status_code: 200,
+      http.url: http://localhost:00000/Home/PleaseSampleMeOut/5555,
+      http.useragent: testhelper,
+      language: dotnet,
+      runtime-id: Guid_1,
+      span.kind: server,
+      version: 1.0.0
+    },
+    Metrics: {
+      process_id: 0,
+      _dd.top_level: 1.0,
+      _dd.tracer_kr: 1.0,
+      _sampling_priority_v1: 1.0
+    }
+  }
+]
diff --git a/tracer/test/snapshots/AspNetMvc4Tests.Integrated.NoFF.__path=_Home_PleaseSampleMeOut_5555_statusCode=200.verified.txt b/tracer/test/snapshots/AspNetMvc4Tests.Integrated.NoFF.__path=_Home_PleaseSampleMeOut_5555_statusCode=200.verified.txt
@@ -0,0 +1,53 @@
+[
+  {
+    TraceId: Id_1,
+    SpanId: Id_2,
+    Name: aspnet-mvc.request,
+    Resource: GET /home/pleasesamplemeout/?,
+    Service: sample,
+    Type: web,
+    ParentId: Id_3,
+    Tags: {
+      aspnet.action: pleasesamplemeout,
+      aspnet.controller: home,
+      aspnet.route: {controller}/{action}/{id},
+      env: integration_tests,
+      http.method: GET,
+      http.request.headers.host: localhost:00000,
+      http.status_code: 200,
+      http.url: http://localhost:00000/Home/PleaseSampleMeOut/5555,
+      http.useragent: testhelper,
+      language: dotnet,
+      span.kind: server,
+      version: 1.0.0
+    }
+  },
+  {
+    TraceId: Id_1,
+    SpanId: Id_3,
+    Name: aspnet.request,
+    Resource: GET /home/pleasesamplemeout/?,
+    Service: sample,
+    Type: web,
+    Tags: {
+      env: integration_tests,
+      http.method: GET,
+      http.request.headers.host: localhost:00000,
+      http.route: {controller}/{action}/{id},
+      http.status_code: 200,
+      http.url: http://localhost:00000/Home/PleaseSampleMeOut/5555,
+      http.useragent: testhelper,
+      language: dotnet,
+      runtime-id: Guid_1,
+      span.kind: server,
+      version: 1.0.0
+    },
+    Metrics: {
+      process_id: 0,
+      _dd.rule_psr: 0.0,
+      _dd.top_level: 1.0,
+      _dd.tracer_kr: 1.0,
+      _sampling_priority_v1: -1.0
+    }
+  }
+]
diff --git a/tracer/test/snapshots/AspNetMvc4Tests.Integrated.WithExpansion.__path=_Home_PleaseSampleMeOut_5555_statusCode=200.verified.txt b/tracer/test/snapshots/AspNetMvc4Tests.Integrated.WithExpansion.__path=_Home_PleaseSampleMeOut_5555_statusCode=200.verified.txt
@@ -0,0 +1,53 @@
+[
+  {
+    TraceId: Id_1,
+    SpanId: Id_2,
+    Name: aspnet-mvc.request,
+    Resource: GET /home/pleasesamplemeout/{id},
+    Service: sample,
+    Type: web,
+    ParentId: Id_3,
+    Tags: {
+      aspnet.action: pleasesamplemeout,
+      aspnet.controller: home,
+      aspnet.route: {controller}/{action}/{id},
+      env: integration_tests,
+      http.method: GET,
+      http.request.headers.host: localhost:00000,
+      http.status_code: 200,
+      http.url: http://localhost:00000/Home/PleaseSampleMeOut/5555,
+      http.useragent: testhelper,
+      language: dotnet,
+      span.kind: server,
+      version: 1.0.0
+    }
+  },
+  {
+    TraceId: Id_1,
+    SpanId: Id_3,
+    Name: aspnet.request,
+    Resource: GET /home/pleasesamplemeout/{id},
+    Service: sample,
+    Type: web,
+    Tags: {
+      env: integration_tests,
+      http.method: GET,
+      http.request.headers.host: localhost:00000,
+      http.route: {controller}/{action}/{id},
+      http.status_code: 200,
+      http.url: http://localhost:00000/Home/PleaseSampleMeOut/5555,
+      http.useragent: testhelper,
+      language: dotnet,
+      runtime-id: Guid_1,
+      span.kind: server,
+      version: 1.0.0
+    },
+    Metrics: {
+      process_id: 0,
+      _dd.rule_psr: 0.0,
+      _dd.top_level: 1.0,
+      _dd.tracer_kr: 1.0,
+      _sampling_priority_v1: -1.0
+    }
+  }
+]
diff --git a/tracer/test/snapshots/AspNetMvc4Tests.Integrated.WithFF.__path=_Home_PleaseSampleMeOut_5555_statusCode=200.verified.txt b/tracer/test/snapshots/AspNetMvc4Tests.Integrated.WithFF.__path=_Home_PleaseSampleMeOut_5555_statusCode=200.verified.txt
@@ -0,0 +1,53 @@
+[
+  {
+    TraceId: Id_1,
+    SpanId: Id_2,
+    Name: aspnet-mvc.request,
+    Resource: GET /home/pleasesamplemeout/{id},
+    Service: sample,
+    Type: web,
+    ParentId: Id_3,
+    Tags: {
+      aspnet.action: pleasesamplemeout,
+      aspnet.controller: home,
+      aspnet.route: {controller}/{action}/{id},
+      env: integration_tests,
+      http.method: GET,
+      http.request.headers.host: localhost:00000,
+      http.status_code: 200,
+      http.url: http://localhost:00000/Home/PleaseSampleMeOut/5555,
+      http.useragent: testhelper,
+      language: dotnet,
+      span.kind: server,
+      version: 1.0.0
+    }
+  },
+  {
+    TraceId: Id_1,
+    SpanId: Id_3,
+    Name: aspnet.request,
+    Resource: GET /home/pleasesamplemeout/{id},
+    Service: sample,
+    Type: web,
+    Tags: {
+      env: integration_tests,
+      http.method: GET,
+      http.request.headers.host: localhost:00000,
+      http.route: {controller}/{action}/{id},
+      http.status_code: 200,
+      http.url: http://localhost:00000/Home/PleaseSampleMeOut/5555,
+      http.useragent: testhelper,
+      language: dotnet,
+      runtime-id: Guid_1,
+      span.kind: server,
+      version: 1.0.0
+    },
+    Metrics: {
+      process_id: 0,
+      _dd.rule_psr: 0.0,
+      _dd.top_level: 1.0,
+      _dd.tracer_kr: 1.0,
+      _sampling_priority_v1: -1.0
+    }
+  }
+]
diff --git a/tracer/test/test-applications/aspnet/Samples.AspNetMvc4/Controllers/HomeController.cs b/tracer/test/test-applications/aspnet/Samples.AspNetMvc4/Controllers/HomeController.cs
@@ -66,5 +66,14 @@ public ActionResult OptionalIdentifier(int? id)
             ViewBag.Message = "Identifier set to " + id;
             return View("About");
         }
+
+        public ActionResult PleaseSampleMeOut(int? id)
+        {
+            // this should be sampled out - the resource name sampled is /home/pleaseSampleMeOut/?
+            // however the one that will be shown is /home/pleaseSampleMeOut/{id} after MVC routes are replaced
+            // but we make the sampling decision BEFORE the MVC routes are known
+            ViewBag.Message = "This should be sampled out based on the ASP.NET Request resource name " + id;
+            return View("About");
+        }
     }
 }

Original file line number	Diff line number	Diff line change
`@@ -66,5 +66,14 @@ public ActionResult OptionalIdentifier(int? id)`
`66`	`66`	`ViewBag.Message = "Identifier set to " + id;`
`67`	`67`	`return View("About");`
`68`	`68`	`}`
	`69`	`+`
	`70`	`+ public ActionResult PleaseSampleMeOut(int? id)`
	`71`	`+ {`
	`72`	`+ // this should be sampled out - the resource name sampled is /home/pleaseSampleMeOut/?`
	`73`	`+ // however the one that will be shown is /home/pleaseSampleMeOut/{id} after MVC routes are replaced`
	`74`	`+ // but we make the sampling decision BEFORE the MVC routes are known`
	`75`	`+ ViewBag.Message = "This should be sampled out based on the ASP.NET Request resource name " + id;`
	`76`	`+ return View("About");`
	`77`	`+ }`
`69`	`78`	`}`
`70`	`79`	`}`