From fcfbbd6f495c3748b5665970217e35c02737a336 Mon Sep 17 00:00:00 2001 From: Tomer Rosenthal Date: Tue, 2 Dec 2025 09:54:02 -0800 Subject: [PATCH 1/2] Add option to include InstanceId in Application Insights operation name Fixes #3258 This change adds a new configuration option 'IncludeInstanceIdInOperationName' that allows users to include the orchestration instance ID in the Application Insights operation name (operation_Name) for better failure grouping and debugging in the .NET Isolated worker model. Changes: - Add IncludeInstanceIdInOperationName option to TraceOptions - Update Schema.SpanNames with overloads that support instance ID - Update TraceHelper to pass instance ID when creating activities - Update WebJobsTelemetryModule to set Operation.Name - Add unit tests for Schema.SpanNames --- .../Correlation/Schema.cs | 21 +- .../Correlation/TraceHelper.cs | 20 +- .../Correlation/WebJobsTelemetryModule.cs | 5 + .../HttpApiHandler.cs | 4 +- .../Options/TraceOptions.cs | 15 ++ .../TaskHubGrpcServer.cs | 8 +- test/Common/SchemaSpanNamesTests.cs | 204 ++++++++++++++++++ 7 files changed, 259 insertions(+), 18 deletions(-) create mode 100644 test/Common/SchemaSpanNamesTests.cs diff --git a/src/WebJobs.Extensions.DurableTask/Correlation/Schema.cs b/src/WebJobs.Extensions.DurableTask/Correlation/Schema.cs index 831708ac9..2cff3c1f7 100644 --- a/src/WebJobs.Extensions.DurableTask/Correlation/Schema.cs +++ b/src/WebJobs.Extensions.DurableTask/Correlation/Schema.cs @@ -31,14 +31,29 @@ internal static class Status internal static class SpanNames { internal static string CallOrSignalEntity(string name, string operation) - => $"{TraceActivityConstants.Entity}:{name}:{operation}"; - - internal static string EntityStartsAnOrchestration(string name) + => $"{TraceActivityConstants.Entity}:{name}:{operation}"; + + internal static string CallOrSignalEntity(string name, string operation, string? instanceId, bool includeInstanceId) + => includeInstanceId && !string.IsNullOrEmpty(instanceId) + ? $"{TraceActivityConstants.Entity}:{name}:{operation} ({instanceId})" + : CallOrSignalEntity(name, operation); + + internal static string EntityStartsAnOrchestration(string name) => $"{name}:{TraceActivityConstants.CreateOrchestration}"; + internal static string EntityStartsAnOrchestration(string name, string? instanceId, bool includeInstanceId) + => includeInstanceId && !string.IsNullOrEmpty(instanceId) + ? $"{name}:{TraceActivityConstants.CreateOrchestration} ({instanceId})" + : EntityStartsAnOrchestration(name); + internal static string CreateOrchestration(string name, string? version) => FormatName(TraceActivityConstants.CreateOrchestration, name, version); + internal static string CreateOrchestration(string name, string? version, string? instanceId, bool includeInstanceId) + => includeInstanceId && !string.IsNullOrEmpty(instanceId) + ? $"{FormatName(TraceActivityConstants.CreateOrchestration, name, version)} ({instanceId})" + : CreateOrchestration(name, version); + private static string FormatName(string prefix, string name, string? version) => string.IsNullOrEmpty(version) ? $"{prefix}:{name}" : $"{prefix}:{name}@{version}"; } diff --git a/src/WebJobs.Extensions.DurableTask/Correlation/TraceHelper.cs b/src/WebJobs.Extensions.DurableTask/Correlation/TraceHelper.cs index 7d1e331b1..7719d8ab5 100644 --- a/src/WebJobs.Extensions.DurableTask/Correlation/TraceHelper.cs +++ b/src/WebJobs.Extensions.DurableTask/Correlation/TraceHelper.cs @@ -17,15 +17,17 @@ internal class TraceHelper private const string Source = "WebJobs.Extensions.DurableTask"; private static readonly ActivitySource ActivityTraceSource = new ActivitySource(Source); - - internal static Activity? StartActivityForNewOrchestration(ExecutionStartedEvent startEvent, ActivityContext parentTraceContext, DateTimeOffset? startTime = default) + + internal static Activity? StartActivityForNewOrchestration(ExecutionStartedEvent startEvent, ActivityContext parentTraceContext, DateTimeOffset? startTime = default, bool includeInstanceIdInOperationName = false) { + string instanceId = startEvent.OrchestrationInstance.InstanceId; + // Start the new activity to represent scheduling the orchestration Activity? newActivity = ActivityTraceSource.StartActivity( - Schema.SpanNames.CreateOrchestration(startEvent.Name, startEvent.Version), - kind: ActivityKind.Producer, + Schema.SpanNames.CreateOrchestration(startEvent.Name, startEvent.Version, instanceId, includeInstanceIdInOperationName), + kind: ActivityKind.Producer, parentContext: parentTraceContext, - startTime: startTime ?? default); + startTime: startTime ?? default); if (newActivity == null) { @@ -47,8 +49,8 @@ internal class TraceHelper return newActivity; } - - internal static Activity? StartActivityForCallingOrSignalingEntity(string targetEntityId, string entityName, string operationName, bool signalEntity, DateTime? scheduledTime, ActivityContext? parentTraceContext, DateTimeOffset? startTime = default, string? entityId = null) + + internal static Activity? StartActivityForCallingOrSignalingEntity(string targetEntityId, string entityName, string operationName, bool signalEntity, DateTime? scheduledTime, ActivityContext? parentTraceContext, DateTimeOffset? startTime = default, string? entityId = null) { // We only want to create a trace activity for calling or signaling an entity in the case that we can successfully get the parent trace context of the request. // Otherwise, we will create an unlinked trace activity with no parent. @@ -60,8 +62,8 @@ internal class TraceHelper Activity? newActivity = ActivityTraceSource.StartActivity( Schema.SpanNames.CallOrSignalEntity(entityName, operationName), kind: signalEntity ? ActivityKind.Producer : ActivityKind.Client, - parentContext: parentTraceContext.Value, - startTime: startTime ?? default); + parentContext: parentTraceContext.Value, + startTime: startTime ?? default); if (newActivity == null) { diff --git a/src/WebJobs.Extensions.DurableTask/Correlation/WebJobsTelemetryModule.cs b/src/WebJobs.Extensions.DurableTask/Correlation/WebJobsTelemetryModule.cs index f06e8a9bf..8c20ea77b 100644 --- a/src/WebJobs.Extensions.DurableTask/Correlation/WebJobsTelemetryModule.cs +++ b/src/WebJobs.Extensions.DurableTask/Correlation/WebJobsTelemetryModule.cs @@ -88,6 +88,11 @@ private static T CreateTelemetryCore(Activity activity) }; telemetry.Context.Operation.Id = activity.RootId; + + // Set Operation.Name for proper grouping in Application Insights. + // The display name may include instance ID if IncludeInstanceIdInOperationName is enabled. + telemetry.Context.Operation.Name = activity.DisplayName; + ActivitySpanId parentId = activity.ParentSpanId; if (parentId != default) { diff --git a/src/WebJobs.Extensions.DurableTask/HttpApiHandler.cs b/src/WebJobs.Extensions.DurableTask/HttpApiHandler.cs index 98ff1d4ba..29245e9c3 100644 --- a/src/WebJobs.Extensions.DurableTask/HttpApiHandler.cs +++ b/src/WebJobs.Extensions.DurableTask/HttpApiHandler.cs @@ -904,11 +904,11 @@ private async Task HandleStartOrchestratorRequestAsync( if (traceParent != null) { ActivityContext.TryParse(traceParent, traceState, out ActivityContext parentActivityContext); - using Activity scheduleOrchestrationActivity = TraceHelper.StartActivityForNewOrchestration(executionStartedEvent, parentActivityContext); + using Activity scheduleOrchestrationActivity = TraceHelper.StartActivityForNewOrchestration(executionStartedEvent, parentActivityContext, includeInstanceIdInOperationName: this.durableTaskOptions.Tracing.IncludeInstanceIdInOperationName); } else { - using Activity scheduleOrchestrationActivity = TraceHelper.StartActivityForNewOrchestration(executionStartedEvent, default); + using Activity scheduleOrchestrationActivity = TraceHelper.StartActivityForNewOrchestration(executionStartedEvent, default, includeInstanceIdInOperationName: this.durableTaskOptions.Tracing.IncludeInstanceIdInOperationName); } await durableClient.DurabilityProvider.CreateTaskOrchestrationAsync( diff --git a/src/WebJobs.Extensions.DurableTask/Options/TraceOptions.cs b/src/WebJobs.Extensions.DurableTask/Options/TraceOptions.cs index 0bab15219..a895aa3f3 100644 --- a/src/WebJobs.Extensions.DurableTask/Options/TraceOptions.cs +++ b/src/WebJobs.Extensions.DurableTask/Options/TraceOptions.cs @@ -71,6 +71,21 @@ public class TraceOptions /// public DurableDistributedTracingVersion Version { get; set; } = DurableDistributedTracingVersion.V1; + /// + /// Gets or sets a value indicating whether to include the orchestration instance ID in the + /// Application Insights operation name (operation_Name). + /// + /// + /// When set to true, the orchestration instance ID will be appended to the operation name + /// in the format "{FunctionName} ({InstanceId})". This allows for better grouping and filtering + /// of failures in Application Insights by individual orchestration instance. + /// The default value is false to maintain backward compatibility. + /// + /// + /// true to include the instance ID in the operation name; false otherwise. + /// + public bool IncludeInstanceIdInOperationName { get; set; } = false; + internal void AddToDebugString(StringBuilder builder) { builder.Append(nameof(this.TraceReplayEvents)).Append(": ").Append(this.TraceReplayEvents).Append(", "); diff --git a/src/WebJobs.Extensions.DurableTask/TaskHubGrpcServer.cs b/src/WebJobs.Extensions.DurableTask/TaskHubGrpcServer.cs index 486ac4eaa..cef34d657 100644 --- a/src/WebJobs.Extensions.DurableTask/TaskHubGrpcServer.cs +++ b/src/WebJobs.Extensions.DurableTask/TaskHubGrpcServer.cs @@ -72,7 +72,7 @@ public override Task Hello(Empty request, ServerCallContext context) // Create a new activity with the parent context ActivityContext.TryParse(traceParent, traceState, out ActivityContext parentActivityContext); - using Activity? scheduleOrchestrationActivity = TraceHelper.StartActivityForNewOrchestration(executionStartedEvent, parentActivityContext, request.RequestTime?.ToDateTimeOffset()); + using Activity? scheduleOrchestrationActivity = TraceHelper.StartActivityForNewOrchestration(executionStartedEvent, parentActivityContext, request.RequestTime?.ToDateTimeOffset(), this.extension.Options.Tracing.IncludeInstanceIdInOperationName); // Schedule the orchestration await this.GetDurabilityProvider(context).CreateTaskOrchestrationAsync( @@ -301,11 +301,11 @@ private OrchestrationStatus[] GetStatusesNotToOverride() public async override Task RewindInstance(P.RewindInstanceRequest request, ServerCallContext context) { - try - { + try + { #pragma warning disable CS0618 // Type or member is obsolete await this.GetClient(context).RewindAsync(request.InstanceId, request.Reason); -#pragma warning restore CS0618 // Type or member is obsolete +#pragma warning restore CS0618 // Type or member is obsolete } catch (ArgumentException ex) { diff --git a/test/Common/SchemaSpanNamesTests.cs b/test/Common/SchemaSpanNamesTests.cs new file mode 100644 index 000000000..8cb0b8638 --- /dev/null +++ b/test/Common/SchemaSpanNamesTests.cs @@ -0,0 +1,204 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See LICENSE in the project root for license information. + +using Microsoft.Azure.WebJobs.Extensions.DurableTask.Correlation; +using Xunit; + +namespace Microsoft.Azure.WebJobs.Extensions.DurableTask.Tests +{ + /// + /// Unit tests for the Schema.SpanNames class. + /// + public class SchemaSpanNamesTests + { + [Fact] + public void CreateOrchestration_WithoutInstanceId_ReturnsExpectedFormat() + { + // Arrange + string name = "MyOrchestration"; + string version = null; + + // Act + string result = Schema.SpanNames.CreateOrchestration(name, version); + + // Assert + Assert.Equal($"{TraceActivityConstants.CreateOrchestration}:{name}", result); + } + + [Fact] + public void CreateOrchestration_WithVersion_ReturnsExpectedFormat() + { + // Arrange + string name = "MyOrchestration"; + string version = "1.0"; + + // Act + string result = Schema.SpanNames.CreateOrchestration(name, version); + + // Assert + Assert.Equal($"{TraceActivityConstants.CreateOrchestration}:{name}@{version}", result); + } + + [Theory] + [InlineData(true, "test-instance-123")] + [InlineData(true, "abc-def-ghi")] + public void CreateOrchestration_WithIncludeInstanceIdTrue_IncludesInstanceId(bool includeInstanceId, string instanceId) + { + // Arrange + string name = "MyOrchestration"; + string version = null; + + // Act + string result = Schema.SpanNames.CreateOrchestration(name, version, instanceId, includeInstanceId); + + // Assert + Assert.Equal($"{TraceActivityConstants.CreateOrchestration}:{name} ({instanceId})", result); + } + + [Theory] + [InlineData(false, "test-instance-123")] + [InlineData(false, "abc-def-ghi")] + public void CreateOrchestration_WithIncludeInstanceIdFalse_ExcludesInstanceId(bool includeInstanceId, string instanceId) + { + // Arrange + string name = "MyOrchestration"; + string version = null; + + // Act + string result = Schema.SpanNames.CreateOrchestration(name, version, instanceId, includeInstanceId); + + // Assert + Assert.Equal($"{TraceActivityConstants.CreateOrchestration}:{name}", result); + } + + [Fact] + public void CreateOrchestration_WithNullInstanceId_ExcludesInstanceId() + { + // Arrange + string name = "MyOrchestration"; + string version = null; + string instanceId = null; + + // Act + string result = Schema.SpanNames.CreateOrchestration(name, version, instanceId, includeInstanceId: true); + + // Assert + Assert.Equal($"{TraceActivityConstants.CreateOrchestration}:{name}", result); + } + + [Fact] + public void CreateOrchestration_WithEmptyInstanceId_ExcludesInstanceId() + { + // Arrange + string name = "MyOrchestration"; + string version = null; + string instanceId = string.Empty; + + // Act + string result = Schema.SpanNames.CreateOrchestration(name, version, instanceId, includeInstanceId: true); + + // Assert + Assert.Equal($"{TraceActivityConstants.CreateOrchestration}:{name}", result); + } + + [Fact] + public void CreateOrchestration_WithVersionAndInstanceId_ReturnsExpectedFormat() + { + // Arrange + string name = "MyOrchestration"; + string version = "1.0"; + string instanceId = "test-instance-123"; + + // Act + string result = Schema.SpanNames.CreateOrchestration(name, version, instanceId, includeInstanceId: true); + + // Assert + Assert.Equal($"{TraceActivityConstants.CreateOrchestration}:{name}@{version} ({instanceId})", result); + } + + [Fact] + public void CallOrSignalEntity_WithoutInstanceId_ReturnsExpectedFormat() + { + // Arrange + string name = "Counter"; + string operation = "Add"; + + // Act + string result = Schema.SpanNames.CallOrSignalEntity(name, operation); + + // Assert + Assert.Equal($"{TraceActivityConstants.Entity}:{name}:{operation}", result); + } + + [Fact] + public void CallOrSignalEntity_WithIncludeInstanceIdTrue_IncludesInstanceId() + { + // Arrange + string name = "Counter"; + string operation = "Add"; + string instanceId = "entity-instance-123"; + + // Act + string result = Schema.SpanNames.CallOrSignalEntity(name, operation, instanceId, includeInstanceId: true); + + // Assert + Assert.Equal($"{TraceActivityConstants.Entity}:{name}:{operation} ({instanceId})", result); + } + + [Fact] + public void CallOrSignalEntity_WithIncludeInstanceIdFalse_ExcludesInstanceId() + { + // Arrange + string name = "Counter"; + string operation = "Add"; + string instanceId = "entity-instance-123"; + + // Act + string result = Schema.SpanNames.CallOrSignalEntity(name, operation, instanceId, includeInstanceId: false); + + // Assert + Assert.Equal($"{TraceActivityConstants.Entity}:{name}:{operation}", result); + } + + [Fact] + public void EntityStartsAnOrchestration_WithoutInstanceId_ReturnsExpectedFormat() + { + // Arrange + string name = "Counter"; + + // Act + string result = Schema.SpanNames.EntityStartsAnOrchestration(name); + + // Assert + Assert.Equal($"{name}:{TraceActivityConstants.CreateOrchestration}", result); + } + + [Fact] + public void EntityStartsAnOrchestration_WithIncludeInstanceIdTrue_IncludesInstanceId() + { + // Arrange + string name = "Counter"; + string instanceId = "orch-instance-123"; + + // Act + string result = Schema.SpanNames.EntityStartsAnOrchestration(name, instanceId, includeInstanceId: true); + + // Assert + Assert.Equal($"{name}:{TraceActivityConstants.CreateOrchestration} ({instanceId})", result); + } + + [Fact] + public void EntityStartsAnOrchestration_WithIncludeInstanceIdFalse_ExcludesInstanceId() + { + // Arrange + string name = "Counter"; + string instanceId = "orch-instance-123"; + + // Act + string result = Schema.SpanNames.EntityStartsAnOrchestration(name, instanceId, includeInstanceId: false); + + // Assert + Assert.Equal($"{name}:{TraceActivityConstants.CreateOrchestration}", result); + } + } +} From 54ab1aa8f71c35c128bdfd0b37ae21ddb0a0170f Mon Sep 17 00:00:00 2001 From: Tomer Rosenthal Date: Tue, 2 Dec 2025 10:16:21 -0800 Subject: [PATCH 2/2] Add Category trait to unit tests for CI compatibility --- test/Common/SchemaSpanNamesTests.cs | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/test/Common/SchemaSpanNamesTests.cs b/test/Common/SchemaSpanNamesTests.cs index 8cb0b8638..f4ff64cc6 100644 --- a/test/Common/SchemaSpanNamesTests.cs +++ b/test/Common/SchemaSpanNamesTests.cs @@ -12,6 +12,7 @@ namespace Microsoft.Azure.WebJobs.Extensions.DurableTask.Tests public class SchemaSpanNamesTests { [Fact] + [Trait("Category", PlatformSpecificHelpers.TestCategory)] public void CreateOrchestration_WithoutInstanceId_ReturnsExpectedFormat() { // Arrange @@ -26,6 +27,7 @@ public void CreateOrchestration_WithoutInstanceId_ReturnsExpectedFormat() } [Fact] + [Trait("Category", PlatformSpecificHelpers.TestCategory)] public void CreateOrchestration_WithVersion_ReturnsExpectedFormat() { // Arrange @@ -40,6 +42,7 @@ public void CreateOrchestration_WithVersion_ReturnsExpectedFormat() } [Theory] + [Trait("Category", PlatformSpecificHelpers.TestCategory)] [InlineData(true, "test-instance-123")] [InlineData(true, "abc-def-ghi")] public void CreateOrchestration_WithIncludeInstanceIdTrue_IncludesInstanceId(bool includeInstanceId, string instanceId) @@ -56,6 +59,7 @@ public void CreateOrchestration_WithIncludeInstanceIdTrue_IncludesInstanceId(boo } [Theory] + [Trait("Category", PlatformSpecificHelpers.TestCategory)] [InlineData(false, "test-instance-123")] [InlineData(false, "abc-def-ghi")] public void CreateOrchestration_WithIncludeInstanceIdFalse_ExcludesInstanceId(bool includeInstanceId, string instanceId) @@ -72,6 +76,7 @@ public void CreateOrchestration_WithIncludeInstanceIdFalse_ExcludesInstanceId(bo } [Fact] + [Trait("Category", PlatformSpecificHelpers.TestCategory)] public void CreateOrchestration_WithNullInstanceId_ExcludesInstanceId() { // Arrange @@ -87,6 +92,7 @@ public void CreateOrchestration_WithNullInstanceId_ExcludesInstanceId() } [Fact] + [Trait("Category", PlatformSpecificHelpers.TestCategory)] public void CreateOrchestration_WithEmptyInstanceId_ExcludesInstanceId() { // Arrange @@ -102,6 +108,7 @@ public void CreateOrchestration_WithEmptyInstanceId_ExcludesInstanceId() } [Fact] + [Trait("Category", PlatformSpecificHelpers.TestCategory)] public void CreateOrchestration_WithVersionAndInstanceId_ReturnsExpectedFormat() { // Arrange @@ -117,6 +124,7 @@ public void CreateOrchestration_WithVersionAndInstanceId_ReturnsExpectedFormat() } [Fact] + [Trait("Category", PlatformSpecificHelpers.TestCategory)] public void CallOrSignalEntity_WithoutInstanceId_ReturnsExpectedFormat() { // Arrange @@ -131,6 +139,7 @@ public void CallOrSignalEntity_WithoutInstanceId_ReturnsExpectedFormat() } [Fact] + [Trait("Category", PlatformSpecificHelpers.TestCategory)] public void CallOrSignalEntity_WithIncludeInstanceIdTrue_IncludesInstanceId() { // Arrange @@ -146,6 +155,7 @@ public void CallOrSignalEntity_WithIncludeInstanceIdTrue_IncludesInstanceId() } [Fact] + [Trait("Category", PlatformSpecificHelpers.TestCategory)] public void CallOrSignalEntity_WithIncludeInstanceIdFalse_ExcludesInstanceId() { // Arrange @@ -161,6 +171,7 @@ public void CallOrSignalEntity_WithIncludeInstanceIdFalse_ExcludesInstanceId() } [Fact] + [Trait("Category", PlatformSpecificHelpers.TestCategory)] public void EntityStartsAnOrchestration_WithoutInstanceId_ReturnsExpectedFormat() { // Arrange @@ -174,6 +185,7 @@ public void EntityStartsAnOrchestration_WithoutInstanceId_ReturnsExpectedFormat( } [Fact] + [Trait("Category", PlatformSpecificHelpers.TestCategory)] public void EntityStartsAnOrchestration_WithIncludeInstanceIdTrue_IncludesInstanceId() { // Arrange @@ -188,6 +200,7 @@ public void EntityStartsAnOrchestration_WithIncludeInstanceIdTrue_IncludesInstan } [Fact] + [Trait("Category", PlatformSpecificHelpers.TestCategory)] public void EntityStartsAnOrchestration_WithIncludeInstanceIdFalse_ExcludesInstanceId() { // Arrange