Distributed Tracing for Entities (In-Process)

src/DurableTask.Core/Entities/EventFormat/RequestMessage.cs

@@ -98,6 +98,12 @@ internal class RequestMessage : EntityMessage
         [DataMember]
         public bool IsLockRequest => LockSet != null;
 
+        /// <summary>
+        /// Whether this message already has a trace created for it
+        /// </summary>
+        [DataMember(Name = "traceCreated")]
+        public bool TraceCreated { get; set; }
+
         /// <inheritdoc/>
         public override string GetShortDescription()
         {

src/DurableTask.Core/OrchestrationTags.cs

@@ -36,6 +36,16 @@ public static class OrchestrationTags
         /// </remarks>
         public const string FireAndForget = "FireAndForget";
 
+        /// <summary>
+        /// The ID of the parent trace that created this orchestration instance (see https://www.w3.org/TR/trace-context/#traceparent-header)
+        /// </summary>
+        public const string TraceParent = "TraceParent";
+
+        /// <summary>
+        /// The trace state of the parent trace that created this orchestration instance (see https://www.w3.org/TR/trace-context/#tracestate-header)
+        /// </summary>
+        public const string TraceState = "TraceState";
+
         /// <summary>
         /// Check whether the given tags contain the fire and forget tag
         /// </summary>

src/DurableTask.Core/TaskOrchestrationDispatcher.cs

@@ -425,12 +425,29 @@ protected async Task<bool> OnProcessWorkItemAsync(TaskOrchestrationWorkItem work
                                     break;
                                 case OrchestratorActionType.CreateSubOrchestration:
                                     var createSubOrchestrationAction = (CreateSubOrchestrationAction)decision;
+                                    ActivityContext? parentTraceContext = traceActivity?.Context;
+                                    var spanId = ActivitySpanId.CreateRandom();
+                                    if (createSubOrchestrationAction.Tags != null 
+                                        && createSubOrchestrationAction.Tags.TryGetValue(OrchestrationTags.TraceParent, out string traceParent) 
+                                        && createSubOrchestrationAction.Tags.TryGetValue(OrchestrationTags.TraceState, out string traceState))
+                                    {
+                                        try
+                                        {
+                                            parentTraceContext = ActivityContext.Parse(traceParent, traceState);
+                                            spanId = parentTraceContext.Value.SpanId;
+                                        }
+                                        catch (Exception e)
+                                        {
+                                            TraceHelper.TraceException(TraceEventType.Error, "TaskOrchestrationDispatcher-CreateSubOrchestration-MalformedParentTrace", e);
+                                        }
+                                    }
                                     orchestratorMessages.Add(
                                         this.ProcessCreateSubOrchestrationInstanceDecision(
                                             createSubOrchestrationAction,
                                             runtimeState,
                                             this.IncludeParameters,
-                                            traceActivity));
+                                            parentTraceContext,
+                                            spanId));
                                     break;
                                 case OrchestratorActionType.SendEvent:
                                     var sendEventAction = (SendEventOrchestratorAction)decision;
@@ -1083,7 +1100,8 @@ TaskMessage ProcessCreateSubOrchestrationInstanceDecision(
             CreateSubOrchestrationAction createSubOrchestrationAction,
             OrchestrationRuntimeState runtimeState,
             bool includeParameters,
-            Activity? parentTraceActivity)
+            ActivityContext? parentTraceContext,
+            ActivitySpanId spanId)
         {
             var historyEvent = new SubOrchestrationInstanceCreatedEvent(createSubOrchestrationAction.Id)
             {
@@ -1096,8 +1114,7 @@ TaskMessage ProcessCreateSubOrchestrationInstanceDecision(
                 historyEvent.Input = createSubOrchestrationAction.Input;
             }
 
-            ActivitySpanId clientSpanId = ActivitySpanId.CreateRandom();
-            historyEvent.ClientSpanId = clientSpanId.ToString();
+            historyEvent.ClientSpanId = spanId.ToString();
 
             runtimeState.AddEvent(historyEvent);
 
@@ -1122,9 +1139,13 @@ TaskMessage ProcessCreateSubOrchestrationInstanceDecision(
                 Version = createSubOrchestrationAction.Version
             };
 
-            if (parentTraceActivity != null)
+            if (parentTraceContext is ActivityContext parentContext)
             {
-                ActivityContext activityContext = new ActivityContext(parentTraceActivity.TraceId, clientSpanId, parentTraceActivity.ActivityTraceFlags, parentTraceActivity.TraceStateString);
+                ActivityContext activityContext = new ActivityContext(
+                    parentContext.TraceId, 
+                    spanId,
+                    parentContext.TraceFlags,
+                    parentContext.TraceState);
                 startedEvent.SetParentTraceContext(activityContext);
             }
 

src/DurableTask.Core/Tracing/TraceHelper.cs

@@ -54,7 +54,7 @@ public class TraceHelper
                 {
                     newActivity.SetTag(Schema.Task.Version, startEvent.Version);
                 }
-                
+
                 startEvent.SetParentTraceContext(newActivity);
             }
 
@@ -357,6 +357,12 @@ internal static void EmitTraceActivityForSubOrchestrationFailed(
             OrchestrationInstance? instance,
             string? targetInstanceId)
         {
+            // There is a possibility that we mislabel the event as an entity event if entities are not enabled
+            if (Entities.IsEntityInstance(targetInstanceId ?? string.Empty) || Entities.IsEntityInstance(instance?.InstanceId ?? string.Empty))
+            { 
+                return null;
+            }
+
             Activity? newActivity = ActivityTraceSource.StartActivity(
                 CreateSpanName(TraceActivityConstants.OrchestrationEvent, eventRaisedEvent.Name, null),
                 kind: ActivityKind.Producer,
@@ -390,6 +396,12 @@ internal static void EmitTraceActivityForSubOrchestrationFailed(
         /// </returns>
         internal static Activity? StartActivityForNewEventRaisedFromClient(EventRaisedEvent eventRaised, OrchestrationInstance instance)
         {
+            // There is a possibility that we mislabel the event as an entity event if entities are not enabled
+            if (Entities.IsEntityInstance(instance.InstanceId))
+            {
+                return null;
+            }
+
             Activity? newActivity = ActivityTraceSource.StartActivity(
                 CreateSpanName(TraceActivityConstants.OrchestrationEvent, eventRaised.Name, null),
                 kind: ActivityKind.Producer,