Distributed Tracing for Entities (In-Process)

src/DurableTask.Core/OrchestrationTags.cs

@@ -36,6 +36,16 @@ public static class OrchestrationTags
         /// </remarks>
         public const string FireAndForget = "FireAndForget";
 
+        /// <summary>
+        /// The ID of the parent trace that created this orchestration instance (see https://www.w3.org/TR/trace-context/#traceparent-header)
+        /// </summary>
+        public const string TraceParent = "MS_Entities_TraceParent";
+
+        /// <summary>
+        /// The trace state of the parent trace that created this orchestration instance (see https://www.w3.org/TR/trace-context/#tracestate-header)
+        /// </summary>
+        public const string TraceState = "MS_Entities_TraceState";
+
         /// <summary>
         /// Check whether the given tags contain the fire and forget tag
         /// </summary>

src/DurableTask.Core/TaskHubClient.cs

@@ -746,6 +746,19 @@ void CreateAndTrackDependencyTelemetry(TraceContextBase? requestTraceContext)
         /// <param name="eventName">Name of the event</param>
         /// <param name="eventData">Data for the event</param>
         public async Task RaiseEventAsync(OrchestrationInstance orchestrationInstance, string eventName, object eventData)
+        {
+            await this.RaiseEventAsync(orchestrationInstance, eventName, eventData, emitTraceActivity: true);
+        }
+
+        /// <summary>
+        ///     Raises an event in the specified orchestration instance, which eventually causes the OnEvent() method in the
+        ///     orchestration to fire.
+        /// </summary>
+        /// <param name="orchestrationInstance">Instance in which to raise the event</param>
+        /// <param name="eventName">Name of the event</param>
+        /// <param name="eventData">Data for the event</param>
+        /// <param name="emitTraceActivity">Whether or not to emit a trace activity for this event.</param>
+        public async Task RaiseEventAsync(OrchestrationInstance orchestrationInstance, string eventName, object eventData, bool emitTraceActivity = true)
         {
 
             if (string.IsNullOrWhiteSpace(orchestrationInstance.InstanceId))
@@ -757,7 +770,11 @@ public async Task RaiseEventAsync(OrchestrationInstance orchestrationInstance, s
 
             // Distributed Tracing
             EventRaisedEvent eventRaisedEvent = new EventRaisedEvent(-1, serializedInput) { Name = eventName };
-            using Activity? traceActivity = TraceHelper.StartActivityForNewEventRaisedFromClient(eventRaisedEvent, orchestrationInstance);
+            Activity? traceActivity = null;
+            if (emitTraceActivity)
+            {
+                traceActivity = TraceHelper.StartActivityForNewEventRaisedFromClient(eventRaisedEvent, orchestrationInstance);
+            }
 
             var taskMessage = new TaskMessage
             {
@@ -776,6 +793,10 @@ public async Task RaiseEventAsync(OrchestrationInstance orchestrationInstance, s
                 TraceHelper.AddErrorDetailsToSpan(traceActivity, e);
                 throw;
             }
+            finally
+            {
+                traceActivity?.Dispose();
+            }
         }
 
         /// <summary>

src/DurableTask.Core/TaskOrchestrationDispatcher.cs

@@ -116,6 +116,11 @@ public async Task StopAsync(bool forced)
         /// </summary>
         public bool IncludeParameters { get; set; }
 
+        /// <summary>
+        /// Gets or sets the flag for whether or not entities are enabled
+        /// </summary>
+        public bool EntitiesEnabled { get; set; }
+
         /// <summary>
         /// Method to get the next work item to process within supplied timeout
         /// </summary>
@@ -1140,9 +1145,6 @@ TaskMessage ProcessCreateSubOrchestrationInstanceDecision(
                 historyEvent.Input = createSubOrchestrationAction.Input;
             }
 
-            ActivitySpanId clientSpanId = ActivitySpanId.CreateRandom();
-            historyEvent.ClientSpanId = clientSpanId.ToString();
-
             runtimeState.AddEvent(historyEvent);
 
             var taskMessage = new TaskMessage();
@@ -1166,8 +1168,22 @@ TaskMessage ProcessCreateSubOrchestrationInstanceDecision(
                 Version = createSubOrchestrationAction.Version
             };
 
-            if (parentTraceActivity != null)
+            // If a parent trace context was provided via the CreateSubOrchestrationAction.Tags, we will use this as the parent trace context of the suborchestration execution Activity rather than Activity.Current.Context.
+            if (createSubOrchestrationAction.Tags != null
+                && createSubOrchestrationAction.Tags.TryGetValue(OrchestrationTags.TraceParent, out string traceParent))
+            {
+                // If a parent trace context was provided but we fail to parse it, we don't want to attach any parent trace context to the start event since that will incorrectly link the trace corresponding to the orchestration execution
+                // as a child of Activity.Current, which is not truly the parent of the request
+                if (createSubOrchestrationAction.Tags.TryGetValue(OrchestrationTags.TraceState, out string traceState)
+                    && ActivityContext.TryParse(traceParent, traceState, out ActivityContext parentTraceContext))
+                {
+                    startedEvent.SetParentTraceContext(parentTraceContext);
+                }
+            }
+            else if (parentTraceActivity != null)
             {
+                ActivitySpanId clientSpanId = ActivitySpanId.CreateRandom();
+                historyEvent.ClientSpanId = clientSpanId.ToString();
                 ActivityContext activityContext = new ActivityContext(parentTraceActivity.TraceId, clientSpanId, parentTraceActivity.ActivityTraceFlags, parentTraceActivity.TraceStateString);
                 startedEvent.SetParentTraceContext(activityContext);
             }
@@ -1201,7 +1217,7 @@ TaskMessage ProcessSendEventDecision(
 
             // Distributed Tracing: start a new trace activity derived from the orchestration
             // for an EventRaisedEvent (external event)
-            using Activity? traceActivity = TraceHelper.StartTraceActivityForEventRaisedFromWorker(eventRaisedEvent, runtimeState.OrchestrationInstance, sendEventAction.Instance?.InstanceId);
+            using Activity? traceActivity = TraceHelper.StartTraceActivityForEventRaisedFromWorker(eventRaisedEvent, runtimeState.OrchestrationInstance, this.EntitiesEnabled, sendEventAction.Instance?.InstanceId);
 
             this.logHelper.RaisingEvent(runtimeState.OrchestrationInstance!, historyEvent);
 

src/DurableTask.Core/Tracing/TraceHelper.cs

@@ -284,7 +284,10 @@ internal static void EmitTraceActivityForTaskFailed(
                 return null;
             }
 
-            activity.SetSpanId(createdEvent.ClientSpanId);
+            if (!string.IsNullOrEmpty(createdEvent.ClientSpanId))
+            {
+                activity.SetSpanId(createdEvent.ClientSpanId);
+            }
 
             activity.SetTag(Schema.Task.Type, TraceActivityConstants.Orchestration);
             activity.SetTag(Schema.Task.Name, createdEvent.Name);
@@ -348,15 +351,23 @@ internal static void EmitTraceActivityForSubOrchestrationFailed(
         /// </summary>
         /// <param name="eventRaisedEvent">The associated <see cref="EventRaisedEvent"/>.</param>
         /// <param name="instance">The associated <see cref="OrchestrationInstance"/>.</param>
+        /// <param name="entitiesEnabled">Whether or not entities are enabled, meaning this event could possibly correspond to entity.</param>
         /// <param name="targetInstanceId">The instance id of the orchestration that will receive the event.</param>
         /// <returns>
         /// Returns a newly started <see cref="Activity"/> with (task) activity and orchestration-specific metadata.
         /// </returns>
         internal static Activity? StartTraceActivityForEventRaisedFromWorker(
             EventRaisedEvent eventRaisedEvent,
             OrchestrationInstance? instance,
+            bool entitiesEnabled,
             string? targetInstanceId)
         {
+            // We don't want to emit tracing for external events when they are related to entities
+            if (entitiesEnabled && (Entities.IsEntityInstance(targetInstanceId ?? string.Empty) || Entities.IsEntityInstance(instance?.InstanceId ?? string.Empty)))
+            {
+                return null;
+            }
+
             Activity? newActivity = ActivityTraceSource.StartActivity(
                 CreateSpanName(TraceActivityConstants.OrchestrationEvent, eventRaisedEvent.Name, null),
                 kind: ActivityKind.Producer,