diff --git a/src/DurableTask.Core/Entities/ClientEntityHelpers.cs b/src/DurableTask.Core/Entities/ClientEntityHelpers.cs
index 94bc0512a..155eb1eca 100644
--- a/src/DurableTask.Core/Entities/ClientEntityHelpers.cs
+++ b/src/DurableTask.Core/Entities/ClientEntityHelpers.cs
@@ -18,6 +18,7 @@ namespace DurableTask.Core.Entities
using Newtonsoft.Json.Linq;
using Newtonsoft.Json;
using System;
+ using DurableTask.Core.Tracing;
///
/// Utility functions for clients that interact with entities, either by sending events or by accessing the entity state directly in storage
@@ -32,8 +33,11 @@ public static class ClientEntityHelpers
/// The name of the operation.
/// The serialized input for the operation.
/// The time to schedule this signal, or null if not a scheduled signal
+ /// The parent trace context for this operation.
+ /// The time at which the request was made.
+ /// Whether to create a trace for this signal operation.
/// The event to send.
- public static EntityMessageEvent EmitOperationSignal(OrchestrationInstance targetInstance, Guid requestId, string operationName, string? input, (DateTime Original, DateTime Capped)? scheduledTimeUtc)
+ public static EntityMessageEvent EmitOperationSignal(OrchestrationInstance targetInstance, Guid requestId, string operationName, string? input, (DateTime Original, DateTime Capped)? scheduledTimeUtc, DistributedTraceContext? parentTraceContext = null, DateTimeOffset? requestTime = null, bool createTrace = false)
{
var request = new RequestMessage()
{
@@ -44,6 +48,9 @@ public static EntityMessageEvent EmitOperationSignal(OrchestrationInstance targe
Operation = operationName,
ScheduledTime = scheduledTimeUtc?.Original,
Input = input,
+ ParentTraceContext = parentTraceContext,
+ RequestTime = requestTime,
+ CreateTrace = createTrace,
};
var eventName = scheduledTimeUtc.HasValue
diff --git a/src/DurableTask.Core/Entities/EventFormat/RequestMessage.cs b/src/DurableTask.Core/Entities/EventFormat/RequestMessage.cs
index e46d1759f..ce355ab12 100644
--- a/src/DurableTask.Core/Entities/EventFormat/RequestMessage.cs
+++ b/src/DurableTask.Core/Entities/EventFormat/RequestMessage.cs
@@ -13,6 +13,7 @@
#nullable enable
namespace DurableTask.Core.Entities.EventFormat
{
+ using DurableTask.Core.Tracing;
using System;
using System.Runtime.Serialization;
@@ -98,6 +99,29 @@ internal class RequestMessage : EntityMessage
[DataMember]
public bool IsLockRequest => LockSet != null;
+ ///
+ /// Parent trace context of this request message.
+ ///
+ [DataMember(Name = "parentTraceContext", EmitDefaultValue = false)]
+ public DistributedTraceContext? ParentTraceContext { get; set; }
+
+ ///
+ /// Whether or not to create an entity-specific trace for this request message
+ ///
+ [DataMember(Name = "createTrace")]
+ public bool CreateTrace { get; set; }
+
+ ///
+ /// The time the request was generated.
+ ///
+ [DataMember(Name = "requestTime", EmitDefaultValue = false)]
+ public DateTimeOffset? RequestTime { get; set; }
+
+ ///
+ /// The client span ID of this request.
+ ///
+ public string? ClientSpanId { get; set; }
+
///
public override string GetShortDescription()
{
diff --git a/src/DurableTask.Core/Entities/OperationFormat/OperationRequest.cs b/src/DurableTask.Core/Entities/OperationFormat/OperationRequest.cs
index ab249f88f..6b75c65c0 100644
--- a/src/DurableTask.Core/Entities/OperationFormat/OperationRequest.cs
+++ b/src/DurableTask.Core/Entities/OperationFormat/OperationRequest.cs
@@ -13,6 +13,7 @@
#nullable enable
namespace DurableTask.Core.Entities.OperationFormat
{
+ using DurableTask.Core.Tracing;
using System;
///
@@ -37,5 +38,10 @@ public class OperationRequest
/// The input for the operation. Can be null if no input was given.
///
public string? Input { get; set; }
+
+ ///
+ /// The trace context for the operation, if any.
+ ///
+ public DistributedTraceContext? TraceContext { get; set; }
}
}
\ No newline at end of file
diff --git a/src/DurableTask.Core/Entities/OperationFormat/OperationResult.cs b/src/DurableTask.Core/Entities/OperationFormat/OperationResult.cs
index 01cc41b11..a4c97fa8b 100644
--- a/src/DurableTask.Core/Entities/OperationFormat/OperationResult.cs
+++ b/src/DurableTask.Core/Entities/OperationFormat/OperationResult.cs
@@ -11,6 +11,8 @@
// limitations under the License.
// ----------------------------------------------------------------------------------
#nullable enable
+using System;
+
namespace DurableTask.Core.Entities.OperationFormat
{
///
@@ -46,5 +48,15 @@ public bool IsError
/// this field exclusively when collecting error information.
///
public FailureDetails? FailureDetails { get; set; }
+
+ ///
+ /// The start time of the operation.
+ ///
+ public DateTime? StartTimeUtc { get; set; }
+
+ ///
+ /// The completion time of the operation.
+ ///
+ public DateTime? EndTimeUtc { get; set; }
}
}
diff --git a/src/DurableTask.Core/Entities/OperationFormat/SendSignalOperationAction.cs b/src/DurableTask.Core/Entities/OperationFormat/SendSignalOperationAction.cs
index 04531ac31..fe22cbf49 100644
--- a/src/DurableTask.Core/Entities/OperationFormat/SendSignalOperationAction.cs
+++ b/src/DurableTask.Core/Entities/OperationFormat/SendSignalOperationAction.cs
@@ -13,6 +13,7 @@
#nullable enable
namespace DurableTask.Core.Entities.OperationFormat
{
+ using DurableTask.Core.Tracing;
using System;
///
@@ -45,5 +46,15 @@ public class SendSignalOperationAction : OperationAction
/// Optionally, a scheduled delivery time for the signal.
///
public DateTime? ScheduledTime { get; set; }
+
+ ///
+ /// The time the signal request was generated.
+ ///
+ public DateTimeOffset? RequestTime { get; set; }
+
+ ///
+ /// The parent trace context for the signal, if any.
+ ///
+ public DistributedTraceContext? ParentTraceContext { get; set; }
}
}
\ No newline at end of file
diff --git a/src/DurableTask.Core/Entities/OperationFormat/StartNewOrchestrationOperationAction.cs b/src/DurableTask.Core/Entities/OperationFormat/StartNewOrchestrationOperationAction.cs
index 4c06f80cd..29aca841b 100644
--- a/src/DurableTask.Core/Entities/OperationFormat/StartNewOrchestrationOperationAction.cs
+++ b/src/DurableTask.Core/Entities/OperationFormat/StartNewOrchestrationOperationAction.cs
@@ -13,6 +13,7 @@
#nullable enable
namespace DurableTask.Core.Entities.OperationFormat
{
+ using DurableTask.Core.Tracing;
using System;
using System.Collections.Generic;
@@ -52,5 +53,15 @@ public class StartNewOrchestrationOperationAction : OperationAction
///
public DateTime? ScheduledStartTime { get; set; }
+ ///
+ /// The time of the new orchestration request creation.
+ ///
+ public DateTimeOffset? RequestTime { get; set; }
+
+ ///
+ /// The parent trace context for the operation, if any.
+ ///
+ public DistributedTraceContext? ParentTraceContext { get; set; }
+
}
}
\ No newline at end of file
diff --git a/src/DurableTask.Core/Entities/OrchestrationEntityContext.cs b/src/DurableTask.Core/Entities/OrchestrationEntityContext.cs
index 3e21f31a6..8a84df162 100644
--- a/src/DurableTask.Core/Entities/OrchestrationEntityContext.cs
+++ b/src/DurableTask.Core/Entities/OrchestrationEntityContext.cs
@@ -228,6 +228,8 @@ public IEnumerable EmitLockReleaseMessages()
/// A unique identifier for this request.
/// A time for which to schedule the delivery, or null if this is not a scheduled message
/// The operation input
+ /// The time at which the request was made.
+ /// Whether or not to create an entity-specific trace for this event
/// The event to send.
public EntityMessageEvent EmitRequestMessage(
OrchestrationInstance target,
@@ -235,7 +237,9 @@ public EntityMessageEvent EmitRequestMessage(
bool oneWay,
Guid operationId,
(DateTime Original, DateTime Capped)? scheduledTimeUtc,
- string? input)
+ string? input,
+ DateTimeOffset? requestTime = null,
+ bool createTrace = false)
{
this.CheckEntitySupport();
@@ -248,6 +252,8 @@ public EntityMessageEvent EmitRequestMessage(
Operation = operationName,
ScheduledTime = scheduledTimeUtc?.Original,
Input = input,
+ RequestTime = requestTime,
+ CreateTrace = createTrace,
};
this.AdjustOutgoingMessage(target.InstanceId, request, scheduledTimeUtc?.Capped, out string eventName);
diff --git a/src/DurableTask.Core/OrchestrationTags.cs b/src/DurableTask.Core/OrchestrationTags.cs
index c3065de46..00ff3a2b1 100644
--- a/src/DurableTask.Core/OrchestrationTags.cs
+++ b/src/DurableTask.Core/OrchestrationTags.cs
@@ -46,6 +46,16 @@ public static class OrchestrationTags
///
public const string TraceState = "MS_Entities_TraceState";
+ ///
+ /// The time the request for a new orchestration was created.
+ ///
+ public const string RequestTime = "MS_Entities_RequestTime";
+
+ ///
+ /// Whether or not to create a trace for the of the orchestration
+ ///
+ public const string CreateTraceForNewOrchestration = "MS_CreateTrace";
+
///
/// Check whether the given tags contain the fire and forget tag
///
diff --git a/src/DurableTask.Core/TaskEntityDispatcher.cs b/src/DurableTask.Core/TaskEntityDispatcher.cs
index aad78675d..a91ae97e2 100644
--- a/src/DurableTask.Core/TaskEntityDispatcher.cs
+++ b/src/DurableTask.Core/TaskEntityDispatcher.cs
@@ -504,18 +504,28 @@ void DetermineWork(OrchestrationRuntimeState runtimeState, out SchedulerState sc
// We handle this by rescheduling the message instead of processing it.
deliverNow = Array.Empty();
batch.AddMessageToBeRescheduled(requestMessage);
+
+ // We do not want to create the Activity for the request yet since it will be redelivered again later. In the case that the parent trace context was attached
+ // to the EventRaisedEvent and not the RequestMessage, we want to attach it to the RequestMessage such that when it is redelivered the parent trace context can be used
+ // to create the Activity for the request then.
+ if (requestMessage.ParentTraceContext == null && eventRaisedEvent.ParentTraceContext != null)
+ {
+ requestMessage.ParentTraceContext = eventRaisedEvent.ParentTraceContext;
+ }
}
else
{
// the message is scheduled to be delivered immediately.
// There are no FIFO guarantees for scheduled messages, so we skip the message sorter.
deliverNow = new RequestMessage[] { requestMessage };
+ StartTraceActivityForSignalingEntity(requestMessage, eventRaisedEvent, instanceId);
}
}
else
{
// run this through the message sorter to help with reordering and duplicate filtering
deliverNow = schedulerState.MessageSorter.ReceiveInOrder(requestMessage, this.entityBackendProperties.EntityMessageReorderWindow);
+ StartTraceActivityForSignalingEntity(requestMessage, eventRaisedEvent, instanceId);
}
foreach (var message in deliverNow)
@@ -647,20 +657,50 @@ public void ToBeContinued(SchedulerState schedulerState)
}
}
- public List GetOperationRequests()
+ public (List, List) GetOperationRequestsAndTraceActivities(string instanceId)
{
var operations = new List(this.operationBatch.Count);
+ var traceActivities = new List(this.operationBatch.Count);
for (int i = 0; i < this.operationBatch.Count; i++)
{
var request = this.operationBatch[i];
+
+ Activity traceActivity = null;
+ // We only want to create a trace activity for processing the entity invocation in the case that we can successfully parse the trace context of the request that led to this entity invocation.
+ // Otherwise, we will create an unlinked trace activity with no parent
+ if (ActivityContext.TryParse(request.ParentTraceContext?.TraceParent, request.ParentTraceContext?.TraceState, out ActivityContext parentTraceContext))
+ {
+ if (!request.IsSignal)
+ {
+ var clientSpanId = ActivitySpanId.CreateRandom();
+
+ // In that case that we are processing a call request as a server, we want to generate a new span ID that will also be used by the Activity we create at the end corresponding to the client call request
+ // That way, this server Activity corresponding to processing the call request will be correctly linked as the child of the Activity for the client call request.
+ parentTraceContext = new ActivityContext(parentTraceContext.TraceId, clientSpanId, parentTraceContext.TraceFlags, parentTraceContext.TraceState);
+ request.ClientSpanId = clientSpanId.ToString();
+ }
+ traceActivity = TraceHelper.StartActivityForProcessingEntityInvocation(
+ instanceId,
+ EntityId.FromString(instanceId).Name,
+ request.Operation,
+ request.IsSignal,
+ parentTraceContext);
+ }
+
+ // We still want to add the trace activity to the list even if it was not successfully created and is null. This is because otherwise we have no easy way of mapping OperationResults to Activities otherwise if the lists
+ // do not have the same length in TraceHelper.EndActivitiesForProcessingEntityInvocation. We will simply skip ending the Activity if it is null in this method
+ traceActivities.Add(traceActivity);
+
+ // The trace context of the operation request will be the Activity just created - this can become the parent of future operations started by the entity once it processes the OperationRequest
operations.Add(new OperationRequest()
{
Operation = request.Operation,
Id = request.Id,
Input = request.Input,
+ TraceContext = traceActivity != null ? new DistributedTraceContext(traceActivity.Id, traceActivity.TraceStateString) : null,
});
}
- return operations;
+ return (operations, traceActivities);
}
public Queue RemoveDeferredWork(int index)
@@ -686,6 +726,22 @@ public Queue RemoveDeferredWork(int index)
void SendResultMessage(WorkItemEffects effects, RequestMessage request, OperationResult result)
{
+ // We only want to create a trace activity for calling an entity in the case that we can successfully get the parent trace context of the request.
+ // Otherwise, we will create an unlinked trace activity with no parent.
+ // Note that we create the Activity once the result has been sent to capture the full length of calling the entity and receiving its response.
+ if (ActivityContext.TryParse(request.ParentTraceContext?.TraceParent, request.ParentTraceContext?.TraceState, out ActivityContext parentTraceContext))
+ {
+ using var traceActivity = TraceHelper.StartActivityForCallingOrSignalingEntity(
+ effects.InstanceId,
+ EntityId.FromString(effects.InstanceId).Name,
+ request.Operation,
+ request.IsSignal,
+ request.ScheduledTime,
+ parentTraceContext,
+ request.RequestTime);
+ traceActivity?.SetSpanId(request.ClientSpanId);
+ }
+
var destination = new OrchestrationInstance()
{
InstanceId = request.ParentInstanceId,
@@ -728,6 +784,25 @@ void SendSignalMessage(WorkItemEffects effects, SchedulerState schedulerState, S
eventName = EntityMessageEventNames.RequestMessageEventName;
schedulerState.MessageSorter.LabelOutgoingMessage(message, action.InstanceId, DateTime.UtcNow, this.entityBackendProperties.EntityMessageReorderWindow);
}
+
+ // We only want to create a trace activity for signaling the entity in the case that we can successfully parse the parent trace context of the signal request.
+ // Otherwise, we will create an unlinked trace activity with no parent
+ if (ActivityContext.TryParse(action.ParentTraceContext?.TraceParent, action.ParentTraceContext?.TraceState, out ActivityContext parentTraceContext))
+ {
+ using var traceActivity = TraceHelper.StartActivityForCallingOrSignalingEntity(
+ destination.InstanceId,
+ EntityId.FromString(destination.InstanceId).Name,
+ action.Name,
+ signalEntity: true,
+ action.ScheduledTime,
+ parentTraceContext,
+ action.RequestTime,
+ entityId: effects.InstanceId);
+ if (traceActivity != null)
+ {
+ message.ParentTraceContext = new DistributedTraceContext(traceActivity.Id, traceActivity.TraceStateString);
+ }
+ }
this.ProcessSendEventMessage(effects, destination, eventName, message);
}
@@ -816,6 +891,23 @@ internal void ProcessSendStartMessage(WorkItemEffects effects, OrchestrationRunt
Name = action.Name,
Version = action.Version,
};
+
+ // We only want to create a trace activity for an entity starting an orchestration in the case that we can successfully parse the parent trace context of the start orchestration request.
+ // Otherwise, we will create an unlinked trace activity with no parent
+ if (ActivityContext.TryParse(action.ParentTraceContext?.TraceParent, action.ParentTraceContext?.TraceState, out ActivityContext parentTraceContext))
+ {
+ using var traceActivity = TraceHelper.StartActivityForEntityStartingAnOrchestration(
+ runtimeState.OrchestrationInstance.InstanceId,
+ EntityId.FromString(runtimeState.OrchestrationInstance.InstanceId).Name,
+ destination.InstanceId,
+ parentTraceContext,
+ action.RequestTime,
+ scheduledTime: action.ScheduledStartTime);
+ if (traceActivity != null)
+ {
+ executionStartedEvent.ParentTraceContext = new DistributedTraceContext(traceActivity.Id, traceActivity.TraceStateString);
+ }
+ }
this.logHelper.SchedulingOrchestration(executionStartedEvent);
effects.InstanceMessages.Add(new TaskMessage
@@ -829,12 +921,13 @@ internal void ProcessSendStartMessage(WorkItemEffects effects, OrchestrationRunt
async Task ExecuteViaMiddlewareAsync(Work workToDoNow, OrchestrationInstance instance, string serializedEntityState)
{
+ var (operations, traceActivities) = workToDoNow.GetOperationRequestsAndTraceActivities(instance.InstanceId);
// the request object that will be passed to the worker
var request = new EntityBatchRequest()
{
InstanceId = instance.InstanceId,
EntityState = serializedEntityState,
- Operations = workToDoNow.GetOperationRequests(),
+ Operations = operations,
};
this.logHelper.EntityBatchExecuting(request);
@@ -876,10 +969,57 @@ await this.dispatchPipeline.RunAsync(dispatchContext, async _ =>
});
var result = dispatchContext.GetProperty();
+ TraceHelper.EndActivitiesForProcessingEntityInvocation(traceActivities, result.Results, result.FailureDetails);
this.logHelper.EntityBatchExecuted(request, result);
return result;
}
+
+ private static void StartTraceActivityForSignalingEntity(RequestMessage requestMessage, EventRaisedEvent eventRaisedEvent, string instanceId)
+ {
+ if (requestMessage.CreateTrace)
+ {
+ // In the case that we are calling an entity, we want to create the Activity once the result for the call is returned and so we do not create now
+ if (requestMessage.IsSignal)
+ {
+ var successfullyParsed = false;
+ ActivityContext parentTraceContext;
+ if (requestMessage.ParentTraceContext is { } parentContext)
+ {
+ // If a parent trace context was provided but we fail to successfully parse it, we should not create the Activity even if the EventRaisedEvent has a parent trace context attached.
+ // Otherwise we will incorrectly link the created Activity to a context that is not truly its parent.
+ if (ActivityContext.TryParse(parentContext.TraceParent, parentContext.TraceState, out parentTraceContext))
+ {
+ successfullyParsed = true;
+ }
+ }
+ else if (eventRaisedEvent.TryGetParentTraceContext(out parentTraceContext))
+ {
+ successfullyParsed = true;
+ }
+ if (successfullyParsed)
+ {
+ using var traceActivity = TraceHelper.StartActivityForCallingOrSignalingEntity(
+ instanceId,
+ EntityId.FromString(instanceId).Name,
+ requestMessage.Operation,
+ requestMessage.IsSignal,
+ requestMessage.ScheduledTime,
+ parentTraceContext,
+ requestMessage.RequestTime);
+ if (traceActivity != null)
+ {
+ requestMessage.ParentTraceContext = new DistributedTraceContext(traceActivity.Id, traceActivity.TraceStateString);
+ }
+ }
+ }
+ // We still want to attach a parent trace context to the request in the case of a call to an entity so that when we create the Activity for the call this information is available.
+ else if (requestMessage.ParentTraceContext == null && eventRaisedEvent.ParentTraceContext != null)
+ {
+ requestMessage.ParentTraceContext = eventRaisedEvent.ParentTraceContext;
+ }
+ }
+ }
}
}
diff --git a/src/DurableTask.Core/TaskOrchestrationDispatcher.cs b/src/DurableTask.Core/TaskOrchestrationDispatcher.cs
index 51a64b4ef..764d885e9 100644
--- a/src/DurableTask.Core/TaskOrchestrationDispatcher.cs
+++ b/src/DurableTask.Core/TaskOrchestrationDispatcher.cs
@@ -1217,6 +1217,10 @@ TaskMessage ProcessSendEventDecision(
{
Name = sendEventAction.EventName
};
+ if (Activity.Current != null)
+ {
+ eventRaisedEvent.SetParentTraceContext(Activity.Current.Context);
+ }
// Distributed Tracing: start a new trace activity derived from the orchestration
// for an EventRaisedEvent (external event)
diff --git a/src/DurableTask.Core/Tracing/Schema.cs b/src/DurableTask.Core/Tracing/Schema.cs
index 8a91e3fb1..647726221 100644
--- a/src/DurableTask.Core/Tracing/Schema.cs
+++ b/src/DurableTask.Core/Tracing/Schema.cs
@@ -26,6 +26,9 @@ internal static class Task
internal const string TaskId = "durabletask.task.task_id";
internal const string EventTargetInstanceId = "durabletask.event.target_instance_id";
internal const string FireAt = "durabletask.fire_at";
+ internal const string Operation = "durabletask.task.operation";
+ internal const string ScheduledTime = "durabletask.task.scheduled_time";
+ internal const string ErrorMessage = "durabletask.entity.error_message";
}
internal static class Status
diff --git a/src/DurableTask.Core/Tracing/TraceActivityConstants.cs b/src/DurableTask.Core/Tracing/TraceActivityConstants.cs
index 927d1bc93..ef56f52d8 100644
--- a/src/DurableTask.Core/Tracing/TraceActivityConstants.cs
+++ b/src/DurableTask.Core/Tracing/TraceActivityConstants.cs
@@ -20,8 +20,12 @@ internal class TraceActivityConstants
public const string Activity = "activity";
public const string Event = "event";
public const string Timer = "timer";
+ public const string Entity = "entity";
public const string CreateOrchestration = "create_orchestration";
public const string OrchestrationEvent = "orchestration_event";
+
+ public const string CallEntity = "call_entity";
+ public const string SignalEntity = "signal_entity";
}
}
diff --git a/src/DurableTask.Core/Tracing/TraceHelper.cs b/src/DurableTask.Core/Tracing/TraceHelper.cs
index 8df8c0fa4..5f80784cd 100644
--- a/src/DurableTask.Core/Tracing/TraceHelper.cs
+++ b/src/DurableTask.Core/Tracing/TraceHelper.cs
@@ -20,6 +20,11 @@ namespace DurableTask.Core.Tracing
using System.Runtime.ExceptionServices;
using DurableTask.Core.Common;
using DurableTask.Core.History;
+ using Newtonsoft.Json;
+ using DurableTask.Core.Entities.EventFormat;
+ using DurableTask.Core.Entities;
+ using DurableTask.Core.Entities.OperationFormat;
+ using System.Linq;
///
/// Helper class for logging/tracing
@@ -39,9 +44,24 @@ public class TraceHelper
///
internal static Activity? StartActivityForNewOrchestration(ExecutionStartedEvent startEvent)
{
+ if (!startEvent.TryGetParentTraceContext(out ActivityContext parentTraceContext) && startEvent.ParentTraceContext != null)
+ {
+ return null;
+ }
+
+ DateTimeOffset? startTime = null;
+ // In the case that a request time for the start orchestration request is provided via ExecutionStartedEvent.Tags, we will use this as the start time of the Activity rather than the current time
+ if (startEvent.Tags != null && startEvent.Tags.ContainsKey(OrchestrationTags.RequestTime) &&
+ DateTimeOffset.TryParse(startEvent.Tags[OrchestrationTags.RequestTime], CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal, out DateTimeOffset requestTime))
+ {
+ startTime = requestTime;
+ }
+
Activity? newActivity = ActivityTraceSource.StartActivity(
- name: CreateSpanName(TraceActivityConstants.CreateOrchestration, startEvent.Name, startEvent.Version),
- kind: ActivityKind.Producer);
+ CreateSpanName(TraceActivityConstants.CreateOrchestration, startEvent.Name, startEvent.Version),
+ kind: ActivityKind.Producer,
+ parentContext: parentTraceContext,
+ startTime: startTime ?? default);
if (newActivity != null)
{
@@ -75,6 +95,14 @@ public class TraceHelper
return null;
}
+ if (startEvent.Tags != null && startEvent.Tags.ContainsKey(OrchestrationTags.CreateTraceForNewOrchestration))
+ {
+ startEvent.Tags.Remove(OrchestrationTags.CreateTraceForNewOrchestration);
+ // Note that if we create the trace activity for starting a new orchestration here, then its duration will be longer since its end time will be set to once we
+ // start processing the orchestration rather than when the request for a new orchestration is committed to storage.
+ using var activityForNewOrchestration = StartActivityForNewOrchestration(startEvent);
+ }
+
if (!startEvent.TryGetParentTraceContext(out ActivityContext activityContext))
{
return null;
@@ -446,6 +474,168 @@ internal static void EmitTraceActivityForTimer(
}
}
+ ///
+ /// Starts a new trace activity for calling or signaling an entity.
+ ///
+ /// The instance ID of the entity being called or signaled
+ /// The entity name
+ /// The operation name
+ /// Whether or not this is a signal request (as opposed to a call request)
+ /// The scheduled time of the request
+ /// The trace context of the parent that is calling or signaling the entity
+ /// The start time of the Activity, which is the time the request to the entity was generated
+ /// In the case that this is an entity signaling another entity, the instance ID of the signaling entity
+ ///
+ /// Returns a newly started with entity-specific metadata.
+ ///
+ internal static Activity? StartActivityForCallingOrSignalingEntity(string targetEntityId, string entityName, string operationName, bool signalEntity, DateTime? scheduledTime, ActivityContext parentTraceContext, DateTimeOffset? startTime, string? entityId = null)
+ {
+ Activity? newActivity = ActivityTraceSource.StartActivity(
+ CreateEntitySpanName(entityName, operationName),
+ kind: signalEntity ? ActivityKind.Producer : ActivityKind.Client,
+ parentContext: parentTraceContext,
+ startTime: startTime ?? default);
+
+ if (newActivity == null)
+ {
+ return null;
+ }
+
+ newActivity.SetTag(Schema.Task.Type, TraceActivityConstants.Entity);
+ newActivity.SetTag(Schema.Task.Operation, signalEntity ? TraceActivityConstants.SignalEntity : TraceActivityConstants.CallEntity);
+ newActivity.SetTag(Schema.Task.EventTargetInstanceId, targetEntityId);
+
+ if (!string.IsNullOrEmpty(entityId))
+ {
+ newActivity.SetTag(Schema.Task.InstanceId, entityId);
+ }
+
+ if (scheduledTime != null)
+ {
+ newActivity.SetTag(Schema.Task.ScheduledTime, scheduledTime.Value.ToString());
+ }
+
+ return newActivity;
+ }
+
+ ///
+ /// Starts a new trace Activity for an entity starting an orchestration.
+ ///
+ /// The instance ID of the entity starting the orchestration
+ /// The entity name
+ /// The instance ID of the orchestration being started
+ /// The trace context of the parent entity invocation that led to this start orchestration request
+ /// The start time of the Activity, which is the time the start orchestration request was generated
+ /// The scheduled time of the request
+ ///
+ /// Returns a newly started with entity and orchestration-specific metadata.
+ ///
+ internal static Activity? StartActivityForEntityStartingAnOrchestration(string entityId, string entityName, string targetInstanceId, ActivityContext parentTraceContext, DateTimeOffset? startTime, DateTime? scheduledTime = null)
+ {
+ Activity? newActivity = ActivityTraceSource.StartActivity(
+ CreateSpanName(entityName, TraceActivityConstants.CreateOrchestration, null),
+ kind: ActivityKind.Producer,
+ parentContext: parentTraceContext,
+ startTime: startTime ?? default);
+
+ if (newActivity == null)
+ {
+ return null;
+ }
+
+ newActivity.SetTag(Schema.Task.Type, TraceActivityConstants.Entity);
+ newActivity.SetTag(Schema.Task.EventTargetInstanceId, targetInstanceId);
+ newActivity.SetTag(Schema.Task.InstanceId, entityId);
+
+ if (scheduledTime != null)
+ {
+ newActivity.SetTag(Schema.Task.ScheduledTime, scheduledTime.Value.ToString());
+ }
+
+ return newActivity;
+ }
+
+ ///
+ /// Starts a new trace Activity for an entity processing a signal/call request.
+ ///
+ /// The instance ID of the entity being callled or signaled
+ /// The entity name
+ /// The name of the operation the entity is processing
+ /// Whether or not this is a signal request (as opposed to a call request)
+ /// The trace context of the parent signal/call request which led to this invocation
+ ///
+ /// Returns a newly started with entity-specific metadata.
+ ///
+ internal static Activity? StartActivityForProcessingEntityInvocation(string entityId, string entityName, string operationName, bool signalEntity, ActivityContext? parentTraceContext)
+ {
+ Activity? newActivity = ActivityTraceSource.StartActivity(
+ CreateEntitySpanName(entityName, operationName),
+ kind: signalEntity ? ActivityKind.Consumer : ActivityKind.Server,
+ parentContext: parentTraceContext ?? default);
+
+ if (newActivity == null)
+ {
+ return null;
+ }
+
+ newActivity.SetTag(Schema.Task.Type, TraceActivityConstants.Entity);
+ newActivity.SetTag(Schema.Task.Operation, signalEntity ? TraceActivityConstants.SignalEntity : TraceActivityConstants.CallEntity);
+ newActivity.SetTag(Schema.Task.InstanceId, entityId);
+
+ return newActivity;
+ }
+
+ ///
+ /// Ends the activities for an entity processing a batch of signal/call requests.
+ ///
+ /// The batch of trace activities to end.
+ /// The results returned by the entity for the batch of that it processed.
+ /// The , if any were provided. This will be used to set the error message of all the Activities in the case that
+ /// the entity did not return results for all of the requests
+ internal static void EndActivitiesForProcessingEntityInvocation(List traceActivities, List results, FailureDetails? batchFailureDetails)
+ {
+ if (results.Count == traceActivities.Count)
+ {
+ foreach (var (activity, result) in traceActivities.Zip(results, (activity, result) => (activity, result)))
+ {
+ if (activity != null)
+ {
+ if (result.ErrorMessage != null || result.FailureDetails != null)
+ {
+ activity.SetTag(Schema.Task.ErrorMessage, result.ErrorMessage ?? result.FailureDetails!.ErrorMessage);
+ }
+ if (result.StartTimeUtc is DateTime startTime)
+ {
+ activity.SetStartTime(startTime);
+ }
+ if (result.EndTimeUtc is DateTime endTime)
+ {
+ activity.SetEndTime(endTime);
+ }
+ activity.Dispose();
+ }
+ }
+ }
+ // This can happen if some of the operations failed and have no corresponding OperationResult
+ // There is no way to map the successful operation results to the corresponding operation requests or trace activities, so we will just "fail" the trace activities in this case and dispose them
+ else
+ {
+ string errorMessage = "Unable to generate a trace activity for the entity invocation even though it may have succeeded.";
+ if (batchFailureDetails is FailureDetails failureDetails)
+ {
+ errorMessage += $" If it failed, it may be due to {failureDetails.ErrorMessage}";
+ }
+ foreach (var activity in traceActivities)
+ {
+ if (activity != null)
+ {
+ activity.SetTag(Schema.Task.ErrorMessage, errorMessage);
+ activity.Dispose();
+ }
+ }
+ }
+ }
+
internal static void SetRuntimeStatusTag(string runtimeStatus)
{
DistributedTraceActivity.Current?.SetTag(Schema.Task.Status, runtimeStatus);
@@ -473,6 +663,11 @@ static string CreateTimerSpanName(string orchestrationName)
return $"{TraceActivityConstants.Orchestration}:{orchestrationName}:{TraceActivityConstants.Timer}";
}
+ static string CreateEntitySpanName(string entityName, string operationName)
+ {
+ return $"{TraceActivityConstants.Entity}:{entityName}:{operationName}";
+ }
+
///
/// Simple trace with no iid or eid
///