Skip to content

Commit 2ab6f31

Browse files
committed
Emit function invocation latency events (#1030)
1 parent f4a9393 commit 2ab6f31

File tree

18 files changed

+391
-62
lines changed

18 files changed

+391
-62
lines changed

src/WebJobs.Script.WebHost/Diagnostics/EventGenerator.cs

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -36,9 +36,9 @@ public void LogFunctionTraceEvent(TraceLevel level, string subscriptionId, strin
3636
}
3737
}
3838

39-
public void LogFunctionMetricEvent(string subscriptionId, string appName, string eventName, long average, long minimum, long maximum, long count, DateTime eventTimestamp)
39+
public void LogFunctionMetricEvent(string subscriptionId, string appName, string functionName, string eventName, long average, long minimum, long maximum, long count, DateTime eventTimestamp)
4040
{
41-
FunctionsSystemLogsEventSource.Instance.LogFunctionMetricEvent(subscriptionId, appName, eventName, average, minimum, maximum, count, ScriptHost.Version, eventTimestamp.ToString(EventTimestamp));
41+
FunctionsSystemLogsEventSource.Instance.LogFunctionMetricEvent(subscriptionId, appName, functionName, eventName, average, minimum, maximum, count, ScriptHost.Version, eventTimestamp.ToString(EventTimestamp));
4242
}
4343

4444
public void LogFunctionExecutionAggregateEvent(string siteName, string functionName, long executionTimeInMs, long functionStartedCount, long functionCompletedCount, long functionFailedCount)
@@ -132,11 +132,11 @@ public void RaiseFunctionsEventError(string SubscriptionId, string AppName, stri
132132
}
133133

134134
[Event(65524, Level = EventLevel.Informational, Channel = EventChannel.Operational, Version = 1)]
135-
public void LogFunctionMetricEvent(string SubscriptionId, string AppName, string EventName, long Average, long Minimum, long Maximum, long Count, string HostVersion, string EventTimestamp)
135+
public void LogFunctionMetricEvent(string SubscriptionId, string AppName, string FunctionName, string EventName, long Average, long Minimum, long Maximum, long Count, string HostVersion, string EventTimestamp)
136136
{
137137
if (IsEnabled())
138138
{
139-
WriteEvent(65524, SubscriptionId, AppName, EventName, Average, Minimum, Maximum, Count, HostVersion, EventTimestamp);
139+
WriteEvent(65524, SubscriptionId, AppName, FunctionName, EventName, Average, Minimum, Maximum, Count, HostVersion, EventTimestamp);
140140
}
141141
}
142142
}

src/WebJobs.Script.WebHost/Diagnostics/IEventGenerator.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ public interface IEventGenerator
1010
{
1111
void LogFunctionTraceEvent(TraceLevel level, string subscriptionId, string appName, string functionName, string eventName, string source, string details, string summary, Exception exception = null);
1212

13-
void LogFunctionMetricEvent(string subscriptionId, string appName, string eventName, long average, long minimum, long maximum, long count, DateTime eventTimestamp);
13+
void LogFunctionMetricEvent(string subscriptionId, string appName, string functionName, string eventName, long average, long minimum, long maximum, long count, DateTime eventTimestamp);
1414

1515
void LogFunctionExecutionAggregateEvent(string siteName, string functionName, long executionTimeInMs, long functionStartedCount, long functionCompletedCount, long functionFailedCount);
1616

src/WebJobs.Script.WebHost/Diagnostics/MetricsEventManager.cs

Lines changed: 28 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@ public MetricsEventManager(ScriptSettingsManager settingsManager, IEventGenerato
4747
/// </summary>
4848
public ConcurrentDictionary<string, SystemMetricEvent> QueuedEvents { get; }
4949

50-
public object BeginEvent(string eventName)
50+
public object BeginEvent(string eventName, string functionName = null)
5151
{
5252
if (string.IsNullOrEmpty(eventName))
5353
{
@@ -56,6 +56,7 @@ public object BeginEvent(string eventName)
5656

5757
return new SystemMetricEvent
5858
{
59+
FunctionName = functionName,
5960
EventName = eventName.ToLowerInvariant(),
6061
Timestamp = DateTime.UtcNow
6162
};
@@ -74,13 +75,19 @@ public void EndEvent(object eventHandle)
7475
evt.Duration = DateTime.UtcNow - evt.Timestamp;
7576
long latencyMS = (long)evt.Duration.TotalMilliseconds;
7677

77-
QueuedEvents.AddOrUpdate(evt.EventName,
78+
// event aggregation is based on this key
79+
// for each unique key, there will be only 1
80+
// queued event that we aggregate into
81+
string key = GetAggregateKey(evt.EventName, evt.FunctionName);
82+
83+
QueuedEvents.AddOrUpdate(key,
7884
(name) =>
7985
{
8086
// create the default event that will be added
81-
// if an event isn't already queued for this event name
87+
// if an event isn't already queued for this key
8288
return new SystemMetricEvent
8389
{
90+
FunctionName = evt.FunctionName,
8491
EventName = evt.EventName,
8592
Minimum = latencyMS,
8693
Maximum = latencyMS,
@@ -103,20 +110,22 @@ public void EndEvent(object eventHandle)
103110
}
104111
}
105112

106-
public void LogEvent(string eventName)
113+
public void LogEvent(string eventName, string functionName = null)
107114
{
108115
if (string.IsNullOrEmpty(eventName))
109116
{
110117
throw new ArgumentNullException(nameof(eventName));
111118
}
112119

113-
QueuedEvents.AddOrUpdate(eventName,
120+
string key = GetAggregateKey(eventName, functionName);
121+
QueuedEvents.AddOrUpdate(key,
114122
(name) =>
115123
{
116124
// create the default event that will be added
117-
// if an event isn't already queued for this event name
125+
// if an event isn't already queued for this key
118126
return new SystemMetricEvent
119127
{
128+
FunctionName = functionName,
120129
EventName = eventName.ToLowerInvariant(),
121130
Count = 1
122131
};
@@ -187,6 +196,18 @@ internal void HostStarted(ScriptHost scriptHost)
187196
}
188197
}
189198

199+
/// <summary>
200+
/// Constructs the aggregate key used to group events. When metric events are
201+
/// added for later aggregation on flush, they'll be grouped by this key.
202+
/// </summary>
203+
internal static string GetAggregateKey(string eventName, string functionName = null)
204+
{
205+
string key = string.IsNullOrEmpty(functionName) ?
206+
eventName : $"{eventName}_{functionName}";
207+
208+
return key.ToLowerInvariant();
209+
}
210+
190211
private static string SerializeBindings(IEnumerable<BindingMetadata> bindings)
191212
{
192213
if (bindings != null)
@@ -247,6 +268,7 @@ protected internal virtual void WriteMetricEvents(SystemMetricEvent[] metricEven
247268
_eventGenerator.LogFunctionMetricEvent(
248269
subscriptionId,
249270
appName,
271+
metricEvent.FunctionName ?? string.Empty,
250272
metricEvent.EventName.ToLowerInvariant(),
251273
metricEvent.Average,
252274
metricEvent.Minimum,

src/WebJobs.Script.WebHost/Diagnostics/SystemMetricEvent.cs

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,9 +6,23 @@
66

77
namespace Microsoft.Azure.WebJobs.Script.WebHost.Diagnostics
88
{
9-
[DebuggerDisplay("{EventName} {Count}")]
9+
[DebuggerDisplay("{DebugValue,nq}")]
1010
public class SystemMetricEvent : MetricEvent
1111
{
12+
private string DebugValue
13+
{
14+
get
15+
{
16+
string key = string.Empty;
17+
if (!string.IsNullOrEmpty(FunctionName))
18+
{
19+
key = $"Function: {FunctionName}, ";
20+
}
21+
key += $"Event: {EventName}, Count: {Count}";
22+
return $"({key})";
23+
}
24+
}
25+
1226
public string EventName { get; set; }
1327

1428
public long Average { get; set; }

src/WebJobs.Script.WebHost/Diagnostics/WebHostMetricsLogger.cs

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -27,9 +27,9 @@ public WebHostMetricsLogger(ScriptSettingsManager settingsManager, IEventGenerat
2727
_metricsEventManager = new MetricsEventManager(settingsManager, eventGenerator, metricEventIntervalInSeconds);
2828
}
2929

30-
public object BeginEvent(string eventName)
30+
public object BeginEvent(string eventName, string functionName = null)
3131
{
32-
return _metricsEventManager.BeginEvent(eventName);
32+
return _metricsEventManager.BeginEvent(eventName, functionName);
3333
}
3434

3535
public void BeginEvent(MetricEvent metricEvent)
@@ -70,9 +70,9 @@ public void LogEvent(MetricEvent metricEvent)
7070
}
7171
}
7272

73-
public void LogEvent(string eventName)
73+
public void LogEvent(string eventName, string functionName = null)
7474
{
75-
_metricsEventManager.LogEvent(eventName);
75+
_metricsEventManager.LogEvent(eventName, functionName);
7676
}
7777

7878
protected virtual void Dispose(bool disposing)

src/WebJobs.Script.WebHost/GlobalSuppressions.cs

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -85,4 +85,6 @@
8585
[assembly: System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Usage", "CA2213:DisposableFieldsShouldBeDisposed", MessageId = "_httpConfiguration", Scope = "member", Target = "Microsoft.Azure.WebJobs.Script.WebHost.WebHooks.WebHookReceiverManager.#Dispose(System.Boolean)")]
8686
[assembly: System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Usage", "CA2214:DoNotCallOverridableMethodsInConstructors", Scope = "member", Target = "Microsoft.Azure.WebJobs.Script.WebHost.SecretManager.#.ctor(Microsoft.Azure.WebJobs.Script.WebHost.ISecretsRepository,Microsoft.Azure.WebJobs.Script.WebHost.IKeyValueConverterFactory,Microsoft.Azure.WebJobs.Host.TraceWriter,System.Boolean)")]
8787
[assembly: System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Reliability", "CA2000:Dispose objects before losing scope", Scope = "member", Target = "Microsoft.Azure.WebJobs.Script.WebHost.DefaultSecretManagerFactory.#Create(Microsoft.Azure.WebJobs.Script.Config.ScriptSettingsManager,Microsoft.Azure.WebJobs.Host.TraceWriter,Microsoft.Azure.WebJobs.Script.WebHost.ISecretsRepository)")]
88-
[assembly: System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Reliability", "CA2000:Dispose objects before losing scope", Scope = "member", Target = "Microsoft.Azure.WebJobs.Script.WebHost.WebScriptHostManager.#.ctor(Microsoft.Azure.WebJobs.Script.ScriptHostConfiguration,Microsoft.Azure.WebJobs.Script.WebHost.ISecretManagerFactory,Microsoft.Azure.WebJobs.Script.Config.ScriptSettingsManager,Microsoft.Azure.WebJobs.Script.WebHost.WebHostSettings,Microsoft.Azure.WebJobs.Script.IScriptHostFactory)")]
88+
[assembly: System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Reliability", "CA2000:Dispose objects before losing scope", Scope = "member", Target = "Microsoft.Azure.WebJobs.Script.WebHost.WebScriptHostManager.#.ctor(Microsoft.Azure.WebJobs.Script.ScriptHostConfiguration,Microsoft.Azure.WebJobs.Script.WebHost.ISecretManagerFactory,Microsoft.Azure.WebJobs.Script.Config.ScriptSettingsManager,Microsoft.Azure.WebJobs.Script.WebHost.WebHostSettings,Microsoft.Azure.WebJobs.Script.IScriptHostFactory)")]
89+
[assembly: System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Performance", "CA1822:MarkMembersAsStatic", Scope = "member", Target = "Microsoft.Azure.WebJobs.Script.WebHost.Diagnostics.MetricsEventManager.#BeginEvent(System.String,System.String)")]
90+
[assembly: System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode", Scope = "member", Target = "Microsoft.Azure.WebJobs.Script.WebHost.Diagnostics.SystemMetricEvent.#DebugValue")]

src/WebJobs.Script/Description/FunctionInvokerBase.cs

Lines changed: 10 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -109,10 +109,9 @@ public async Task Invoke(object[] parameters)
109109

110110
string invocationId = functionExecutionContext.InvocationId.ToString();
111111

112-
FunctionStartedEvent startedEvent = new FunctionStartedEvent(functionExecutionContext.InvocationId, Metadata);
112+
var startedEvent = new FunctionStartedEvent(functionExecutionContext.InvocationId, Metadata);
113113
_metrics.BeginEvent(startedEvent);
114-
115-
LogInvocationMetrics(_metrics, Metadata.Bindings);
114+
var invokeLatencyEvent = LogInvocationMetrics(_metrics, Metadata);
116115

117116
try
118117
{
@@ -158,21 +157,25 @@ public async Task Invoke(object[] parameters)
158157
{
159158
_metrics.EndEvent(startedEvent);
160159
}
160+
if (invokeLatencyEvent != null)
161+
{
162+
_metrics.EndEvent(invokeLatencyEvent);
163+
}
161164
}
162165
}
163166

164-
internal static void LogInvocationMetrics(IMetricsLogger metrics, Collection<BindingMetadata> bindings)
167+
internal static object LogInvocationMetrics(IMetricsLogger metrics, FunctionMetadata metadata)
165168
{
166-
metrics.LogEvent(MetricEventNames.FunctionInvoke);
167-
168169
// log events for each of the binding types used
169-
foreach (var binding in bindings)
170+
foreach (var binding in metadata.Bindings)
170171
{
171172
string eventName = binding.IsTrigger ?
172173
string.Format(MetricEventNames.FunctionBindingTypeFormat, binding.Type) :
173174
string.Format(MetricEventNames.FunctionBindingTypeDirectionFormat, binding.Type, binding.Direction);
174175
metrics.LogEvent(eventName);
175176
}
177+
178+
return metrics.BeginEvent(MetricEventNames.FunctionInvokeLatency, metadata.Name);
176179
}
177180

178181
private void LogFunctionFailed(FunctionStartedEvent startedEvent, string resultString, string invocationId)

src/WebJobs.Script/Description/FunctionMetadata.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@ public FunctionMetadata()
4747
/// </summary>
4848
public bool IsExcluded { get; set; }
4949

50-
public Collection<BindingMetadata> Bindings { get; private set; }
50+
public Collection<BindingMetadata> Bindings { get; }
5151

5252
public IEnumerable<BindingMetadata> InputBindings
5353
{

src/WebJobs.Script/Diagnostics/IMetricsLogger.cs

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,8 +13,9 @@ public interface IMetricsLogger
1313
/// Begins an event.
1414
/// </summary>
1515
/// <param name="eventName">The event name.</param>
16+
/// <param name="functionName">Optional function name for function specific events.</param>
1617
/// <returns>A handle to the event that was started.</returns>
17-
object BeginEvent(string eventName);
18+
object BeginEvent(string eventName, string functionName = null);
1819

1920
/// <summary>
2021
/// Begins an event.
@@ -44,6 +45,7 @@ public interface IMetricsLogger
4445
/// Raises an event.
4546
/// </summary>
4647
/// <param name="eventName">The event name.</param>
47-
void LogEvent(string eventName);
48+
/// <param name="functionName">Optional function name for function specific events.</param>
49+
void LogEvent(string eventName, string functionName = null);
4850
}
4951
}

src/WebJobs.Script/Diagnostics/MetricEvent.cs

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,8 @@ namespace Microsoft.Azure.WebJobs.Script.Diagnostics
77
{
88
public abstract class MetricEvent
99
{
10+
public string FunctionName { get; set; }
11+
1012
public DateTime Timestamp { get; set; }
1113

1214
public TimeSpan Duration { get; set; }

0 commit comments

Comments
 (0)