Adding support for faas.invoke_duration metric (#10929)

RohitRanjanMS · web-flow · commit d1e06f1d9816 · 2025-03-17T12:02:21.000-07:00
* Adding support for faas.invoke_duration metric
diff --git a/release_notes.md b/release_notes.md
@@ -11,3 +11,4 @@
 - Adjusting the logic to determine the warmup call in placeholder simulation mode to align with the production flow (#10918)
 - Fixing invalid DateTimes in status blobs when invoking via portal (#10916)
 - Bug fix for platform release channel bundles resolution casing issue and additional logging (#10921)
+- Adding support for faas.invoke_duration metric and other spec related updates (#10929)
diff --git a/src/WebJobs.Script.Grpc/Channel/GrpcWorkerChannel.cs b/src/WebJobs.Script.Grpc/Channel/GrpcWorkerChannel.cs
@@ -1718,7 +1718,6 @@ private void AddAdditionalTraceContext(MapField<string, string> attributes, Scri
             if (isOtelEnabled)
             {
                 Activity.Current?.AddTag(ResourceSemanticConventions.FaaSName, context.FunctionMetadata.Name);
-                Activity.Current?.AddTag(ResourceSemanticConventions.FaaSTrigger, OpenTelemetryConstants.ResolveTriggerType(context.FunctionMetadata?.Trigger?.Type));
             }
         }
 
diff --git a/src/WebJobs.Script.WebHost/Diagnostics/FunctionInstanceLogger.cs b/src/WebJobs.Script.WebHost/Diagnostics/FunctionInstanceLogger.cs
@@ -131,6 +131,7 @@ private void EndFunction(FunctionInstanceLogEntry item)
             }
 
             _metrics.EndEvent(invokeLatencyEvent);
+            _hostMetrics.TrackFunctionExecutionDuration(item.Duration.TotalSeconds, functionName);
         }
 
         public Task FlushAsync(CancellationToken cancellationToken = default(CancellationToken)) => Task.CompletedTask;
diff --git a/src/WebJobs.Script/Diagnostics/OpenTelemetry/FunctionsResourceDetector.cs b/src/WebJobs.Script/Diagnostics/OpenTelemetry/FunctionsResourceDetector.cs
@@ -4,7 +4,6 @@
 using System;
 using System.Collections.Generic;
 using System.Diagnostics;
-using System.Reflection;
 using OpenTelemetry.Resources;
 
 namespace Microsoft.Azure.WebJobs.Script.Diagnostics.OpenTelemetry
@@ -22,7 +21,6 @@ public Resource Detect()
                 attributeList.Add(new KeyValuePair<string, object>(ResourceSemanticConventions.ServiceVersion, version));
                 attributeList.Add(new KeyValuePair<string, object>(ResourceSemanticConventions.AISDKPrefix, $@"{OpenTelemetryConstants.SDKPrefix}:{version}"));
                 attributeList.Add(new KeyValuePair<string, object>(ResourceSemanticConventions.ProcessId, Process.GetCurrentProcess().Id));
-                attributeList.Add(new KeyValuePair<string, object>(ResourceSemanticConventions.FaaSVersion, version));
 
                 // Add these attributes only if running in Azure.
                 if (!string.IsNullOrEmpty(serviceName))
diff --git a/src/WebJobs.Script/Diagnostics/OpenTelemetry/OpenTelemetryConfigurationExtensions.cs b/src/WebJobs.Script/Diagnostics/OpenTelemetry/OpenTelemetryConfigurationExtensions.cs
@@ -2,15 +2,20 @@
 // Licensed under the MIT License. See License.txt in the project root for license information.
 
 using System;
+using System.Diagnostics;
 using System.Diagnostics.Tracing;
+using System.Linq;
 using Azure.Core;
 using Azure.Identity;
 using Azure.Monitor.OpenTelemetry.Exporter;
 using Azure.Monitor.OpenTelemetry.LiveMetrics;
+using Microsoft.AspNetCore.Routing;
+using Microsoft.Azure.WebJobs.Script.Metrics;
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Hosting;
 using Microsoft.Extensions.Logging;
+using OpenTelemetry;
 using OpenTelemetry.Logs;
 using OpenTelemetry.Metrics;
 using OpenTelemetry.Resources;
@@ -47,52 +52,94 @@ internal static void ConfigureOpenTelemetry(this ILoggingBuilder loggingBuilder,
             }
 
             loggingBuilder
-                .AddOpenTelemetry(o =>
-                {
-                    o.SetResourceBuilder(ConfigureResource(ResourceBuilder.CreateDefault()));
-                    if (enableOtlp)
-                    {
-                        o.AddOtlpExporter();
-                    }
-                    if (enableAzureMonitor)
-                    {
-                        o.AddAzureMonitorLogExporter(options => ConfigureAzureMonitorOptions(options, azMonConnectionString, credential));
-                    }
-                    o.IncludeFormattedMessage = true;
-                    o.IncludeScopes = false;
-                })
-                .AddDefaultOpenTelemetryFilters();
+                .ConfigureLogging(enableOtlp, enableAzureMonitor, azMonConnectionString, credential).Services
+                .AddOpenTelemetry()
+                .ConfigureResource(r => ConfigureResource(r))
+                .ConfigureMetrics(enableOtlp, enableAzureMonitor, azMonConnectionString, credential)
+                .ConfigureTracing(enableOtlp, enableAzureMonitor, azMonConnectionString, credential)
+                .ConfigureEventLogLevel(context.Configuration);
 
             // Azure SDK instrumentation is experimental.
             AppContext.SetSwitch("Azure.Experimental.EnableActivitySource", true);
+        }
 
-            ConfigureTracing(loggingBuilder, enableOtlp, enableAzureMonitor, azMonConnectionString, credential);
+        private static IOpenTelemetryBuilder ConfigureMetrics(this IOpenTelemetryBuilder builder, bool enableOtlp, bool enableAzureMonitor, string azMonConnectionString, TokenCredential credential)
+        {
+            return builder.WithMetrics(builder =>
+            {
+                builder.AddAspNetCoreInstrumentation();
+                builder.AddMeter(HostMetrics.FaasMeterName);
+                builder.AddView(HostMetrics.FaasInvokeDuration, new ExplicitBucketHistogramConfiguration
+                {
+                    Boundaries = new double[] { 0.005, 0.01, 0.025, 0.05, 0.075, 0.1, 0.25, 0.5, 0.75, 1, 2.5, 5, 7.5, 10 }
+                });
 
-            ConfigureEventLogLevel(loggingBuilder, context.Configuration);
+                if (enableOtlp)
+                {
+                    builder.AddOtlpExporter();
+                }
+                if (enableAzureMonitor)
+                {
+                    builder.AddAzureMonitorMetricExporter(opt => ConfigureAzureMonitorOptions(opt, azMonConnectionString, credential));
+                }
+            });
         }
 
-        private static void ConfigureTracing(ILoggingBuilder loggingBuilder, bool enableOtlp, bool enableAzureMonitor, string azMonConnectionString, TokenCredential credential)
+        private static IOpenTelemetryBuilder ConfigureTracing(this IOpenTelemetryBuilder builder, bool enableOtlp, bool enableAzureMonitor, string azMonConnectionString, TokenCredential credential)
         {
-            loggingBuilder.Services.AddOpenTelemetry()
-                .ConfigureResource(r => ConfigureResource(r))
-                .WithTracing(builder =>
+            return builder.WithTracing(builder =>
+            {
+                builder.AddSource("Azure.*")
+                .AddAspNetCoreInstrumentation(o =>
                 {
-                    builder.AddSource("Azure.*")
-                           .AddAspNetCoreInstrumentation();
-
-                    if (enableOtlp)
+                    o.EnrichWithHttpResponse = (activity, httpResponse) =>
                     {
-                        builder.AddOtlpExporter();
-                    }
+                        if (Activity.Current != null)
+                        {
+                            var routingFeature = httpResponse.HttpContext.Features.Get<AspNetCore.Routing.IRoutingFeature>();
+                            var template = routingFeature.RouteData.Routers.FirstOrDefault(r => r is Route) as Route;
+
+                            Activity.Current.DisplayName = $"{Activity.Current.DisplayName} {template?.RouteTemplate}";
+                            Activity.Current.AddTag(ResourceSemanticConventions.HttpRoute, template?.RouteTemplate);
+                            Activity.Current.AddTag(ResourceSemanticConventions.FaaSTrigger, OpenTelemetryConstants.HttpTriggerType);
+                        }
+                    };
+                });
 
-                    if (enableAzureMonitor)
-                    {
-                        builder.AddAzureMonitorTraceExporter(opt => ConfigureAzureMonitorOptions(opt, azMonConnectionString, credential));
-                        builder.AddLiveMetrics(opt => ConfigureAzureMonitorOptions(opt, azMonConnectionString, credential));
-                    }
+                if (enableOtlp)
+                {
+                    builder.AddOtlpExporter();
+                }
 
-                    builder.AddProcessor(ActivitySanitizingProcessor.Instance);
-                });
+                if (enableAzureMonitor)
+                {
+                    builder.AddAzureMonitorTraceExporter(opt => ConfigureAzureMonitorOptions(opt, azMonConnectionString, credential));
+                    builder.AddLiveMetrics(opt => ConfigureAzureMonitorOptions(opt, azMonConnectionString, credential));
+                }
+
+                builder.AddProcessor(ActivitySanitizingProcessor.Instance);
+            });
+        }
+
+        private static ILoggingBuilder ConfigureLogging(this ILoggingBuilder builder, bool enableOtlp, bool enableAzureMonitor, string azMonConnectionString, TokenCredential credential)
+        {
+            builder.AddOpenTelemetry(o =>
+            {
+                o.SetResourceBuilder(ConfigureResource(ResourceBuilder.CreateDefault()));
+                if (enableOtlp)
+                {
+                    o.AddOtlpExporter();
+                }
+                if (enableAzureMonitor)
+                {
+                    o.AddAzureMonitorLogExporter(options => ConfigureAzureMonitorOptions(options, azMonConnectionString, credential));
+                }
+                o.IncludeFormattedMessage = true;
+                o.IncludeScopes = false;
+            });
+            builder.AddDefaultOpenTelemetryFilters();
+
+            return builder;
         }
 
         private static ILoggingBuilder AddDefaultOpenTelemetryFilters(this ILoggingBuilder loggingBuilder)
@@ -101,23 +148,27 @@ private static ILoggingBuilder AddDefaultOpenTelemetryFilters(this ILoggingBuild
                 // These are messages piped back to the host from the worker - we don't handle these anymore if the worker has OpenTelemetry enabled.
                 // Instead, we expect the user's own code to be logging these where they want them to go.
                 .AddFilter<OpenTelemetryLoggerProvider>("Function.*", _ => !ScriptHost.WorkerOpenTelemetryEnabled)
-                .AddFilter<OpenTelemetryLoggerProvider>("Azure.*", _ => !ScriptHost.WorkerOpenTelemetryEnabled)
+
+                // Always filter out these logs
+                .AddFilter<OpenTelemetryLoggerProvider>("Azure.*", _ => false)
                 // Host.Results and Host.Aggregator are used to emit metrics, ignoring these categories.
-                .AddFilter<OpenTelemetryLoggerProvider>("Host.Results", _ => !ScriptHost.WorkerOpenTelemetryEnabled)
-                .AddFilter<OpenTelemetryLoggerProvider>("Host.Aggregator", _ => !ScriptHost.WorkerOpenTelemetryEnabled)
+                .AddFilter<OpenTelemetryLoggerProvider>("Host.Results", _ => false)
+                .AddFilter<OpenTelemetryLoggerProvider>("Host.Aggregator", _ => false)
                 // Ignoring all Microsoft.Azure.WebJobs.* logs like /getScriptTag and /lock.
-                .AddFilter<OpenTelemetryLoggerProvider>("Microsoft.Azure.WebJobs.*", _ => !ScriptHost.WorkerOpenTelemetryEnabled);
+                .AddFilter<OpenTelemetryLoggerProvider>("Microsoft.Azure.WebJobs.*", _ => false);
         }
 
-        private static void ConfigureEventLogLevel(ILoggingBuilder loggingBuilder, IConfiguration configuration)
+        private static IOpenTelemetryBuilder ConfigureEventLogLevel(this IOpenTelemetryBuilder builder, IConfiguration configuration)
         {
             string eventLogLevel = GetConfigurationValue(EnvironmentSettingNames.OpenTelemetryEventListenerLogLevel, configuration);
             EventLevel level = !string.IsNullOrEmpty(eventLogLevel) &&
                                Enum.TryParse(eventLogLevel, ignoreCase: true, out EventLevel parsedLevel)
                                ? parsedLevel
                                : EventLevel.Warning;
 
-            loggingBuilder.Services.AddHostedService(_ => new OpenTelemetryEventListenerService(level));
+            builder.Services.AddHostedService(_ => new OpenTelemetryEventListenerService(level));
+
+            return builder;
         }
 
         private static ResourceBuilder ConfigureResource(ResourceBuilder builder)
diff --git a/src/WebJobs.Script/Diagnostics/OpenTelemetry/OpenTelemetryConstants.cs b/src/WebJobs.Script/Diagnostics/OpenTelemetry/OpenTelemetryConstants.cs
@@ -13,16 +13,6 @@ internal static class OpenTelemetryConstants
         internal const string ResourceGroupEnvVar = "WEBSITE_RESOURCE_GROUP";
         internal const string OwnerNameEnvVar = "WEBSITE_OWNER_NAME";
         internal const string AzureFunctionsGroup = "azure.functions.group";
-
-        internal static string ResolveTriggerType(string trigger)
-        {
-            switch (trigger)
-            {
-                case "httpTrigger":
-                    return "http";
-                default:
-                    return trigger;
-            }
-        }
+        internal const string HttpTriggerType = "http";
     }
 }
diff --git a/src/WebJobs.Script/Diagnostics/OpenTelemetry/ResourceSemanticConventions.cs b/src/WebJobs.Script/Diagnostics/OpenTelemetry/ResourceSemanticConventions.cs
@@ -14,14 +14,13 @@ internal static class ResourceSemanticConventions
         internal const string CloudProvider = "cloud.provider";
         internal const string CloudPlatform = "cloud.platform";
         internal const string CloudRegion = "cloud.region";
-        internal const string CloudResourceId = "cloud.resource.id";
+        internal const string CloudResourceId = "cloud.resource_id";
 
         //FaaS
         internal const string FaaSTrigger = "faas.trigger";
         internal const string FaaSInvocationId = "faas.invocation_id";
         internal const string FaaSColdStart = "faas.coldstart";
         internal const string FaaSName = "faas.name";
-        internal const string FaaSVersion = "faas.version";
         internal const string FaaSInstance = "faas.instance";
 
         // Process
@@ -30,6 +29,7 @@ internal static class ResourceSemanticConventions
         // Http
         internal const string QueryUrl = "url.query";
         internal const string FullUrl = "url.full";
+        internal const string HttpRoute = "http.route";
 
         // AI
         internal const string AISDKPrefix = "ai.sdk.prefix";
diff --git a/src/WebJobs.Script/Metrics/HostMetrics.cs b/src/WebJobs.Script/Metrics/HostMetrics.cs
@@ -16,13 +16,18 @@ public class HostMetrics : IHostMetrics
         private readonly ILogger _logger;
 
         public const string MeterName = "Microsoft.Azure.WebJobs.Script.Host.Internal";
+        public const string FaasMeterName = "Microsoft.Azure.Functions.Host";
         public const string CloudPlatformName = "azure_functions";
         public const string AppFailureCount = "azure.functions.app_failures";
         public const string ActiveInvocationCount = "azure.functions.active_invocations";
         public const string StartedInvocationCount = "azure.functions.started_invocations";
 
+        // FaaS Metrics
+        public const string FaasInvokeDuration = "faas.invoke_duration";
+
         private Counter<long> _appFailureCount;
         private Counter<long> _startedInvocationCount;
+        private Histogram<double> _faasInvokeDuration;
 
         private KeyValuePair<string, object>? _cachedFunctionGroupTag = null;
 
@@ -59,6 +64,12 @@ public HostMetrics(IMeterFactory meterFactory, IEnvironment environment, ILogger
 
             _appFailureCount = meter.CreateCounter<long>(AppFailureCount, "numeric", "Number of times the host has failed to start.");
             _startedInvocationCount = meter.CreateCounter<long>(StartedInvocationCount, "numeric", "Number of function invocations that have started.");
+
+            var faasMeter = meterFactory.Create(new MeterOptions(FaasMeterName));
+            _faasInvokeDuration = faasMeter.CreateHistogram<double>(
+                name: FaasInvokeDuration,
+                unit: "s",
+                description: "Measures the duration of the function's logic execution.");
         }
 
         private KeyValuePair<string, object> FunctionGroupTag
@@ -95,5 +106,14 @@ private KeyValuePair<string, object> FunctionGroupTag
         public void AppFailure() => _appFailureCount.Add(1, FunctionGroupTag);
 
         public void IncrementStartedInvocationCount() => _startedInvocationCount.Add(1, FunctionGroupTag);
+
+        public void TrackFunctionExecutionDuration(double duration, string functionName)
+        {
+            var tags = new TagList()
+            {
+                { ResourceSemanticConventions.FaaSName, functionName }
+            };
+            _faasInvokeDuration.Record(duration, tags);
+        }
     }
 }
diff --git a/src/WebJobs.Script/Metrics/IHostMetrics.cs b/src/WebJobs.Script/Metrics/IHostMetrics.cs
@@ -19,5 +19,12 @@ public interface IHostMetrics
         /// that have started on a given instance.
         /// </summary>
         public void IncrementStartedInvocationCount();
+
+        /// <summary>
+        /// Measures the duration of the function�s logic execution.
+        /// </summary>
+        /// <param name="duration">Invoke duration.</param>
+        /// <param name="functionName">Function name.</param>
+        void TrackFunctionExecutionDuration(double duration, string functionName);
     }
 }
diff --git a/test/WebJobs.Script.Tests/Diagnostics/OpenTelemetry/OpenTelemetryConfigurationExtensionsTests.cs b/test/WebJobs.Script.Tests/Diagnostics/OpenTelemetry/OpenTelemetryConfigurationExtensionsTests.cs

Original file line number	Diff line number	Diff line change
`@@ -1718,7 +1718,6 @@ private void AddAdditionalTraceContext(MapField<string, string> attributes, Scri`
`1718`	`1718`	`if (isOtelEnabled)`
`1719`	`1719`	`{`
`1720`	`1720`	`Activity.Current?.AddTag(ResourceSemanticConventions.FaaSName, context.FunctionMetadata.Name);`
`1721`		`- Activity.Current?.AddTag(ResourceSemanticConventions.FaaSTrigger, OpenTelemetryConstants.ResolveTriggerType(context.FunctionMetadata?.Trigger?.Type));`
`1722`	`1721`	`}`
`1723`	`1722`	`}`
`1724`	`1723`
Original file line number	Diff line number	Diff line change
`@@ -131,6 +131,7 @@ private void EndFunction(FunctionInstanceLogEntry item)`
`131`	`131`	`}`
`132`	`132`
`133`	`133`	`_metrics.EndEvent(invokeLatencyEvent);`
	`134`	`+ _hostMetrics.TrackFunctionExecutionDuration(item.Duration.TotalSeconds, functionName);`
`134`	`135`	`}`
`135`	`136`
`136`	`137`	`public Task FlushAsync(CancellationToken cancellationToken = default(CancellationToken)) => Task.CompletedTask;`
Original file line number	Diff line number	Diff line change
`@@ -19,5 +19,12 @@ public interface IHostMetrics`
`19`	`19`	`/// that have started on a given instance.`
`20`	`20`	`/// </summary>`
`21`	`21`	`public void IncrementStartedInvocationCount();`
	`22`	`+`
	`23`	`+ /// <summary>`
	`24`	`+ /// Measures the duration of the function’s logic execution.`
	`25`	`+ /// </summary>`
	`26`	`+ /// <param name="duration">Invoke duration.</param>`
	`27`	`+ /// <param name="functionName">Function name.</param>`
	`28`	`+ void TrackFunctionExecutionDuration(double duration, string functionName);`
`22`	`29`	`}`
`23`	`30`	`}`