[Flex] Adding support for OpenTelemetry (#10094)

* OpenTelemetry support (#9985)

Adding support for OpenTelemetry

---------
Co-authored-by: Jacob Viau <[email protected]>
Co-authored-by: Shyju Krishnankutty <[email protected]>
Co-authored-by: Daniel Castro <[email protected]>

* Updating Otel nuget packages. (#10014)

* Updating Otel nuget packages.

* Update Azure.Identity to 1.11.2 (#10037)

* Update Microsoft.Identity.Client to 4.60.3

* Update Azure.Identity to 1.11.2

* Remove Microsoft.Identity.Client

* Update WebJobs package in test projects (#10061)

* Update PowerShell workers to latest (#10016)

* Update Python Worker Version to 4.28.0 (#10009)

Co-authored-by: AzureFunctionsPython <[email protected]>

* Update Node.js Worker to 3.10.0 (#9999)

* Update Node.js Worker to 3.10.0

* Add PR reference

* Update Python Worker Version to 4.28.1 (#10028)

Co-authored-by: AzureFunctionsPython <[email protected]>

---------

Co-authored-by: Jacob Viau <[email protected]>
Co-authored-by: andystaples <[email protected]>
Co-authored-by: gavin-aguiar <[email protected]>
Co-authored-by: AzureFunctionsPython <[email protected]>
Co-authored-by: Daniel Castro <[email protected]>

Author: 6 people

build/python.props

@@ -1,5 +1,5 @@
 <Project>
   <ItemGroup>
-    <PackageReference Include="Microsoft.Azure.Functions.PythonWorker" Version="4.27.0" />
+    <PackageReference Include="Microsoft.Azure.Functions.PythonWorker" Version="4.28.1" />
   </ItemGroup>
 </Project>

release_notes.md

@@ -3,6 +3,11 @@
 <!-- Please add your release notes in the following format:
 - My change description (#PR)
 -->
+- Update Python Worker Version to [4.28.1](https://github.com/Azure/azure-functions-python-worker/releases/tag/4.28.1)
 - DotnetIsolated worker artifact clean up (#9976)
   - Move symbols from dotnet-isolated worker to symbols package
-  - Removed linux executables from dotnet-isolated worker.
+  - Removed linux executables from dotnet-isolated worker.
+- Update Azure.Identity to 1.11.0 (#10002)
+- Update PowerShell worker 7.2 to [4.0.3220](https://github.com/Azure/azure-functions-powershell-worker/releases/tag/v4.0.3220)
+- Update PowerShell worker 7.4 to [4.0.3219](https://github.com/Azure/azure-functions-powershell-worker/releases/tag/v4.0.3219)
+- Update Node.js Worker Version to [3.10.0](https://github.com/Azure/azure-functions-nodejs-worker/releases/tag/v3.10.0) (#9999)

src/WebJobs.Script.Grpc/Channel/GrpcWorkerChannel.cs

@@ -20,6 +20,7 @@
 using Microsoft.Azure.WebJobs.Script.Config;
 using Microsoft.Azure.WebJobs.Script.Description;
 using Microsoft.Azure.WebJobs.Script.Diagnostics;
+using Microsoft.Azure.WebJobs.Script.Diagnostics.OpenTelemetry;
 using Microsoft.Azure.WebJobs.Script.Eventing;
 using Microsoft.Azure.WebJobs.Script.Extensions;
 using Microsoft.Azure.WebJobs.Script.Grpc.Eventing;
@@ -565,6 +566,12 @@ internal void ApplyCapabilities(IDictionary<string, string> capabilities, GrpcCa
                 _isWorkerApplicationInsightsLoggingEnabled = true;
             }
 
+            if (bool.TryParse(_workerCapabilities.GetCapabilityState(RpcWorkerConstants.WorkerOpenTelemetryEnabled), out bool otelEnabled) &&
+                otelEnabled)
+            {
+                ScriptHost.WorkerOpenTelemetryEnabled = true;
+            }
+
             // If http proxying is enabled, we need to get the proxying endpoint of this worker
             var httpUri = _workerCapabilities.GetCapabilityState(RpcWorkerConstants.HttpUri);
             if (!string.IsNullOrEmpty(httpUri))
@@ -1670,29 +1677,47 @@ private void AddSample<T>(List<T> samples, T sample)
 
         private void AddAdditionalTraceContext(MapField<string, string> attributes, ScriptInvocationContext context)
         {
-            // This is only applicable for AI agents running along side worker
-            if (_environment.IsApplicationInsightsAgentEnabled())
+            bool isOtelEnabled = _scriptHostOptions?.Value.TelemetryMode == TelemetryMode.OpenTelemetry;
+            bool isAIEnabled = _environment.IsApplicationInsightsAgentEnabled();
+
+            if (isOtelEnabled || isAIEnabled)
             {
-                attributes[ScriptConstants.LogPropertyProcessIdKey] = Convert.ToString(_rpcWorkerProcess.Id);
                 if (context.FunctionMetadata.Properties.TryGetValue(ScriptConstants.LogPropertyHostInstanceIdKey, out var hostInstanceIdValue))
                 {
-                    attributes[ScriptConstants.LogPropertyHostInstanceIdKey] = Convert.ToString(hostInstanceIdValue);
-                }
-                if (context.FunctionMetadata.Properties.TryGetValue(LogConstants.CategoryNameKey, out var categoryNameValue))
-                {
-                    attributes[LogConstants.CategoryNameKey] = Convert.ToString(categoryNameValue);
+                    string id = Convert.ToString(hostInstanceIdValue);
+
+                    if (isOtelEnabled)
+                    {
+                        Activity.Current?.AddTag(ResourceSemanticConventions.FaaSInstance, id);
+                    }
+                    if (isAIEnabled)
+                    {
+                        attributes[ScriptConstants.LogPropertyHostInstanceIdKey] = id;
+                    }
                 }
-                string sessionid = Activity.Current?.GetBaggageItem(ScriptConstants.LiveLogsSessionAIKey);
-                if (!string.IsNullOrEmpty(sessionid))
+            }
+
+            if (isAIEnabled)
+            {
+                attributes[ScriptConstants.LogPropertyProcessIdKey] = Convert.ToString(_rpcWorkerProcess.Id);
+                attributes[ScriptConstants.OperationNameKey] = context.FunctionMetadata.Name;
+                string sessionId = Activity.Current?.GetBaggageItem(ScriptConstants.LiveLogsSessionAIKey);
+                if (!string.IsNullOrEmpty(sessionId))
                 {
-                    attributes[ScriptConstants.LiveLogsSessionAIKey] = sessionid;
+                    attributes[ScriptConstants.LiveLogsSessionAIKey] = sessionId;
                 }
-                string operationName = context.FunctionMetadata.Name;
-                if (!string.IsNullOrEmpty(operationName))
+
+                if (context.FunctionMetadata.Properties.TryGetValue(LogConstants.CategoryNameKey, out var categoryNameValue))
                 {
-                    attributes[ScriptConstants.OperationNameKey] = operationName;
+                    attributes[LogConstants.CategoryNameKey] = Convert.ToString(categoryNameValue);
                 }
             }
+
+            if (isOtelEnabled)
+            {
+                Activity.Current?.AddTag(ResourceSemanticConventions.FaaSName, context.FunctionMetadata.Name);
+                Activity.Current?.AddTag(ResourceSemanticConventions.FaaSTrigger, OpenTelemetryConstants.ResolveTriggerType(context.FunctionMetadata?.Trigger?.Type));
+            }
         }
 
         private sealed class ExecutingInvocation : IDisposable

src/WebJobs.Script.WebHost/Features/FunctionExecutionFeature.cs

@@ -3,11 +3,13 @@
 
 using System;
 using System.Collections.Generic;
+using System.Diagnostics;
 using System.Threading;
 using System.Threading.Tasks;
 using Microsoft.AspNetCore.Http;
 using Microsoft.Azure.WebJobs.Script.Description;
 using Microsoft.Azure.WebJobs.Script.Diagnostics;
+using Microsoft.Azure.WebJobs.Script.Diagnostics.OpenTelemetry;
 using Microsoft.Azure.WebJobs.Script.Extensions;
 using Microsoft.Azure.WebJobs.Script.Host;
 using Microsoft.Extensions.Logging;
@@ -58,6 +60,9 @@ public async Task ExecuteAsync(HttpRequest request, CancellationToken cancellati
                 {
                     coldStartData.Add("dispatchDuration", dispatchStopwatch.GetElapsedTime().TotalMilliseconds);
                 }
+
+                // Add tag for cold start
+                Activity.Current?.AddTag(ResourceSemanticConventions.FaaSColdStart, true);
             }
 
             var sw = ValueStopwatch.StartNew();

src/WebJobs.Script.WebHost/Metrics/HostMetricsProvider.cs

@@ -8,6 +8,7 @@
 using System.Linq;
 using System.Threading;
 using Microsoft.Azure.WebJobs.Host.Executors;
+using Microsoft.Azure.WebJobs.Script.Diagnostics.OpenTelemetry;
 using Microsoft.Azure.WebJobs.Script.Metrics;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Hosting;
@@ -60,7 +61,7 @@ public void Start()
                 {
                     listener.EnableMeasurementEvents(instrument);
 
-                    var instanceIdTag = instrument.Meter.Tags.FirstOrDefault(t => t.Key == TelemetryAttributes.ServiceInstanceId);
+                    var instanceIdTag = instrument.Meter.Tags.FirstOrDefault(t => t.Key == ResourceSemanticConventions.ServiceInstanceId);
                     InstanceId = instanceIdTag.Value?.ToString() ?? string.Empty;
                 }
             };

src/WebJobs.Script.WebHost/WebJobs.Script.WebHost.csproj

@@ -62,7 +62,7 @@
   </ItemGroup>
 
   <ItemGroup>
-    <PackageReference Include="Azure.Identity" Version="1.10.2" />
+    <PackageReference Include="Azure.Identity" Version="1.11.2" />
     <PackageReference Include="Azure.Security.KeyVault.Secrets" Version="4.2.0" />
     <PackageReference Include="Microsoft.ApplicationInsights" Version="2.22.0" />
     <PackageReference Include="Microsoft.ApplicationInsights.AspNetCore" Version="2.22.0" />

src/WebJobs.Script.WebHost/WebJobsScriptHostService.cs

@@ -28,6 +28,9 @@
 using Microsoft.Extensions.Hosting;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
+using OpenTelemetry.Logs;
+using OpenTelemetry.Metrics;
+using OpenTelemetry.Trace;
 using static Microsoft.Azure.WebJobs.Script.EnvironmentSettingNames;
 using IApplicationLifetime = Microsoft.AspNetCore.Hosting.IApplicationLifetime;
 
@@ -872,6 +875,7 @@ private async Task Orphan(IHost instance, CancellationToken cancellationToken =
                 else
                 {
                     DisposeDependencyTrackingModule(instance);
+                    FlushOpenTelemetry(instance);
                     instance.Dispose();
                     _logger.LogDebug("ScriptHost disposed");
                 }
@@ -958,6 +962,28 @@ public object GetService(Type serviceType)
             return Services?.GetService(serviceType);
         }
 
+        private void FlushOpenTelemetry(IHost host)
+        {
+            var logger = GetHostLogger(host);
+            foreach (var meterProvider in host.Services.GetServices<MeterProvider>())
+            {
+                logger.LogDebug(@"Flushing {providerName} ...", meterProvider);
+                meterProvider.ForceFlush();
+            }
+
+            foreach (var tracerProvider in host.Services.GetServices<TracerProvider>())
+            {
+                logger.LogDebug(@"Flushing {providerName} ...", tracerProvider);
+                tracerProvider.ForceFlush();
+            }
+
+            foreach (var logProvider in host.Services.GetServices<ILoggerProvider>().Where(i => i is OpenTelemetryLoggerProvider))
+            {
+                logger.LogDebug(@"Disposing {providerName} ...", logProvider);
+                logProvider.Dispose();
+            }
+        }
+
         protected virtual void Dispose(bool disposing)
         {
             if (!_disposed)

src/WebJobs.Script/Config/ConfigurationSectionNames.cs

@@ -24,5 +24,6 @@ public static class ConfigurationSectionNames
         public const string Retry = "retry";
         public const string SequentialJobHostRestart = JobHost + ":sequentialRestart";
         public const string SendCanceledInvocationsToWorker = "sendCanceledInvocationsToWorker";
+        public const string TelemetryMode = "telemetryMode";
     }
 }

src/WebJobs.Script/Config/ScriptJobHostOptions.cs

@@ -6,6 +6,7 @@
 using System.Collections.Immutable;
 using System.Collections.ObjectModel;
 using Microsoft.Azure.WebJobs.Script.Description;
+using Microsoft.Azure.WebJobs.Script.Diagnostics.OpenTelemetry;
 
 namespace Microsoft.Azure.WebJobs.Script
 {
@@ -42,7 +43,7 @@ public string RootScriptPath
         public string InstanceId { get; }
 
         /// <summary>
-        /// Gets or sets NugetFallBackPath
+        /// Gets or sets NugetFallBackPath.
         /// </summary>
         public string NugetFallBackPath { get; set; }
 
@@ -129,5 +130,10 @@ public string RootScriptPath
         /// invocation to the worker.
         /// </summary>
         public bool SendCanceledInvocationsToWorker { get; set; } = true;
+
+        /// <summary>
+        /// Gets or sets the telemetry mode.
+        /// </summary>
+        internal TelemetryMode TelemetryMode { get; set; } = TelemetryMode.ApplicationInsights;
     }
 }

src/WebJobs.Script/Config/ScriptJobHostOptionsSetup.cs

@@ -2,6 +2,7 @@
 // Licensed under the MIT License. See License.txt in the project root for license information.
 
 using System;
+using Microsoft.Azure.WebJobs.Script.Diagnostics.OpenTelemetry;
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.Options;
 
@@ -73,6 +74,16 @@ public void Configure(ScriptJobHostOptions options)
             options.TestDataPath = webHostOptions.TestDataPath;
             options.IsFileSystemReadOnly = webHostOptions.IsFileSystemReadOnly;
             options.IsStandbyConfiguration = webHostOptions.IsStandbyConfiguration;
+
+            var telemetryModeSection = jobHostSection.GetSection(ConfigurationSectionNames.TelemetryMode);
+            if (telemetryModeSection.Exists() && Enum.TryParse(telemetryModeSection.Value, true, out TelemetryMode telemetryMode))
+            {
+                options.TelemetryMode = telemetryMode;
+            }
+            else
+            {
+               options.TelemetryMode = TelemetryMode.ApplicationInsights;
+            }
         }
 
         private void ConfigureFunctionTimeout(ScriptJobHostOptions options)