Merge branch 'release/3.0' into v3.x

brettsam · brettsam · commit 1768ee541601 · 2020-02-25T06:32:02.000-08:00
diff --git a/src/WebJobs.Script.WebHost/Diagnostics/LinuxContainerEventGenerator.cs b/src/WebJobs.Script.WebHost/Diagnostics/LinuxContainerEventGenerator.cs
@@ -104,7 +104,7 @@ private void ConsoleWriter(string evt)
 
         public override void LogAzureMonitorDiagnosticLogEvent(LogLevel level, string resourceId, string operationName, string category, string regionName, string properties)
         {
-            _writeEvent($"{ScriptConstants.LinuxAzureMonitorEventStreamName} {(int)ToEventLevel(level)},{resourceId},{operationName},{category},{regionName},{NormalizeString(properties)},{_containerName},{TenantId},{DateTime.UtcNow.ToString()}");
+//            _writeEvent($"{ScriptConstants.LinuxAzureMonitorEventStreamName} {(int)ToEventLevel(level)},{resourceId},{operationName},{category},{regionName},{NormalizeString(properties)},{_containerName},{TenantId},{DateTime.UtcNow.ToString()}");
         }
     }
 }
diff --git a/src/WebJobs.Script/Description/DotNet/FunctionAssemblyLoadContext.cs b/src/WebJobs.Script/Description/DotNet/FunctionAssemblyLoadContext.cs
@@ -155,8 +155,8 @@ private static bool IsMinorMatchOrLowerPolicyEvaluator(AssemblyName requestedAss
         {
             AssemblyName runtimeAssemblyName = AssemblyNameCache.GetName(runtimeAssembly);
 
-            return requestedAssembly.Version.Major == runtimeAssemblyName.Version.Major &&
-                requestedAssembly.Version.Minor <= runtimeAssemblyName.Version.Minor;
+            return requestedAssembly.Version == null || (requestedAssembly.Version.Major == runtimeAssemblyName.Version.Major &&
+                requestedAssembly.Version.Minor <= runtimeAssemblyName.Version.Minor);
         }
 
         private bool IsRuntimeAssembly(AssemblyName assemblyName)
diff --git a/src/WebJobs.Script/Description/Workers/WorkerFunctionInvoker.cs b/src/WebJobs.Script/Description/Workers/WorkerFunctionInvoker.cs
@@ -90,7 +90,11 @@ protected override async Task<object> InvokeCore(object[] parameters, FunctionIn
 
         private async Task DelayUntilFunctionDispatcherInitializedOrShutdown()
         {
-            if (_functionDispatcher != null && _functionDispatcher.State == FunctionInvocationDispatcherState.Initializing)
+            // Don't delay if functionDispatcher is already initialized OR is skipping initialization for one of
+            // these reasons: started in placeholder, has no functions, functions do not match set language.
+            bool ready = _functionDispatcher.State == FunctionInvocationDispatcherState.Initialized || _functionDispatcher.State == FunctionInvocationDispatcherState.Default;
+
+            if (!ready)
             {
                 _logger.LogDebug($"functionDispatcher state: {_functionDispatcher.State}");
                 bool result = await Utility.DelayAsync((_functionDispatcher.ErrorEventsThreshold + 1) * WorkerConstants.ProcessStartTimeoutSeconds, WorkerConstants.WorkerReadyCheckPollingIntervalMilliseconds, () =>
diff --git a/src/WebJobs.Script/Host/ScriptHost.cs b/src/WebJobs.Script/Host/ScriptHost.cs
@@ -276,13 +276,16 @@ public async Task InitializeAsync(CancellationToken cancellationToken = default)
                 {
                     string runtimeStack = _workerRuntime;
 
-                    // Appending the runtime version is currently only enabled for linux consumption. This will be eventually enabled for
-                    // Windows Consumption as well.
-                    string runtimeVersion = _environment.GetEnvironmentVariable(RpcWorkerConstants.FunctionWorkerRuntimeVersionSettingName);
-
-                    if (!string.IsNullOrEmpty(runtimeVersion))
+                    if (!string.IsNullOrEmpty(_environment.GetEnvironmentVariable(RpcWorkerConstants.FunctionWorkerRuntimeSettingName)))
                     {
-                        runtimeStack = string.Concat(runtimeStack, "-", runtimeVersion);
+                        // Appending the runtime version is currently only enabled for linux consumption. This will be eventually enabled for
+                        // Windows Consumption as well.
+                        string runtimeVersion = _environment.GetEnvironmentVariable(RpcWorkerConstants.FunctionWorkerRuntimeVersionSettingName);
+
+                        if (!string.IsNullOrEmpty(runtimeVersion))
+                        {
+                            runtimeStack = string.Concat(runtimeStack, "-", runtimeVersion);
+                        }
                     }
 
                     _metricsLogger.LogEvent(string.Format(MetricEventNames.HostStartupRuntimeLanguage, runtimeStack));
diff --git a/src/WebJobs.Script/Workers/FunctionInvocationDispatcherState.cs b/src/WebJobs.Script/Workers/FunctionInvocationDispatcherState.cs
@@ -11,15 +11,30 @@ public enum FunctionInvocationDispatcherState
         Default,
 
         /// <summary>
-        /// The FunctionDispatcherState is starting.
+        /// The FunctionDispatcher is starting.
         /// </summary>
         Initializing,
 
         /// <summary>
-        /// The FunctionDispatcherState has been fully initialized and can accept direct function
-        /// invocations. All functions have been indexed. Listeners may not yet
-        /// be not yet running.
+        /// The FunctionDispatcher has been fully initialized and can accept function
+        /// invocations. All functions have been indexed. At least one worker process is initialized.
         /// </summary>
-        Initialized
+        Initialized,
+
+        /// <summary>
+        /// The FunctionDispatcher was previously "Initialized" but no longer has any initialized
+        /// worker processes to handle function invocations.
+        /// </summary>
+        WorkerProcessRestarting,
+
+        /// <summary>
+        /// The FunctionDispatcher is disposing
+        /// </summary>
+        Disposing,
+
+        /// <summary>
+        /// The FunctionDispatcher is disposed
+        /// </summary>
+        Disposed
     }
 }
diff --git a/src/WebJobs.Script/Workers/Http/HttpFunctionInvocationDispatcher.cs b/src/WebJobs.Script/Workers/Http/HttpFunctionInvocationDispatcher.cs
@@ -103,7 +103,7 @@ public async void WorkerError(HttpWorkerErrorEvent workerError)
         {
             if (!_disposing)
             {
-                _logger.LogDebug("Handling WorkerErrorEvent for workerId:{workerId}", workerError.WorkerId);
+                _logger.LogDebug("Handling WorkerErrorEvent for workerId:{workerId}. Failed with: {exception}", workerError.WorkerId, workerError.Exception);
                 AddOrUpdateErrorBucket(workerError);
                 await DisposeAndRestartWorkerChannel(workerError.WorkerId);
             }
@@ -120,6 +120,9 @@ public async void WorkerRestart(HttpWorkerRestartEvent workerRestart)
 
         private async Task DisposeAndRestartWorkerChannel(string workerId)
         {
+            // Since we only have one HTTP worker process, as soon as we dispose it, InvokeAsync will fail. Set state to
+            // indicate we are not ready to receive new requests.
+            State = FunctionInvocationDispatcherState.WorkerProcessRestarting;
             _logger.LogDebug("Disposing channel for workerId: {channelId}", workerId);
             if (_httpWorkerChannel != null)
             {
@@ -166,13 +169,15 @@ protected virtual void Dispose(bool disposing)
                 _workerErrorSubscription.Dispose();
                 _workerRestartSubscription.Dispose();
                 (_httpWorkerChannel as IDisposable)?.Dispose();
+                State = FunctionInvocationDispatcherState.Disposed;
                 _disposed = true;
             }
         }
 
         public void Dispose()
         {
             _disposing = true;
+            State = FunctionInvocationDispatcherState.Disposing;
             Dispose(true);
         }
 
diff --git a/src/WebJobs.Script/Workers/Rpc/FunctionRegistration/RpcFunctionInvocationDispatcher.cs b/src/WebJobs.Script/Workers/Rpc/FunctionRegistration/RpcFunctionInvocationDispatcher.cs
@@ -284,7 +284,7 @@ public async void WorkerError(WorkerErrorEvent workerError)
             {
                 if (string.Equals(_workerRuntime, workerError.Language))
                 {
-                    _logger.LogDebug("Handling WorkerErrorEvent for runtime:{runtime}, workerId:{workerId}", workerError.Language, workerError.WorkerId);
+                    _logger.LogDebug("Handling WorkerErrorEvent for runtime:{runtime}, workerId:{workerId}. Failed with: {exception}", workerError.Language, _workerRuntime, workerError.Exception);
                     AddOrUpdateErrorBucket(workerError);
                     await DisposeAndRestartWorkerChannel(workerError.Language, workerError.WorkerId);
                 }
@@ -332,8 +332,16 @@ private async Task DisposeAndRestartWorkerChannel(string runtime, string workerI
 
             if (ShouldRestartWorkerChannel(runtime, isWebHostChannel, isJobHostChannel))
             {
+                // Set state to "WorkerProcessRestarting" if there are no other workers to handle work
+                if ((await GetInitializedWorkerChannelsAsync()).Count() == 0)
+                {
+                    State = FunctionInvocationDispatcherState.WorkerProcessRestarting;
+                    _logger.LogDebug("No initialized worker channels for runtime '{runtime}'. Delaying future invocations", runtime);
+                }
+                // Restart worker channel
                 _logger.LogDebug("Restarting worker channel for runtime:{runtime}", runtime);
                 await RestartWorkerChannel(runtime, workerId);
+                // State is set back to "Initialized" when worker channel is up again
             }
             else
             {
@@ -386,13 +394,15 @@ protected virtual void Dispose(bool disposing)
                 _processStartCancellationToken.Cancel();
                 _processStartCancellationToken.Dispose();
                 _jobHostLanguageWorkerChannelManager.DisposeAndRemoveChannels();
+                State = FunctionInvocationDispatcherState.Disposed;
                 _disposed = true;
             }
         }
 
         public void Dispose()
         {
             _disposing = true;
+            State = FunctionInvocationDispatcherState.Disposing;
             Dispose(true);
         }
     }
diff --git a/test/WebJobs.Script.Tests/Description/DotNet/FunctionAssemblyLoadContextTests.cs b/test/WebJobs.Script.Tests/Description/DotNet/FunctionAssemblyLoadContextTests.cs
@@ -28,8 +28,8 @@ public class FunctionAssemblyLoadContextTests
         [InlineData("Microsoft.Azure.WebJobs.Script, Version=3.0.0.0")]
         [InlineData("Microsoft.Azure.WebJobs.Script.Grpc, Version=3.0.0.0")]
         [InlineData("Microsoft.Azure.WebJobs.Script.WebHost, Version=3.0.0.0")]
-        [InlineData("Microsoft.Azure.WebSites.DataProtection, Version=0.0.0.0")]
-        [InlineData("System.IO, Version=0.0.0.0")] // System.*
+        [InlineData("Microsoft.Azure.WebSites.DataProtection")]
+        [InlineData("System.IO")] // System.*
         public void RuntimeAssemblies_AreLoadedInDefaultContext(string assemblyName)
         {
             var functionContext = new FunctionAssemblyLoadContext(AppContext.BaseDirectory);
diff --git a/test/WebJobs.Script.Tests/Description/Worker/WorkerFunctionInvokerTests.cs b/test/WebJobs.Script.Tests/Description/Worker/WorkerFunctionInvokerTests.cs
@@ -56,6 +56,29 @@ public async Task InvokeTimeout_CallsShutdown()
             _applicationLifetime.Verify(a => a.StopApplication(), Times.Once);
         }
 
+        [Theory]
+        [InlineData(FunctionInvocationDispatcherState.Default, false)]
+        [InlineData(FunctionInvocationDispatcherState.Initializing, true)]
+        [InlineData(FunctionInvocationDispatcherState.Initialized, false)]
+        [InlineData(FunctionInvocationDispatcherState.WorkerProcessRestarting, true)]
+        [InlineData(FunctionInvocationDispatcherState.Disposing, true)]
+        [InlineData(FunctionInvocationDispatcherState.Disposed, true)]
+        public async Task FunctionDispatcher_DelaysInvoke_WhenNotReady(FunctionInvocationDispatcherState state, bool delaysExecution)
+        {
+            _mockFunctionInvocationDispatcher.Setup(a => a.State).Returns(state);
+            var timeoutTask = Task.Delay(TimeSpan.FromSeconds(5));
+            var invokeCoreTask = _testFunctionInvoker.InvokeCore(new object[] { }, null);
+            var result = await Task.WhenAny(invokeCoreTask, timeoutTask);
+            if (delaysExecution)
+            {
+                Assert.Equal(timeoutTask, result);
+            }
+            else
+            {
+                Assert.Equal(invokeCoreTask, result);
+            }
+        }
+
         [Fact]
         public async Task InvokeInitialized_DoesNotCallShutdown()
         {
diff --git a/test/WebJobs.Script.Tests/Diagnostics/LinuxContainerEventGeneratorTests.cs b/test/WebJobs.Script.Tests/Diagnostics/LinuxContainerEventGeneratorTests.cs
@@ -193,7 +193,7 @@ public void ToEventLevel_ReturnsExpectedValue(LogLevel logLevel, EventLevel even
             Assert.Equal(eventLevel, LinuxEventGenerator.ToEventLevel(logLevel));
         }
 
-        [Theory]
+        [Theory(Skip = "https://github.com/Azure/azure-functions-host/issues/5636")]
         [MemberData(nameof(LinuxEventGeneratorTestData.GetAzureMonitorEvents), MemberType = typeof(LinuxEventGeneratorTestData))]
         public void ParseAzureMonitoringEvents(LogLevel level, string resourceId, string operationName, string category, string regionName, string properties)
         {
diff --git a/test/WebJobs.Script.Tests/ScriptHostTests.cs b/test/WebJobs.Script.Tests/ScriptHostTests.cs
@@ -15,6 +15,7 @@
 using Microsoft.Azure.WebJobs.Script.Config;
 using Microsoft.Azure.WebJobs.Script.Configuration;
 using Microsoft.Azure.WebJobs.Script.Description;
+using Microsoft.Azure.WebJobs.Script.Diagnostics;
 using Microsoft.Azure.WebJobs.Script.Eventing;
 using Microsoft.Azure.WebJobs.Script.Workers.Rpc;
 using Microsoft.Extensions.Configuration;
@@ -404,6 +405,67 @@ public async Task Initialize_WithLatestSiteExtensionVersion_LogsWarning()
             }
         }
 
+        [Theory]
+        [InlineData("dotnet", "", "", "dotnet")]
+        [InlineData("dotnet", "", "~2", "dotnet-~2")]
+        [InlineData("python", "~2", "", "python")]
+        [InlineData("python", "~2", "3.6", "python-3.6")]
+        [InlineData("python", "~3", "3.7", "python-3.7")]
+        [InlineData("node", "~3", "~8", "node-~8")]
+        [InlineData("node", "~2", "~8", "node-~8")]
+        [InlineData("powershell", "~2", "", "powershell")]
+        [InlineData("powershell", "~2", "~7", "powershell-~7")]
+        [InlineData("java", "~3", "", "java")]
+        public async Task Initialize_WithRuntimeAndWorkerVersion_ReportRuntimeToMetricsTable(
+            string functionsWorkerRuntime,
+            string functionsExtensionVersion,
+            string functionsWorkerRuntimeVersion,
+            string expectedRuntimeStack)
+        {
+            try
+            {
+                using (var tempDirectory = new TempDirectory())
+                {
+                    string rootPath = Path.Combine(tempDirectory.Path, Guid.NewGuid().ToString());
+                    Directory.CreateDirectory(rootPath);
+                    var metricsLogger = new TestMetricsLogger();
+                    var environment = new TestEnvironment();
+
+                    environment.SetEnvironmentVariable(
+                        RpcWorkerConstants.FunctionWorkerRuntimeSettingName, functionsWorkerRuntime);
+                    environment.SetEnvironmentVariable(
+                        EnvironmentSettingNames.FunctionsExtensionVersion, functionsExtensionVersion);
+                    environment.SetEnvironmentVariable(
+                        RpcWorkerConstants.FunctionWorkerRuntimeVersionSettingName, functionsWorkerRuntimeVersion);
+
+                    IHost host = new HostBuilder()
+                        .ConfigureServices(s =>
+                        {
+                            s.AddSingleton<IEnvironment>(environment);
+                        })
+                        .ConfigureDefaultTestWebScriptHost(
+                        null,
+                        o =>
+                        {
+                            o.ScriptPath = rootPath;
+                        },
+                        false,
+                        s =>
+                        {
+                            s.AddSingleton<IMetricsLogger>(metricsLogger);
+                        })
+                        .Build();
+                    var scriptHost = host.GetScriptHost();
+                    await scriptHost.InitializeAsync();
+                    Assert.Single(metricsLogger.LoggedEvents, e => e.Equals($"host.startup.runtime.language.{expectedRuntimeStack}"));
+                }
+            }
+            finally
+            {
+                EnvironmentExtensions.BaseDirectory = null;
+            }
+        }
+
         // TODO: Newer TODO - ApplyConfiguration no longer exists. Validate logic (moved to HostJsonFileConfigurationSource)
         // TODO: Move this test into a new WebJobsCoreScriptBindingProvider class since
         // the functionality moved. Also add tests for the ServiceBus config, etc.
diff --git a/test/WebJobs.Script.Tests/Workers/Rpc/RpcFunctionInvocationDispatcherTests.cs b/test/WebJobs.Script.Tests/Workers/Rpc/RpcFunctionInvocationDispatcherTests.cs
@@ -200,8 +200,31 @@ public async Task FunctionDispatcherState_Transitions_From_Starting_To_Initializ
                 func1
             };
             await functionDispatcher.InitializeAsync(functions);
-            Assert.True(functionDispatcher.State == FunctionInvocationDispatcherState.Initializing || functionDispatcher.State == FunctionInvocationDispatcherState.Initialized);
+            Assert.Equal(FunctionInvocationDispatcherState.Initializing, functionDispatcher.State);
             await WaitForFunctionDispactherStateInitialized(functionDispatcher);
+            Assert.Equal(FunctionInvocationDispatcherState.Initialized, functionDispatcher.State);
+        }
+
+        [Fact]
+        public async Task FunctionDispatcherState_Transitions_From_Default_To_Initialized_To_Disposing()
+        {
+            RpcFunctionInvocationDispatcher functionDispatcher = GetTestFunctionDispatcher();
+            Assert.Equal(FunctionInvocationDispatcherState.Default, functionDispatcher.State);
+            FunctionMetadata func1 = new FunctionMetadata()
+            {
+                Name = "func1",
+                Language = "node"
+            };
+            var functions = new List<FunctionMetadata>()
+            {
+                func1
+            };
+            await functionDispatcher.InitializeAsync(functions);
+            Assert.Equal(FunctionInvocationDispatcherState.Initializing, functionDispatcher.State);
+            await WaitForFunctionDispactherStateInitialized(functionDispatcher);
+            Assert.Equal(FunctionInvocationDispatcherState.Initialized, functionDispatcher.State);
+            functionDispatcher.Dispose();
+            Assert.True(functionDispatcher == null || functionDispatcher.State == FunctionInvocationDispatcherState.Disposing || functionDispatcher.State == FunctionInvocationDispatcherState.Disposed);
         }
 
         [Fact]
@@ -225,6 +248,43 @@ public async Task FunctionDispatcher_Restart_ErroredChannels_Succeeds()
             Assert.Equal(expectedProcessCount, finalChannelCount);
         }
 
+        [Fact]
+        public async Task FunctionDispatcher_Restart_ErroredChannels_And_Changes_State()
+        {
+            int expectedProcessCount = 1;
+            RpcFunctionInvocationDispatcher functionDispatcher = GetTestFunctionDispatcher(expectedProcessCount.ToString());
+            Assert.Equal(FunctionInvocationDispatcherState.Default, functionDispatcher.State);
+            // Add worker
+            await functionDispatcher.InitializeAsync(GetTestFunctionsList(RpcWorkerConstants.NodeLanguageWorkerName));
+            await WaitForJobhostWorkerChannelsToStartup(functionDispatcher, expectedProcessCount);
+            TestRpcWorkerChannel testWorkerChannel = (TestRpcWorkerChannel)functionDispatcher.JobHostLanguageWorkerChannelManager.GetChannels().FirstOrDefault();
+            // Restart this channel
+            testWorkerChannel.RaiseWorkerRestart();
+            await TestHelpers.Await(() =>
+            {
+                return functionDispatcher.State == FunctionInvocationDispatcherState.WorkerProcessRestarting;
+            }, 3000);
+            await WaitForJobhostWorkerChannelsToStartup(functionDispatcher, expectedProcessCount);
+            Assert.Equal(FunctionInvocationDispatcherState.Initialized, functionDispatcher.State);
+        }
+
+        [Fact]
+        public async Task FunctionDispatcher_Restart_ErroredChannels_And_DoesNot_Change_State()
+        {
+            int expectedProcessCount = 2;
+            RpcFunctionInvocationDispatcher functionDispatcher = GetTestFunctionDispatcher(expectedProcessCount.ToString());
+            Assert.Equal(FunctionInvocationDispatcherState.Default, functionDispatcher.State);
+            // Add worker
+            await functionDispatcher.InitializeAsync(GetTestFunctionsList(RpcWorkerConstants.NodeLanguageWorkerName));
+            await WaitForJobhostWorkerChannelsToStartup(functionDispatcher, expectedProcessCount);
+            TestRpcWorkerChannel testWorkerChannel = (TestRpcWorkerChannel)functionDispatcher.JobHostLanguageWorkerChannelManager.GetChannels().FirstOrDefault();
+            // Restart one channel
+            testWorkerChannel.RaiseWorkerRestart();
+            Assert.Equal(FunctionInvocationDispatcherState.Initialized, functionDispatcher.State);
+            await WaitForJobhostWorkerChannelsToStartup(functionDispatcher, expectedProcessCount);
+            Assert.Equal(FunctionInvocationDispatcherState.Initialized, functionDispatcher.State);
+        }
+
         [Fact]
         public async Task FunctionDispatcher_Restart_ErroredChannels_ExceedsLimit()
         {
diff --git a/test/WebJobs.Script.Tests/Workers/Rpc/TestRpcWorkerChannel.cs b/test/WebJobs.Script.Tests/Workers/Rpc/TestRpcWorkerChannel.cs

Original file line number	Diff line number	Diff line change
`@@ -104,7 +104,7 @@ private void ConsoleWriter(string evt)`
`104`	`104`
`105`	`105`	`public override void LogAzureMonitorDiagnosticLogEvent(LogLevel level, string resourceId, string operationName, string category, string regionName, string properties)`
`106`	`106`	`{`
`107`		`- _writeEvent($"{ScriptConstants.LinuxAzureMonitorEventStreamName} {(int)ToEventLevel(level)},{resourceId},{operationName},{category},{regionName},{NormalizeString(properties)},{_containerName},{TenantId},{DateTime.UtcNow.ToString()}");`
	`107`	`+// _writeEvent($"{ScriptConstants.LinuxAzureMonitorEventStreamName} {(int)ToEventLevel(level)},{resourceId},{operationName},{category},{regionName},{NormalizeString(properties)},{_containerName},{TenantId},{DateTime.UtcNow.ToString()}");`
`108`	`108`	`}`
`109`	`109`	`}`
`110`	`110`	`}`
Original file line number	Diff line number	Diff line change
`@@ -155,8 +155,8 @@ private static bool IsMinorMatchOrLowerPolicyEvaluator(AssemblyName requestedAss`
`155`	`155`	`{`
`156`	`156`	`AssemblyName runtimeAssemblyName = AssemblyNameCache.GetName(runtimeAssembly);`
`157`	`157`
`158`		`- return requestedAssembly.Version.Major == runtimeAssemblyName.Version.Major &&`
`159`		`- requestedAssembly.Version.Minor <= runtimeAssemblyName.Version.Minor;`
	`158`	`+ return requestedAssembly.Version == null \|\| (requestedAssembly.Version.Major == runtimeAssemblyName.Version.Major &&`
	`159`	`+ requestedAssembly.Version.Minor <= runtimeAssemblyName.Version.Minor);`
`160`	`160`	`}`
`161`	`161`
`162`	`162`	`private bool IsRuntimeAssembly(AssemblyName assemblyName)`
Original file line number	Diff line number	Diff line change
`@@ -90,7 +90,11 @@ protected override async Task<object> InvokeCore(object[] parameters, FunctionIn`
`90`	`90`
`91`	`91`	`private async Task DelayUntilFunctionDispatcherInitializedOrShutdown()`
`92`	`92`	`{`
`93`		`- if (_functionDispatcher != null && _functionDispatcher.State == FunctionInvocationDispatcherState.Initializing)`
	`93`	`+ // Don't delay if functionDispatcher is already initialized OR is skipping initialization for one of`
	`94`	`+ // these reasons: started in placeholder, has no functions, functions do not match set language.`
	`95`	`+ bool ready = _functionDispatcher.State == FunctionInvocationDispatcherState.Initialized \|\| _functionDispatcher.State == FunctionInvocationDispatcherState.Default;`
	`96`	`+`
	`97`	`+ if (!ready)`
`94`	`98`	`{`
`95`	`99`	`_logger.LogDebug($"functionDispatcher state: {_functionDispatcher.State}");`
`96`	`100`	`bool result = await Utility.DelayAsync((_functionDispatcher.ErrorEventsThreshold + 1) * WorkerConstants.ProcessStartTimeoutSeconds, WorkerConstants.WorkerReadyCheckPollingIntervalMilliseconds, () =>`
Original file line number	Diff line number	Diff line change
`@@ -103,7 +103,7 @@ public async void WorkerError(HttpWorkerErrorEvent workerError)`
`103`	`103`	`{`
`104`	`104`	`if (!_disposing)`
`105`	`105`	`{`
`106`		`- _logger.LogDebug("Handling WorkerErrorEvent for workerId:{workerId}", workerError.WorkerId);`
	`106`	`+ _logger.LogDebug("Handling WorkerErrorEvent for workerId:{workerId}. Failed with: {exception}", workerError.WorkerId, workerError.Exception);`
`107`	`107`	`AddOrUpdateErrorBucket(workerError);`
`108`	`108`	`await DisposeAndRestartWorkerChannel(workerError.WorkerId);`
`109`	`109`	`}`
`@@ -120,6 +120,9 @@ public async void WorkerRestart(HttpWorkerRestartEvent workerRestart)`
`120`	`120`
`121`	`121`	`private async Task DisposeAndRestartWorkerChannel(string workerId)`
`122`	`122`	`{`
	`123`	`+ // Since we only have one HTTP worker process, as soon as we dispose it, InvokeAsync will fail. Set state to`
	`124`	`+ // indicate we are not ready to receive new requests.`
	`125`	`+ State = FunctionInvocationDispatcherState.WorkerProcessRestarting;`
`123`	`126`	`_logger.LogDebug("Disposing channel for workerId: {channelId}", workerId);`
`124`	`127`	`if (_httpWorkerChannel != null)`
`125`	`128`	`{`
`@@ -166,13 +169,15 @@ protected virtual void Dispose(bool disposing)`
`166`	`169`	`_workerErrorSubscription.Dispose();`
`167`	`170`	`_workerRestartSubscription.Dispose();`
`168`	`171`	`(_httpWorkerChannel as IDisposable)?.Dispose();`
	`172`	`+ State = FunctionInvocationDispatcherState.Disposed;`
`169`	`173`	`_disposed = true;`
`170`	`174`	`}`
`171`	`175`	`}`
`172`	`176`
`173`	`177`	`public void Dispose()`
`174`	`178`	`{`
`175`	`179`	`_disposing = true;`
	`180`	`+ State = FunctionInvocationDispatcherState.Disposing;`
`176`	`181`	`Dispose(true);`
`177`	`182`	`}`
`178`	`183`
Original file line number	Diff line number	Diff line change
`@@ -284,7 +284,7 @@ public async void WorkerError(WorkerErrorEvent workerError)`
`284`	`284`	`{`
`285`	`285`	`if (string.Equals(_workerRuntime, workerError.Language))`
`286`	`286`	`{`
`287`		`- _logger.LogDebug("Handling WorkerErrorEvent for runtime:{runtime}, workerId:{workerId}", workerError.Language, workerError.WorkerId);`
	`287`	`+ _logger.LogDebug("Handling WorkerErrorEvent for runtime:{runtime}, workerId:{workerId}. Failed with: {exception}", workerError.Language, _workerRuntime, workerError.Exception);`
`288`	`288`	`AddOrUpdateErrorBucket(workerError);`
`289`	`289`	`await DisposeAndRestartWorkerChannel(workerError.Language, workerError.WorkerId);`
`290`	`290`	`}`
`@@ -332,8 +332,16 @@ private async Task DisposeAndRestartWorkerChannel(string runtime, string workerI`
`332`	`332`
`333`	`333`	`if (ShouldRestartWorkerChannel(runtime, isWebHostChannel, isJobHostChannel))`
`334`	`334`	`{`
	`335`	`+ // Set state to "WorkerProcessRestarting" if there are no other workers to handle work`
	`336`	`+ if ((await GetInitializedWorkerChannelsAsync()).Count() == 0)`
	`337`	`+ {`
	`338`	`+ State = FunctionInvocationDispatcherState.WorkerProcessRestarting;`
	`339`	`+ _logger.LogDebug("No initialized worker channels for runtime '{runtime}'. Delaying future invocations", runtime);`
	`340`	`+ }`
	`341`	`+ // Restart worker channel`
`335`	`342`	`_logger.LogDebug("Restarting worker channel for runtime:{runtime}", runtime);`
`336`	`343`	`await RestartWorkerChannel(runtime, workerId);`
	`344`	`+ // State is set back to "Initialized" when worker channel is up again`
`337`	`345`	`}`
`338`	`346`	`else`
`339`	`347`	`{`
`@@ -386,13 +394,15 @@ protected virtual void Dispose(bool disposing)`
`386`	`394`	`_processStartCancellationToken.Cancel();`
`387`	`395`	`_processStartCancellationToken.Dispose();`
`388`	`396`	`_jobHostLanguageWorkerChannelManager.DisposeAndRemoveChannels();`
	`397`	`+ State = FunctionInvocationDispatcherState.Disposed;`
`389`	`398`	`_disposed = true;`
`390`	`399`	`}`
`391`	`400`	`}`
`392`	`401`
`393`	`402`	`public void Dispose()`
`394`	`403`	`{`
`395`	`404`	`_disposing = true;`
	`405`	`+ State = FunctionInvocationDispatcherState.Disposing;`
`396`	`406`	`Dispose(true);`
`397`	`407`	`}`
`398`	`408`	`}`
Original file line number	Diff line number	Diff line change
`@@ -193,7 +193,7 @@ public void ToEventLevel_ReturnsExpectedValue(LogLevel logLevel, EventLevel even`
`193`	`193`	`Assert.Equal(eventLevel, LinuxEventGenerator.ToEventLevel(logLevel));`
`194`	`194`	`}`
`195`	`195`
`196`		`- [Theory]`
	`196`	`+ [Theory(Skip = "https://github.com/Azure/azure-functions-host/issues/5636")]`
`197`	`197`	`[MemberData(nameof(LinuxEventGeneratorTestData.GetAzureMonitorEvents), MemberType = typeof(LinuxEventGeneratorTestData))]`
`198`	`198`	`public void ParseAzureMonitoringEvents(LogLevel level, string resourceId, string operationName, string category, string regionName, string properties)`
`199`	`199`	`{`