Fix for unhealthy host restart (#9851)

Author: soninaren

src/WebJobs.Script/Config/FunctionsHostingConfigOptions.cs

@@ -43,6 +43,22 @@ public bool WorkerIndexingEnabled
             }
         }
 
+        /// <summary>
+        /// Gets or Sets a value indicating whether the host should shutdown webhost worker channels during shutdown.
+        /// </summary>
+        public bool ShutdownWebhostWorkerChannelsOnHostShutdown
+        {
+            get
+            {
+                return GetFeatureOrDefault(RpcWorkerConstants.ShutdownWebhostWorkerChannelsOnHostShutdown, "1") == "1";
+            }
+
+            set
+            {
+                _features[RpcWorkerConstants.ShutdownWebhostWorkerChannelsOnHostShutdown] = value ? "1" : "0";
+            }
+        }
+
         /// <summary>
         /// Gets or sets a value indicating whether SWT tokens should be accepted.
         /// </summary>

src/WebJobs.Script/Workers/FunctionInvocationDispatcherFactory.cs

@@ -3,6 +3,7 @@
 
 using System;
 using Microsoft.AspNetCore.Hosting;
+using Microsoft.Azure.WebJobs.Script.Config;
 using Microsoft.Azure.WebJobs.Script.Diagnostics;
 using Microsoft.Azure.WebJobs.Script.Eventing;
 using Microsoft.Azure.WebJobs.Script.ManagedDependencies;
@@ -31,7 +32,8 @@ public FunctionInvocationDispatcherFactory(IOptions<ScriptJobHostOptions> script
             IJobHostRpcWorkerChannelManager jobHostLanguageWorkerChannelManager,
             IOptions<ManagedDependencyOptions> managedDependencyOptions,
             IRpcFunctionInvocationDispatcherLoadBalancer functionDispatcherLoadBalancer,
-            IOptions<WorkerConcurrencyOptions> workerConcurrencyOptions)
+            IOptions<WorkerConcurrencyOptions> workerConcurrencyOptions,
+            IOptions<FunctionsHostingConfigOptions> hostingConfigOptions)
         {
             if (httpWorkerOptions.Value == null)
             {
@@ -55,7 +57,8 @@ public FunctionInvocationDispatcherFactory(IOptions<ScriptJobHostOptions> script
                 jobHostLanguageWorkerChannelManager,
                 managedDependencyOptions,
                 functionDispatcherLoadBalancer,
-                workerConcurrencyOptions);
+                workerConcurrencyOptions,
+                hostingConfigOptions);
         }
 
         public IFunctionInvocationDispatcher GetFunctionDispatcher() => _functionDispatcher;

src/WebJobs.Script/Workers/Rpc/FunctionRegistration/RpcFunctionInvocationDispatcher.cs

@@ -11,6 +11,7 @@
 using System.Threading.Tasks.Dataflow;
 using Microsoft.AspNetCore.Hosting;
 using Microsoft.Azure.WebJobs.Logging;
+using Microsoft.Azure.WebJobs.Script.Config;
 using Microsoft.Azure.WebJobs.Script.Description;
 using Microsoft.Azure.WebJobs.Script.Diagnostics;
 using Microsoft.Azure.WebJobs.Script.Eventing;
@@ -36,6 +37,7 @@ internal class RpcFunctionInvocationDispatcher : IFunctionInvocationDispatcher
         private readonly IOptions<WorkerConcurrencyOptions> _workerConcurrencyOptions;
         private readonly IEnumerable<RpcWorkerConfig> _workerConfigs;
         private readonly Lazy<Task<int>> _maxProcessCount;
+        private readonly IOptions<FunctionsHostingConfigOptions> _hostingConfigOptions;
 
         private IScriptEventManager _eventManager;
         private IWebHostRpcWorkerChannelManager _webHostLanguageWorkerChannelManager;
@@ -69,7 +71,8 @@ public RpcFunctionInvocationDispatcher(IOptions<ScriptJobHostOptions> scriptHost
             IJobHostRpcWorkerChannelManager jobHostLanguageWorkerChannelManager,
             IOptions<ManagedDependencyOptions> managedDependencyOptions,
             IRpcFunctionInvocationDispatcherLoadBalancer functionDispatcherLoadBalancer,
-            IOptions<WorkerConcurrencyOptions> workerConcurrencyOptions)
+            IOptions<WorkerConcurrencyOptions> workerConcurrencyOptions,
+            IOptions<FunctionsHostingConfigOptions> hostingConfigOptions)
         {
             _metricsLogger = metricsLogger;
             _scriptOptions = scriptHostOptions.Value;
@@ -85,6 +88,7 @@ public RpcFunctionInvocationDispatcher(IOptions<ScriptJobHostOptions> scriptHost
             _workerRuntime = _environment.GetEnvironmentVariable(RpcWorkerConstants.FunctionWorkerRuntimeSettingName);
             _functionDispatcherLoadBalancer = functionDispatcherLoadBalancer;
             _workerConcurrencyOptions = workerConcurrencyOptions;
+            _hostingConfigOptions = hostingConfigOptions;
             State = FunctionInvocationDispatcherState.Default;
 
             _workerErrorSubscription = _eventManager.OfType<WorkerErrorEvent>().Subscribe(WorkerError);
@@ -683,6 +687,10 @@ public void PreShutdown()
         {
             _logger.LogDebug($"Preventing any new worker processes from starting during shutdown.");
             _processStartCancellationToken.Cancel();
+            if (_hostingConfigOptions.Value.ShutdownWebhostWorkerChannelsOnHostShutdown && !_scriptOptions.IsStandbyConfiguration)
+            {
+                ShutdownWebhostLanguageWorkerChannels();
+            }
         }
     }
 }

src/WebJobs.Script/Workers/Rpc/RpcWorkerConstants.cs

@@ -85,6 +85,7 @@ public static class RpcWorkerConstants
         public const string WorkerIndexingEnabled = "WORKER_INDEXING_ENABLED";
         public const string WorkerIndexingDisabledApps = "WORKER_INDEXING_DISABLED_APPS";
         public const string RevertWorkerShutdownBehavior = "REVERT_WORKER_SHUTDOWN_BEHAVIOR";
+        public const string ShutdownWebhostWorkerChannelsOnHostShutdown = "ShutdownWebhostWorkerChannelsOnHostShutdown";
         public const string ThrowOnMissingFunctionsWorkerRuntime = "THROW_ON_MISSING_FUNCTIONS_WORKER_RUNTIME";
     }
 }

test/WebJobs.Script.Tests.Integration/WebHostEndToEnd/SamplesEndToEndTests_Node_MultipleProcessesNoBundle.cs

@@ -34,7 +34,7 @@ public SamplesEndToEndTests_Node_MultipleProcessesNoBundle(MultiplepleProcessesT
             _settingsManager = ScriptSettingsManager.Instance;
         }
 
-        [Fact(Skip = "https://github.com/Azure/azure-functions-host/issues")]
+        [Fact]
         public async Task NodeProcessNoBundleConfigured_Different_AfterHostRestart()
         {
             await SamplesTestHelpers.InvokeAndValidateHttpTrigger(_fixture, "HttpTrigger");

test/WebJobs.Script.Tests/Workers/Rpc/RpcFunctionInvocationDispatcherTests.cs

@@ -8,6 +8,7 @@
 using System.Threading;
 using System.Threading.Tasks;
 using Microsoft.AspNetCore.Hosting;
+using Microsoft.Azure.WebJobs.Script.Config;
 using Microsoft.Azure.WebJobs.Script.Description;
 using Microsoft.Azure.WebJobs.Script.Diagnostics;
 using Microsoft.Azure.WebJobs.Script.Eventing;
@@ -696,7 +697,8 @@ private static RpcFunctionInvocationDispatcher GetTestFunctionDispatcher(int max
                 jobHostLanguageWorkerChannelManager,
                 new OptionsWrapper<ManagedDependencyOptions>(new ManagedDependencyOptions()),
                 mockFunctionDispatcherLoadBalancer.Object,
-                Options.Create(new WorkerConcurrencyOptions()));
+                Options.Create(new WorkerConcurrencyOptions()),
+                Options.Create(new FunctionsHostingConfigOptions()));
         }
 
         private async Task<int> WaitForJobhostWorkerChannelsToStartup(RpcFunctionInvocationDispatcher functionDispatcher, int expectedCount, bool allReadyForInvocations = true)