HealthMonitor improvements (#2972)

Author: mathewc

src/WebJobs.Script.WebHost/WebScriptHostManager.cs

@@ -434,14 +434,24 @@ private void InitializeHttpFunctions(IEnumerable<FunctionDescriptor> functions,
             }
         }
 
-        public override void Shutdown()
+        public override void Shutdown(bool hard = false)
         {
             string message = "Environment shutdown has been triggered. Stopping host and signaling shutdown.";
             Instance?.TraceWriter.Info(message);
             Instance?.Logger?.LogInformation(message);
 
             Stop();
-            HostingEnvironment.InitiateShutdown();
+
+            if (hard)
+            {
+                // "hard" shutdown recycles the process
+                Process.GetCurrentProcess().Kill();
+            }
+            else
+            {
+                // "soft" shutdown recycles the AppDomain
+                HostingEnvironment.InitiateShutdown();
+            }
         }
 
         public async Task DelayUntilHostReady()

src/WebJobs.Script/Host/IScriptHostEnvironment.cs

@@ -19,6 +19,7 @@ public interface IScriptHostEnvironment
         /// <summary>
         /// Stops the <see cref="ScriptHost"/> and shuts down the hosting environment.
         /// </summary>
-        void Shutdown();
+        /// <param name="hard">True if the shutdown should be "hard" - i.e. shut down process.</param>
+        void Shutdown(bool hard = false);
     }
 }

src/WebJobs.Script/Host/NullScriptHostEnvironment.cs

@@ -15,7 +15,7 @@ public void RestartHost()
         {
         }
 
-        public void Shutdown()
+        public void Shutdown(bool hard = false)
         {
         }
     }

src/WebJobs.Script/Host/ScriptHostManager.cs

@@ -12,6 +12,7 @@
 using System.Text;
 using System.Threading;
 using System.Threading.Tasks;
+using Microsoft.Azure.WebJobs.Host;
 using Microsoft.Azure.WebJobs.Script.Config;
 using Microsoft.Azure.WebJobs.Script.Diagnostics;
 using Microsoft.Azure.WebJobs.Script.Eventing;
@@ -266,13 +267,15 @@ private bool ShutdownHostIfUnhealthy()
         {
             if (ShouldMonitorHostHealth && _healthCheckWindow.GetEvents().Where(isHealthy => !isHealthy).Count() > _config.HostHealthMonitor.HealthCheckThreshold)
             {
-                // if the number of times the host has been unhealthy in
+                // If the number of times the host has been unhealthy in
                 // the current time window exceeds the threshold, recover by
-                // initiating shutdown
-                var message = $"Host unhealthy count exceeds the threshold of {_config.HostHealthMonitor.HealthCheckThreshold} for time window {_config.HostHealthMonitor.HealthCheckWindow}. Initiating shutdown.";
-                Instance?.TraceWriter?.Error(message);
+                // initiating a hard shutdown.
+                var message = $"Host unhealthy count exceeds the threshold of {_config.HostHealthMonitor.HealthCheckThreshold} for time window {_config.HostHealthMonitor.HealthCheckWindow}. Initiating hard shutdown.";
+                var shutdownEvent = new TraceEvent(TraceLevel.Error, message);
+                shutdownEvent.Properties.Add(ScriptConstants.TracePropertyEventNameKey, ScriptConstants.ShutdownRecoveryEventName);
+                Instance?.TraceWriter?.Trace(shutdownEvent);
                 Instance?.Logger?.LogError(0, message);
-                _environment.Shutdown();
+                _environment.Shutdown(hard: true);
                 return true;
             }
 
@@ -504,11 +507,11 @@ public virtual void RestartHost()
             _restartHostEvent.Set();
         }
 
-        public virtual void Shutdown()
+        public virtual void Shutdown(bool hard = false)
         {
             Stop();
 
-            Process.GetCurrentProcess().Close();
+            Process.GetCurrentProcess().Kill();
         }
 
         private void OnHostHealthCheckTimer(object state)

src/WebJobs.Script/ScriptConstants.cs

@@ -63,6 +63,7 @@ public static class ScriptConstants
         public const string DefaultMasterKeyName = "master";
         public const string DefaultFunctionKeyName = "default";
         public const string ColdStartEventName = "ColdStart";
+        public const string ShutdownRecoveryEventName = "ShutdownRecovery";
 
         public const string AntaresLogIdHeaderName = "X-ARR-LOG-ID";
         public const string AntaresScaleOutHeaderName = "X-FUNCTION-SCALEOUT";

test/WebJobs.Script.Tests.Integration/Host/ScriptHostManagerTests.cs

@@ -403,7 +403,7 @@ public async Task HostHealthMonitor_TriggersShutdown_WhenHostUnhealthy()
 
             var environmentMock = new Mock<IScriptHostEnvironment>(MockBehavior.Strict);
             bool shutdownCalled = false;
-            environmentMock.Setup(p => p.Shutdown()).Callback(() => shutdownCalled = true);
+            environmentMock.Setup(p => p.Shutdown(true)).Callback(() => shutdownCalled = true);
 
             var mockSettings = new Mock<ScriptSettingsManager>();
             mockSettings.Setup(p => p.IsAzureEnvironment).Returns(true);
@@ -431,7 +431,7 @@ public async Task HostHealthMonitor_TriggersShutdown_WhenHostUnhealthy()
 
             await TestHelpers.Await(() => hostManager.State == ScriptHostState.Error && shutdownCalled);
 
-            environmentMock.Verify(p => p.Shutdown(), Times.Once);
+            environmentMock.Verify(p => p.Shutdown(true), Times.Once);
 
             var traces = testTraceWriter.GetTraces();
 
@@ -441,7 +441,8 @@ public async Task HostHealthMonitor_TriggersShutdown_WhenHostUnhealthy()
 
             var log = traces.Last();
             Assert.True(traces.Count(p => p.Message == "Host is unhealthy. Initiating a restart." && p.Level == TraceLevel.Error) > 0);
-            Assert.Equal("Host unhealthy count exceeds the threshold of 5 for time window 00:00:01. Initiating shutdown.", log.Message);
+            Assert.Equal("Host unhealthy count exceeds the threshold of 5 for time window 00:00:01. Initiating hard shutdown.", log.Message);
+            Assert.Equal(ScriptConstants.ShutdownRecoveryEventName, log.Properties[ScriptConstants.TracePropertyEventNameKey]);
             Assert.Equal(TraceLevel.Error, log.Level);
         }
 

test/WebJobs.Script.Tests.Integration/RawAssemblyEndToEndTests.cs

@@ -59,7 +59,7 @@ public async Task InvokeDotNetFunction(string functionName, string expectedResul
         public void AssemblyChange_TriggersEnvironmentShutdown()
         {
             var manualResetEvent = new ManualResetEvent(false);
-            Fixture.ScriptHostEnvironmentMock.Setup(e => e.Shutdown())
+            Fixture.ScriptHostEnvironmentMock.Setup(e => e.Shutdown(false))
                 .Callback(() => manualResetEvent.Set());
 
             string sourceFile = TestFixture.SharedAssemblyPath;

test/WebJobs.Script.Tests/Description/DotNet/DotNetFunctionInvokerTests.cs

@@ -346,7 +346,7 @@ public async Task RestorePackagesAsync_WithUpdatedReferences_TriggersShutdown(bo
                 // and won't be made immediately
                 await Task.Delay(1000);
 
-                environmentMock.Verify(e => e.Shutdown(), Times.Exactly(shutdownExpected ? 1 : 0));
+                environmentMock.Verify(e => e.Shutdown(false), Times.Exactly(shutdownExpected ? 1 : 0));
             }
         }