Fixing race condition in host restart

Author: mathewc

src/WebJobs.Script.WebHost/App_Start/WebApiConfig.cs

@@ -82,8 +82,8 @@ public static void Register(HttpConfiguration config, ScriptSettingsManager sett
 
         private static void AddMessageHandlers(HttpConfiguration config)
         {
-            config.MessageHandlers.Add(new WebScriptHostHandler(config));
             config.MessageHandlers.Add(new SystemTraceHandler(config));
+            config.MessageHandlers.Add(new WebScriptHostHandler(config));
         }
     }
 }

src/WebJobs.Script/Host/ScriptHostManager.cs

@@ -399,12 +399,15 @@ protected virtual void Dispose(bool disposing)
 
         public virtual void RestartHost()
         {
-            _restartHostEvent.Set();
-
-            // we reset the state immediately here to ensure that
+            // We reset the state immediately here to ensure that
             // any external calls to CanInvoke will return false
-            // immediately after the restart has been initiated
+            // immediately after the restart has been initiated.
+            // Note: this state change must be done BEFORE we set
+            // the event below to avoid a race condition with the
+            // main host lifetime loop.
             State = ScriptHostState.Default;
+
+            _restartHostEvent.Set();
         }
 
         public virtual void Shutdown()

test/WebJobs.Script.Tests.Integration/Host/ScriptHostManagerTests.cs

@@ -287,10 +287,16 @@ public async Task EmptyHost_StartsSuccessfully()
             var eventManagerMock = new Mock<IScriptEventManager>();
             ScriptHostManager hostManager = new ScriptHostManager(config, eventManagerMock.Object);
 
+            // start the host and wait for it to be running
             Task runTask = Task.Run(() => hostManager.RunAndBlock());
+            await TestHelpers.Await(() => hostManager.State == ScriptHostState.Running, timeout: 10000);
 
+            // exercise restart
+            hostManager.RestartHost();
+            Assert.Equal(ScriptHostState.Default, hostManager.State);
             await TestHelpers.Await(() => hostManager.State == ScriptHostState.Running, timeout: 10000);
 
+            // stop the host fully
             hostManager.Stop();
             Assert.Equal(ScriptHostState.Default, hostManager.State);