Host state tracking improvements

Author: mathewc

src/WebJobs.Script.WebHost/WebJobs.Script.WebHost.csproj

@@ -37,7 +37,7 @@
     <PackageReference Include="Microsoft.Azure.AppService.Proxy.Client.Contract" Version="0.3.1.1">
       <NoWarn>NU1701</NoWarn>
     </PackageReference>
-    <PackageReference Include="Microsoft.Azure.WebJobs" Version="3.0.0-beta3-11077" />
+    <PackageReference Include="Microsoft.Azure.WebJobs" Version="3.0.0-beta3-11084" />
     <PackageReference Include="Microsoft.Azure.WebJobs.Extensions" Version="3.0.0-beta3" />
     <PackageReference Include="Microsoft.Azure.WebJobs.Extensions.Http" Version="3.0.0-beta3">
       <NoWarn>NU1701</NoWarn>

src/WebJobs.Script.WebHost/WebScriptHostManager.cs

@@ -205,7 +205,7 @@ protected override void OnInitializeConfig(ScriptHostConfiguration config)
             hostConfig.DashboardConnectionString = null;
         }
 
-        protected override void OnHostCreated()
+        protected override void OnHostInitialized()
         {
             if (InStandbyMode)
             {
@@ -214,7 +214,7 @@ protected override void OnHostCreated()
 
             InitializeHttp();
 
-            base.OnHostCreated();
+            base.OnHostInitialized();
         }
 
         protected override void OnHostStarted()

src/WebJobs.Script/Host/ScriptHost.cs

@@ -108,6 +108,10 @@ protected internal ScriptHost(IScriptHostEnvironment environment,
             _proxyClient = proxyClient;
         }
 
+        public event EventHandler HostInitialized;
+
+        public event EventHandler HostStarted;
+
         public event EventHandler IsPrimaryChanged;
 
         public string InstanceId
@@ -1897,6 +1901,20 @@ internal static string GetAssemblyFileVersion(Assembly assembly)
             return fileVersionAttr?.Version ?? "Unknown";
         }
 
+        protected override void OnHostInitialized()
+        {
+            HostInitialized?.Invoke(this, EventArgs.Empty);
+
+            base.OnHostInitialized();
+         }
+
+        protected override void OnHostStarted()
+        {
+            HostStarted?.Invoke(this, EventArgs.Empty);
+
+            base.OnHostStarted();
+        }
+
         protected override void Dispose(bool disposing)
         {
             if (disposing)

src/WebJobs.Script/Host/ScriptHostManager.cs

@@ -145,7 +145,7 @@ public virtual ScriptHost Instance
         /// <returns>True if the host can accept invoke requests, false otherwise.</returns>
         public bool CanInvoke()
         {
-            return State == ScriptHostState.Created || State == ScriptHostState.Running;
+            return State == ScriptHostState.Initialized || State == ScriptHostState.Running;
         }
 
         public void RunAndBlock(CancellationToken cancellationToken = default(CancellationToken))
@@ -164,7 +164,7 @@ public bool CanInvoke()
                         State = ScriptHostState.Default;
                     }
 
-                    OnHostStarting();
+                    OnCreatingHost();
 
                     // Create a new host config, but keep the host id from existing one
                     _config.HostConfig = new JobHostConfiguration(_settingsManager.Configuration)
@@ -173,6 +173,8 @@ public bool CanInvoke()
                     };
                     OnInitializeConfig(_config);
                     newInstance = _scriptHostFactory.Create(_environment, EventManager, _settingsManager, _config, _loggerFactoryBuilder);
+                    newInstance.HostInitialized += OnHostInitialized;
+                    newInstance.HostStarted += OnHostStarted;
                     _traceWriter = newInstance.TraceWriter;
                     _logger = newInstance.Logger;
 
@@ -183,8 +185,6 @@ public bool CanInvoke()
                         _hostStartCount++;
                     }
 
-                    OnHostCreated();
-
                     string extensionVersion = _settingsManager.GetSetting(EnvironmentSettingNames.FunctionsExtensionVersion);
                     string hostId = newInstance.ScriptConfig.HostConfig.HostId;
                     string message = $"Starting Host (HostId={hostId}, Version={ScriptHost.Version}, ProcessId={Process.GetCurrentProcess().Id}, Debug={newInstance.InDebugMode}, ConsecutiveErrors={consecutiveErrorCount}, StartupCount={_hostStartCount}, FunctionsExtensionVersion={extensionVersion})";
@@ -196,11 +196,6 @@ public bool CanInvoke()
                     // log any function initialization errors
                     LogErrors(newInstance);
 
-                    OnHostStarted();
-
-                    // only after ALL initialization is complete do we set the
-                    // state to Running
-                    State = ScriptHostState.Running;
                     LastError = null;
                     consecutiveErrorCount = 0;
                     _restartDelayTokenSource = null;
@@ -262,6 +257,16 @@ public bool CanInvoke()
             while (!_stopped && !cancellationToken.IsCancellationRequested);
         }
 
+        private void OnHostInitialized(object sender, EventArgs e)
+        {
+            OnHostInitialized();
+        }
+
+        private void OnHostStarted(object sender, EventArgs e)
+        {
+            OnHostStarted();
+        }
+
         private Task CreateRestartBackoffDelay(int consecutiveErrorCount)
         {
             _restartDelayTokenSource = new CancellationTokenSource();
@@ -296,6 +301,9 @@ private static void LogErrors(ScriptHost host)
         /// <param name="forceStop">Forces the call to stop and dispose of the instance, even if it isn't present in the live instances collection.</param>
         private async Task Orphan(ScriptHost instance, bool forceStop = false)
         {
+            instance.HostInitialized -= OnHostInitialized;
+            instance.HostStarted -= OnHostStarted;
+
             lock (_liveInstances)
             {
                 bool removed = _liveInstances.Remove(instance);
@@ -397,20 +405,28 @@ protected virtual void OnInitializeConfig(ScriptHostConfiguration config)
             }
         }
 
-        protected virtual void OnHostCreated()
+        protected virtual void OnHostInitialized()
         {
-            State = ScriptHostState.Created;
+            State = ScriptHostState.Initialized;
         }
 
-        protected virtual void OnHostStarting()
+        /// <summary>
+        /// Called after the host has been started.
+        /// </summary>
+        protected virtual void OnHostStarted()
         {
-            IsHostHealthy(throwWhenUnhealthy: true);
+            var metricsLogger = _config.HostConfig.GetService<IMetricsLogger>();
+            metricsLogger.LogEvent(new HostStarted(Instance));
+
+            State = ScriptHostState.Running;
         }
 
-    protected virtual void OnHostStarted()
+        /// <summary>
+        /// Called immediately before we begin creating the host.
+        /// </summary>
+        protected virtual void OnCreatingHost()
         {
-            var metricsLogger = _config.HostConfig.GetService<IMetricsLogger>();
-            metricsLogger.LogEvent(new HostStarted(Instance));
+            IsHostHealthy(throwWhenUnhealthy: true);
         }
 
         public void Dispose()

src/WebJobs.Script/Host/ScriptHostState.cs

@@ -11,10 +11,11 @@ public enum ScriptHostState
         Default,
 
         /// <summary>
-        /// The host has been created and can accept direct function
-        /// invocations, but listeners are not yet started.
+        /// The host has been fully initialized and can accept direct function
+        /// invocations. All functions have been indexed. Listeners may not yet
+        /// be not yet running.
         /// </summary>
-        Created,
+        Initialized,
 
         /// <summary>
         /// The host is fully running.

src/WebJobs.Script/WebJobs.Script.csproj

@@ -25,7 +25,7 @@
     <PackageReference Include="Microsoft.Azure.Functions.NodeJsWorker" Version="1.0.0-beta1-10022">
       <PrivateAssets>None</PrivateAssets>
     </PackageReference>
-    <PackageReference Include="Microsoft.Azure.WebJobs" Version="3.0.0-beta3-11077" />
+    <PackageReference Include="Microsoft.Azure.WebJobs" Version="3.0.0-beta3-11084" />
     <PackageReference Include="Microsoft.Azure.WebJobs.Extensions" Version="3.0.0-beta3" />
     <PackageReference Include="Microsoft.Azure.WebJobs.Extensions.Http" Version="3.0.0-beta3">
       <NoWarn>NU1701</NoWarn>