Ensure Instance specialized on assignment failure

Author: mathewc

src/WebJobs.Script.WebHost/Management/InstanceManager.cs

@@ -103,11 +103,6 @@ private async Task Assign(HostAssignmentContext assignmentContext)
                 // first make all environment and file system changes required for
                 // the host to be specialized
                 await ApplyContext(assignmentContext);
-
-                // all assignment settings/files have been applied so we can flip
-                // the switch now on specialization
-                _logger.LogInformation("Triggering specialization");
-                _webHostEnvironment.FlagAsSpecializedAndReady();
             }
             catch (Exception ex)
             {
@@ -116,6 +111,13 @@ private async Task Assign(HostAssignmentContext assignmentContext)
             }
             finally
             {
+                // all assignment settings/files have been applied so we can flip
+                // the switch now on specialization
+                // even if there are failures applying context above, we want to
+                // leave placeholder mode
+                _logger.LogInformation("Triggering specialization");
+                _webHostEnvironment.FlagAsSpecializedAndReady();
+
                 _webHostEnvironment.ResumeRequests();
             }
         }
@@ -173,5 +175,11 @@ public IDictionary<string, string> GetInstanceInfo()
                 { "WEBSITE_NODE_DEFAULT_VERSION", "8.5.0" }
             };
         }
+
+        // for testing
+        internal static void Reset()
+        {
+            _assignmentContext = null;
+        }
     }
 }

test/WebJobs.Script.Tests.Integration/Management/InstanceManagerTests.cs

@@ -21,7 +21,7 @@ namespace Microsoft.Azure.WebJobs.Script.Tests.Managment
     public class InstanceManagerTests : IDisposable
     {
         private readonly TestLoggerProvider _loggerProvider;
-        private readonly TestEnvironment _environment;
+        private readonly TestEnvironmentEx _environment;
         private readonly ScriptWebHostEnvironment _scriptWebEnvironment;
         private readonly InstanceManager _instanceManager;
         private readonly HttpClient _httpClient;
@@ -34,11 +34,13 @@ public InstanceManagerTests()
             var loggerFactory = new LoggerFactory();
             loggerFactory.AddProvider(_loggerProvider);
 
-            _environment = new TestEnvironment();
+            _environment = new TestEnvironmentEx();
             _scriptWebEnvironment = new ScriptWebHostEnvironment(_environment);
 
             var optionsFactory = new TestOptionsFactory<ScriptApplicationHostOptions>(new ScriptApplicationHostOptions());
             _instanceManager = new InstanceManager(optionsFactory, _httpClient, _scriptWebEnvironment, _environment, loggerFactory.CreateLogger<InstanceManager>());
+
+            InstanceManager.Reset();
         }
 
         [Fact]
@@ -86,6 +88,29 @@ public async Task StartAssignment_AppliesAssignmentContext()
                 p => Assert.StartsWith("Assign called while host is not in placeholder mode", p));
         }
 
+        [Fact]
+        public async Task StartAssignment_Failure_ExitsPlaceholderMode()
+        {
+            _environment.SetEnvironmentVariable(EnvironmentSettingNames.AzureWebsitePlaceholderMode, "1");
+            var context = new HostAssignmentContext
+            {
+                Environment = new Dictionary<string, string>
+                {
+                    // force the assignment to fail
+                    { "throw", "test" }
+                }
+            };
+            bool result = _instanceManager.StartAssignment(context);
+            Assert.True(result);
+            Assert.True(_scriptWebEnvironment.InStandbyMode);
+
+            await TestHelpers.Await(() => !_scriptWebEnvironment.InStandbyMode, timeout: 5000);
+
+            var error = _loggerProvider.GetAllLogMessages().Where(p => p.Level == LogLevel.Error).First();
+            Assert.Equal("Assign failed", error.FormattedMessage);
+            Assert.Equal("Kaboom!", error.Exception.Message);
+        }
+
         [Fact]
         public void StartAssignment_ReturnsFalse_WhenNotInStandbyMode()
         {
@@ -145,5 +170,17 @@ public void Dispose()
         {
             Environment.SetEnvironmentVariable(EnvironmentSettingNames.AzureWebsitePlaceholderMode, null);
         }
+
+        private class TestEnvironmentEx : TestEnvironment
+        {
+            public override void SetEnvironmentVariable(string name, string value)
+            {
+                if (name == "throw")
+                {
+                    throw new InvalidOperationException("Kaboom!");
+                }
+                base.SetEnvironmentVariable(name, value);
+            }
+        }
     }
 }

test/WebJobs.Script.Tests.Shared/TestEnvironment.cs

@@ -35,7 +35,7 @@ public string GetEnvironmentVariable(string name)
             return result;
         }
 
-        public void SetEnvironmentVariable(string name, string value)
+        public virtual void SetEnvironmentVariable(string name, string value)
         {
             if (value == null && _variables.ContainsKey(name))
             {