Chance of getting 409 while renewing the primary host lock. Fixes #1864

Author: alrod

src/WebJobs.Script/Host/PrimaryHostCoordinator.cs

@@ -36,6 +36,9 @@ internal sealed class PrimaryHostCoordinator : IDisposable
 
         private IDistributedLockManager _lockManager;
 
+        private static PrimaryHostCoordinator latestCreatedInstance;
+        private static object syncLock = new object();
+
         internal PrimaryHostCoordinator(IDistributedLockManager lockManager, TimeSpan leaseTimeout, string hostId, string instanceId, TraceWriter traceWriter,
             ILoggerFactory loggerFactory, TimeSpan? renewalInterval = null)
         {
@@ -100,13 +103,22 @@ public static PrimaryHostCoordinator Create(
                 throw new ArgumentOutOfRangeException(nameof(leaseTimeout), $"The {nameof(leaseTimeout)} should be between 15 and 60 seconds");
             }
 
-            var manager = new PrimaryHostCoordinator(lockManager, leaseTimeout, hostId, instanceId, traceWriter, loggerFactory, renewalInterval);
-            return manager;
+            lock (syncLock)
+            {
+                // Dispose previously created instance to ensure releasing a lease before new instance is created.
+                if (latestCreatedInstance != null)
+                {
+                    latestCreatedInstance.Dispose();
+                }
+
+                latestCreatedInstance = new PrimaryHostCoordinator(lockManager, leaseTimeout, hostId, instanceId, traceWriter, loggerFactory, renewalInterval);
+            }
+            return latestCreatedInstance;
         }
 
         private void ProcessLeaseTimerTick(object state)
         {
-            if (_processingLease)
+            if (_processingLease && !_disposed)
             {
                 return;
             }

test/WebJobs.Script.Tests.Integration/PrimaryHostCoordinatorTests.cs

@@ -229,8 +229,8 @@ public async Task DifferentHosts_UsingSameStorageAccount_CanObtainLease()
             var traceWriter = new TestTraceWriter(TraceLevel.Verbose);
 
             var lockManager = CreateLockManager();
-            using (var manager1 = PrimaryHostCoordinator.Create(lockManager, TimeSpan.FromSeconds(15), hostId1, instanceId, traceWriter, null))
-            using (var manager2 = PrimaryHostCoordinator.Create(lockManager, TimeSpan.FromSeconds(15), hostId2, instanceId, traceWriter, null))
+            using (var manager1 = new PrimaryHostCoordinator(lockManager, TimeSpan.FromSeconds(15), hostId1, instanceId, traceWriter, null))
+            using (var manager2 = new PrimaryHostCoordinator(lockManager, TimeSpan.FromSeconds(15), hostId2, instanceId, traceWriter, null))
             {
                 Task manager1Check = TestHelpers.Await(() => manager1.HasLease);
                 Task manager2Check = TestHelpers.Await(() => manager2.HasLease);
@@ -241,6 +241,34 @@ public async Task DifferentHosts_UsingSameStorageAccount_CanObtainLease()
             await Task.WhenAll(ClearLeaseBlob(hostId1), ClearLeaseBlob(hostId2));
         }
 
+        [Fact]
+        public async Task Create_LastCreatedInstanceAquiresLease()
+        {
+            string hostId = Guid.NewGuid().ToString();
+            string instanceId = Guid.NewGuid().ToString();
+            string connectionString = AmbientConnectionStringProvider.Instance.GetConnectionString(ConnectionStringNames.Storage);
+            var traceWriter = new TestTraceWriter(TraceLevel.Verbose);
+
+            var lockManager = CreateLockManager();
+            using (var manager = PrimaryHostCoordinator.Create(lockManager, TimeSpan.FromSeconds(15), hostId, instanceId, traceWriter, null))
+            {
+                Task managerCheck = TestHelpers.Await(() => manager.HasLease);
+                await Task.WhenAll(managerCheck);
+            }
+
+            var manager1 = PrimaryHostCoordinator.Create(lockManager, TimeSpan.FromSeconds(15), hostId, instanceId, traceWriter, null);
+            Task manager1Check = TestHelpers.Await(() => manager1.HasLease);
+            await Task.WhenAll(manager1Check);
+
+            var manager2 = PrimaryHostCoordinator.Create(lockManager, TimeSpan.FromSeconds(15), hostId, instanceId, traceWriter, null);
+            Task manager2Check = TestHelpers.Await(() => manager2.HasLease);
+            await Task.WhenAll(manager2Check);
+            Assert.True(traceWriter.Traces[traceWriter.Traces.Count() - 2].Message.Contains($"Host instance '{instanceId}' released lock lease"));
+            Assert.True(traceWriter.Traces.Last().Message.Contains($"Host lock lease acquired by instance ID '{instanceId}'"));
+
+            await Task.WhenAll(ClearLeaseBlob(hostId));
+        }
+
         private static async Task<ICloudBlob> GetLockBlobAsync(string accountConnectionString, string hostId)
         {
             CloudStorageAccount account = CloudStorageAccount.Parse(accountConnectionString);