Merge branch 'fix-watchdog-stop' into spike-ravendb-termination

Author: ramonsmits

src/ServiceControl.Audit/Auditing/AuditIngestion.cs

@@ -68,7 +68,7 @@ Task OnCriticalError(string failure, Exception exception)
             return watchdog.OnFailure(failure);
         }
 
-        async Task EnsureStarted(CancellationToken cancellationToken = default)
+        async Task EnsureStarted(CancellationToken cancellationToken)
         {
             try
             {
@@ -177,7 +177,7 @@ async Task StopAndTeardownInfrastructure(CancellationToken cancellationToken)
             }
         }
 
-        async Task EnsureStopped(CancellationToken cancellationToken = default)
+        async Task EnsureStopped(CancellationToken cancellationToken)
         {
             try
             {
@@ -231,7 +231,7 @@ async Task OnMessage(MessageContext messageContext, CancellationToken cancellati
 
         public override async Task StartAsync(CancellationToken cancellationToken)
         {
-            await watchdog.Start(() => applicationLifetime.StopApplication());
+            await watchdog.Start(() => applicationLifetime.StopApplication(), cancellationToken);
             await base.StartAsync(cancellationToken);
         }
 

src/ServiceControl.Infrastructure.Tests/WatchdogTests.cs

@@ -1,6 +1,7 @@
 namespace ServiceControl.Infrastructure.Tests
 {
     using System;
+    using System.Threading;
     using System.Threading.Tasks;
     using NServiceBus.Logging;
     using NUnit.Framework;
@@ -27,7 +28,7 @@ public async Task It_shuts_down_gracefully()
                 return Task.CompletedTask;
             }, x => { }, () => { }, TimeSpan.FromSeconds(1), log);
 
-            await dog.Start(() => { });
+            await dog.Start(() => { }, CancellationToken.None);
 
             await started.Task;
 
@@ -48,7 +49,7 @@ public async Task When_stop_fails_it_reports_the_failure()
                 return Task.CompletedTask;
             }, token => throw new Exception("Simulated"), x => lastFailure = x, () => lastFailure = null, TimeSpan.FromSeconds(1), log);
 
-            await dog.Start(() => { });
+            await dog.Start(() => { }, CancellationToken.None);
 
             await started.Task;
 
@@ -83,7 +84,7 @@ public async Task On_failure_triggers_stopping()
                 return Task.CompletedTask;
             }, x => { }, () => { }, TimeSpan.FromSeconds(1), log);
 
-            await dog.Start(() => { });
+            await dog.Start(() => { }, CancellationToken.None);
 
             await started.Task;
 
@@ -125,7 +126,7 @@ public async Task When_first_start_attempt_works_it_recovers_from_further_errors
                 return Task.CompletedTask;
             }, token => Task.CompletedTask, x => lastFailure = x, () => lastFailure = null, TimeSpan.FromSeconds(1), log);
 
-            await dog.Start(() => { });
+            await dog.Start(() => { }, CancellationToken.None);
 
             await recoveredFromError.Task;
 
@@ -144,7 +145,7 @@ public async Task When_first_start_attempt_fails_onFailedOnStartup_is_called()
 
             var dog = new Watchdog("test process", token => throw new Exception("Simulated"), token => Task.CompletedTask, x => lastFailure = x, () => lastFailure = null, TimeSpan.FromSeconds(1), log);
 
-            await dog.Start(() => { onStartupFailureCalled.SetResult(true); });
+            await dog.Start(() => { onStartupFailureCalled.SetResult(true); }, CancellationToken.None);
 
             await onStartupFailureCalled.Task;
 

src/ServiceControl.Infrastructure/Watchdog.cs

@@ -41,7 +41,7 @@ public Task OnFailure(string failure)
             return ensureStopped(shutdownTokenSource.Token);
         }
 
-        public Task Start(Action onFailedOnStartup)
+        public Task Start(Action onFailedOnStartup, CancellationToken cancellationToken)
         {
             watchdog = Task.Run(async () =>
             {
@@ -53,8 +53,12 @@ public Task Start(Action onFailedOnStartup)
                 {
                     try
                     {
+                        // Host builder start is launching the loop. The watch dog loop task runs in isolation
+                        // We want the start not to run to infinity. An NServiceBus endpoint should easily
+                        // start within 15 seconds.
+                        const int MaxStartDurationMs = 15000;
                         using var cancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(shutdownTokenSource.Token);
-                        cancellationTokenSource.CancelAfter(15000);
+                        cancellationTokenSource.CancelAfter(MaxStartDurationMs);
 
                         log.Debug($"Ensuring {taskName} is running");
                         await ensureStarted(cancellationTokenSource.Token).ConfigureAwait(false);
@@ -88,7 +92,8 @@ public Task Start(Action onFailedOnStartup)
                         //Ignore, no need to log cancellation of delay
                     }
                 }
-            });
+            }, cancellationToken);
+
             return Task.CompletedTask;
         }
 

src/ServiceControl/Operations/ErrorIngestion.cs

@@ -72,7 +72,7 @@ public ErrorIngestion(
 
         public override async Task StartAsync(CancellationToken cancellationToken)
         {
-            await watchdog.Start(() => applicationLifetime.StopApplication());
+            await watchdog.Start(() => applicationLifetime.StopApplication(), cancellationToken);
             await base.StartAsync(cancellationToken);
         }