Teardown Raven.Embedded.EmbeddedServer as part of stop sequence so that we kill the instance if needed as adhering to cancellation is not possible in host builder root container dispose.

Author: ramonsmits

src/ServiceControl.Audit.Persistence.RavenDB/IRavenPersistenceLifecycle.cs

@@ -6,5 +6,6 @@
     interface IRavenPersistenceLifecycle
     {
         Task Initialize(CancellationToken cancellationToken = default);
+        Task Stop(CancellationToken cancellationToken);
     }
 }

src/ServiceControl.Audit.Persistence.RavenDB/RavenEmbeddedPersistenceLifecycle.cs

@@ -72,6 +72,8 @@ public async Task Initialize(CancellationToken cancellationToken = default)
             }
         }
 
+        public Task Stop(CancellationToken cancellationToken) => database!.Stop(cancellationToken);
+
         public void Dispose()
         {
             documentStore?.Dispose();

src/ServiceControl.Audit.Persistence.RavenDB/RavenExternalPersistenceLifecycle.cs

@@ -64,6 +64,8 @@ public async Task Initialize(CancellationToken cancellationToken = default)
             }
         }
 
+        public Task Stop(CancellationToken cancellationToken) => Task.CompletedTask; // We are not stopping an external instance
+
         public void Dispose() => documentStore?.Dispose();
 
         IDocumentStore? documentStore;

src/ServiceControl.Audit.Persistence.RavenDB/RavenPersistenceLifecycleHostedService.cs

@@ -8,6 +8,6 @@ sealed class RavenPersistenceLifecycleHostedService(IRavenPersistenceLifecycle l
     {
         public Task StartAsync(CancellationToken cancellationToken) => lifecycle.Initialize(cancellationToken);
 
-        public Task StopAsync(CancellationToken cancellationToken) => Task.CompletedTask;
+        public Task StopAsync(CancellationToken cancellationToken) => lifecycle.Stop(cancellationToken);
     }
 }

src/ServiceControl.RavenDB/EmbeddedDatabase.cs

@@ -53,7 +53,7 @@ public static EmbeddedDatabase Start(EmbeddedDatabaseConfiguration databaseConfi
             var nugetPackagesPath = Path.Combine(databaseConfiguration.DbPath, "Packages", "NuGet");
 
             Logger.InfoFormat("Loading RavenDB license from {0}", licenseFileNameAndServerDirectory.LicenseFileName);
-            var serverOptions = new ServerOptions
+            serverOptions = new ServerOptions
             {
                 CommandLineArgs =
                 [
@@ -162,23 +162,70 @@ public async Task<IDocumentStore> Connect(CancellationToken cancellationToken)
 
         public async Task DeleteDatabase(string dbName)
         {
-            using var store = await EmbeddedServer.Instance.GetDocumentStoreAsync(new DatabaseOptions(dbName) { SkipCreatingDatabase = true });
+            using var store = await EmbeddedServer.Instance.GetDocumentStoreAsync(new DatabaseOptions(dbName)
+            {
+                SkipCreatingDatabase = true
+            });
             await store.Maintenance.Server.SendAsync(new DeleteDatabasesOperation(dbName, true));
         }
 
+        public async Task Stop(CancellationToken cancellationToken)
+        {
+            Logger.Debug("Stopping RavenDB server");
+            EmbeddedServer.Instance.ServerProcessExited -= OnServerProcessExited;
+
+            await shutdownTokenSource.CancelAsync();
+
+            // TODO: await EmbeddedServer.Instance.StopAsync(cancellationToken);
+
+            var processId = await EmbeddedServer.Instance.GetServerProcessIdAsync(cancellationToken);
+
+            var waitForCancellationTask = Task.Delay(Timeout.Infinite, cancellationToken);
+            var firstTask = await Task.WhenAny(
+                Task.Run(() => EmbeddedServer.Instance.Dispose(), cancellationToken),
+                waitForCancellationTask
+            );
+
+            if (firstTask == waitForCancellationTask)
+            {
+                try
+                {
+                    Logger.Warn("Killing RavenDB child process because host cancelled");
+                    var ravenChildProcess = Process.GetProcessById(processId);
+                    ravenChildProcess.Kill(entireProcessTree: true);
+                }
+                catch (Exception e)
+                {
+                    Logger.Error("Killing RavenDB child process failed", e);
+                }
+            }
+
+            EmbeddedServer.Instance = null!;
+            Logger.Debug("Stopped RavenDB server");
+        }
+
         public void Dispose()
         {
             if (disposed)
             {
                 return;
             }
 
-            EmbeddedServer.Instance.ServerProcessExited -= OnServerProcessExited;
+            if (EmbeddedServer.Instance != null)
+            {
+                EmbeddedServer.Instance.ServerProcessExited -= OnServerProcessExited;
+            }
 
             shutdownTokenSource.Cancel();
-            Logger.Debug("Disposing RavenDB server");
-            EmbeddedServer.Instance.Dispose();
-            Logger.Debug("Dispose RavenDB server");
+
+            if (EmbeddedServer.Instance != null)
+            {
+                serverOptions.GracefulShutdownTimeout = TimeSpan.Zero;
+                Logger.Debug("Disposing RavenDB server");
+                EmbeddedServer.Instance.Dispose();
+                Logger.Debug("Disposed RavenDB server");
+            }
+
             shutdownTokenSource.Dispose();
             applicationStoppingRegistration.Dispose();
 
@@ -262,6 +309,7 @@ static long DirSize(DirectoryInfo d)
         readonly EmbeddedDatabaseConfiguration configuration;
         readonly CancellationToken shutdownCancellationToken;
         readonly CancellationTokenRegistration applicationStoppingRegistration;
+        static ServerOptions serverOptions;
 
         static TimeSpan delayBetweenRestarts = TimeSpan.FromSeconds(60);
         static readonly ILog Logger = LogManager.GetLogger<EmbeddedDatabase>();