🐛 Connecting to RavenDB database can timeout with DatabaseLoadTimeoutException. Will retry until the cancellationtoken is set (#3918)

Author: ramonsmits

src/ServiceControl.Audit.Persistence.RavenDb5/RavenDbEmbeddedPersistenceLifecycle.cs

@@ -3,7 +3,9 @@
     using System;
     using System.Threading;
     using System.Threading.Tasks;
+    using NServiceBus.Logging;
     using Raven.Client.Documents;
+    using Raven.Client.Exceptions.Database;
 
     class RavenDbEmbeddedPersistenceLifecycle : IRavenDbPersistenceLifecycle
     {
@@ -26,7 +28,21 @@ public async Task Start(CancellationToken cancellationToken)
         {
             database = EmbeddedDatabase.Start(databaseConfiguration);
 
-            documentStore = await database.Connect(cancellationToken).ConfigureAwait(false);
+            while (true)
+            {
+                cancellationToken.ThrowIfCancellationRequested();
+
+                try
+                {
+                    documentStore = await database.Connect(cancellationToken).ConfigureAwait(false);
+                    return;
+                }
+                catch (DatabaseLoadTimeoutException e)
+                {
+                    Log.Warn("Could not connect to database. Retrying in 500ms...", e);
+                    await Task.Delay(500, cancellationToken).ConfigureAwait(false);
+                }
+            }
         }
 
         public Task Stop(CancellationToken cancellationToken)
@@ -39,7 +55,7 @@ public Task Stop(CancellationToken cancellationToken)
 
         IDocumentStore documentStore;
         EmbeddedDatabase database;
-
+        static readonly ILog Log = LogManager.GetLogger(typeof(RavenDbEmbeddedPersistenceLifecycle));
         readonly DatabaseConfiguration databaseConfiguration;
     }
-}
+}

src/ServiceControl.Audit.Persistence.Tests.RavenDb5/SharedEmbeddedServer.cs

@@ -6,6 +6,7 @@
     using System.Net.NetworkInformation;
     using System.Threading;
     using System.Threading.Tasks;
+    using NServiceBus.Logging;
     using NUnit.Framework;
     using ServiceControl.Audit.Persistence.RavenDb;
 
@@ -41,8 +42,9 @@ public static async Task<EmbeddedDatabase> GetInstance(CancellationToken cancell
 
                         return embeddedDatabase;
                     }
-                    catch (Exception)
+                    catch (Exception e)
                     {
+                        Log.Warn("Could not connect to database. Retrying in 500ms...", e);
                         await Task.Delay(500, cancellationToken).ConfigureAwait(false);
                     }
                 }
@@ -87,5 +89,6 @@ static int FindAvailablePort(int startPort)
 
         static EmbeddedDatabase embeddedDatabase;
         static SemaphoreSlim semaphoreSlim = new SemaphoreSlim(1, 1);
+        static readonly ILog Log = LogManager.GetLogger(typeof(SharedEmbeddedServer));
     }
 }