Handle race conditions

Author: andreasohlund

src/ServiceControl.Audit.Persistence.Tests.RavenDB/IndexSetupTests.cs

@@ -40,9 +40,6 @@ public async Task Free_text_search_index_can_be_opted_out_from()
     {
         await DatabaseSetup.CreateIndexes(configuration.DocumentStore, false, CancellationToken.None);
 
-        //TODO: find a better way
-        await Task.Delay(1000);
-
         var freeTextIndex = await configuration.DocumentStore.Maintenance.SendAsync(new GetIndexOperation(DatabaseSetup.MessagesViewIndexWithFulltextSearchName));
         var nonFreeTextIndex = await configuration.DocumentStore.Maintenance.SendAsync(new GetIndexOperation(DatabaseSetup.MessagesViewIndexName));
 
@@ -55,48 +52,35 @@ public async Task Indexes_should_be_reset_on_setup()
     {
         var index = new MessagesViewIndexWithFullTextSearch { Configuration = { ["Indexing.Static.SearchEngineType"] = SearchEngineType.Lucene.ToString() } };
 
-        await IndexCreation.CreateIndexesAsync([index], configuration.DocumentStore);
-
-        //TODO: find a better way
-        await Task.Delay(1000);
-
-        var indexStatsBefore = await configuration.DocumentStore.Maintenance.SendAsync(new GetIndexStatisticsOperation(index.IndexName));
+        var indexWithCustomConfigStats = await UpdateIndex(index);
 
-        Assert.That(indexStatsBefore.SearchEngineType, Is.EqualTo(SearchEngineType.Lucene));
+        Assert.That(indexWithCustomConfigStats.SearchEngineType, Is.EqualTo(SearchEngineType.Lucene));
 
         await DatabaseSetup.CreateIndexes(configuration.DocumentStore, true, CancellationToken.None);
 
-        //TODO: find a better way
-        await Task.Delay(1000);
+        WaitForIndexDefinitionUpdate(indexWithCustomConfigStats);
 
-        var indexStatsAfter = await configuration.DocumentStore.Maintenance.SendAsync(new GetIndexStatisticsOperation(index.IndexName));
-        Assert.That(indexStatsAfter.SearchEngineType, Is.EqualTo(SearchEngineType.Corax));
+        var indexAfterResetStats = await configuration.DocumentStore.Maintenance.SendAsync(new GetIndexStatisticsOperation(index.IndexName));
+
+        Assert.That(indexAfterResetStats.SearchEngineType, Is.EqualTo(SearchEngineType.Corax));
     }
 
     [Test]
     public async Task Indexes_should_not_be_reset_on_setup_when_locked_as_ignore()
     {
-        var index = new MessagesViewIndexWithFullTextSearch { Configuration = { ["Indexing.Static.SearchEngineType"] = SearchEngineType.Lucene.ToString() } };
-
-        await IndexCreation.CreateIndexesAsync([index], configuration.DocumentStore);
-
-        await configuration.DocumentStore.Maintenance.SendAsync(new SetIndexesLockOperation(new SetIndexesLockOperation.Parameters
+        var index = new MessagesViewIndexWithFullTextSearch
         {
-            IndexNames = [index.IndexName],
-            Mode = IndexLockMode.LockedIgnore
-        }));
-
-        //TODO: find a better way
-        await Task.Delay(1000);
+            Configuration = { ["Indexing.Static.SearchEngineType"] = SearchEngineType.Lucene.ToString() },
+            LockMode = IndexLockMode.LockedIgnore
+        };
 
-        var indexStatsBefore = await configuration.DocumentStore.Maintenance.SendAsync(new GetIndexStatisticsOperation(index.IndexName));
+        var indexStatsBefore = await UpdateIndex(index);
 
         Assert.That(indexStatsBefore.SearchEngineType, Is.EqualTo(SearchEngineType.Lucene));
 
-
         await DatabaseSetup.CreateIndexes(configuration.DocumentStore, true, CancellationToken.None);
 
-        //TODO: find a better way
+        // raven will ignore the update since index was locked, so best we can do is wait a bit and check that settings hasn't changed
         await Task.Delay(1000);
 
         var indexStatsAfter = await configuration.DocumentStore.Maintenance.SendAsync(new GetIndexStatisticsOperation(index.IndexName));
@@ -106,16 +90,41 @@ await configuration.DocumentStore.Maintenance.SendAsync(new SetIndexesLockOperat
     [Test]
     public async Task Indexes_should_not_be_reset_on_setup_when_locked_as_error()
     {
-        var index = new MessagesViewIndexWithFullTextSearch { Configuration = { ["Indexing.Static.SearchEngineType"] = SearchEngineType.Lucene.ToString() } };
+        var index = new MessagesViewIndexWithFullTextSearch
+        {
+            Configuration = { ["Indexing.Static.SearchEngineType"] = SearchEngineType.Lucene.ToString() },
+            LockMode = IndexLockMode.LockedError
+        };
+
+        await UpdateIndex(index);
+
+        Assert.ThrowsAsync<IndexCreationException>(async () => await DatabaseSetup.CreateIndexes(configuration.DocumentStore, true, CancellationToken.None));
+    }
+
+    async Task<IndexStats> UpdateIndex(IAbstractIndexCreationTask index)
+    {
+        var statsBefore = await configuration.DocumentStore.Maintenance.SendAsync(new GetIndexStatisticsOperation(index.IndexName));
 
         await IndexCreation.CreateIndexesAsync([index], configuration.DocumentStore);
 
-        await configuration.DocumentStore.Maintenance.SendAsync(new SetIndexesLockOperation(new SetIndexesLockOperation.Parameters
-        {
-            IndexNames = [index.IndexName],
-            Mode = IndexLockMode.LockedError
-        }));
+        WaitForIndexDefinitionUpdate(statsBefore);
 
-        Assert.ThrowsAsync<IndexCreationException>(async () => await DatabaseSetup.CreateIndexes(configuration.DocumentStore, true, CancellationToken.None));
+        return await configuration.DocumentStore.Maintenance.SendAsync(new GetIndexStatisticsOperation(index.IndexName));
+    }
+
+    void WaitForIndexDefinitionUpdate(IndexStats indexStats)
+    {
+        Assert.That(SpinWait.SpinUntil(() =>
+        {
+            try
+            {
+                return configuration.DocumentStore.Maintenance.Send(new GetIndexStatisticsOperation(indexStats.Name)).CreatedTimestamp > indexStats.CreatedTimestamp;
+            }
+            catch (OperationCanceledException)
+            {
+                // keep going since we can get this if we query right when the update happens
+                return false;
+            }
+        }, TimeSpan.FromSeconds(10)), Is.True);
     }
 }