Skip to content

Commit 0e39e56

Browse files
andreasohlundandreasohlund
authored
Add tests to make sure that indexes being locked prevents index resets on instance restarts (#4849)
* Initial cleanup * Add tests to verify default behavior * Add test to check that indexes are reset on setup * Add lock mode tests * Add test for non free text search * Handle race conditions * Use async wait * Add test cancellation
1 parent bfecbdb commit 0e39e56

File tree

4 files changed

+314
-165
lines changed

4 files changed

+314
-165
lines changed

‎src/ServiceControl.Audit.Persistence.RavenDB/DatabaseSetup.cs‎

Lines changed: 99 additions & 93 deletions
Original file line numberDiff line numberDiff line change
@@ -1,120 +1,126 @@
1-
namespace ServiceControl.Audit.Persistence.RavenDB
1+
namespace ServiceControl.Audit.Persistence.RavenDB;
2+
3+
using System;
4+
using System.Collections.Generic;
5+
using System.Threading;
6+
using System.Threading.Tasks;
7+
using Raven.Client.Documents;
8+
using Raven.Client.Documents.Indexes;
9+
using Raven.Client.Documents.Operations.Expiration;
10+
using Raven.Client.Documents.Operations.Indexes;
11+
using Raven.Client.Exceptions;
12+
using Raven.Client.ServerWide;
13+
using Raven.Client.ServerWide.Operations;
14+
using Raven.Client.ServerWide.Operations.Configuration;
15+
using Indexes;
16+
using SagaAudit;
17+
18+
class DatabaseSetup(DatabaseConfiguration configuration)
219
{
3-
using System;
4-
using System.Collections.Generic;
5-
using System.Threading;
6-
using System.Threading.Tasks;
7-
using Raven.Client.Documents;
8-
using Raven.Client.Documents.Indexes;
9-
using Raven.Client.Documents.Operations.Expiration;
10-
using Raven.Client.Documents.Operations.Indexes;
11-
using Raven.Client.Exceptions;
12-
using Raven.Client.ServerWide;
13-
using Raven.Client.ServerWide.Operations;
14-
using Raven.Client.ServerWide.Operations.Configuration;
15-
using ServiceControl.Audit.Persistence.RavenDB.Indexes;
16-
using ServiceControl.SagaAudit;
17-
18-
class DatabaseSetup(DatabaseConfiguration configuration)
20+
public async Task Execute(IDocumentStore documentStore, CancellationToken cancellationToken)
1921
{
20-
public async Task Execute(IDocumentStore documentStore, CancellationToken cancellationToken)
21-
{
22-
await CreateDatabase(documentStore, configuration.Name, cancellationToken);
23-
await UpdateDatabaseSettings(documentStore, configuration.Name, cancellationToken);
22+
await CreateDatabase(documentStore, configuration.Name, cancellationToken);
2423

25-
await CreateIndexes(documentStore, cancellationToken);
24+
await UpdateDatabaseSettings(documentStore, configuration.Name, cancellationToken);
2625

27-
await ConfigureExpiration(documentStore, cancellationToken);
28-
}
26+
await CreateIndexes(documentStore, configuration.EnableFullTextSearch, cancellationToken);
2927

30-
async Task CreateDatabase(IDocumentStore documentStore, string databaseName, CancellationToken cancellationToken)
31-
{
32-
var dbRecord = await documentStore.Maintenance.Server.SendAsync(new GetDatabaseRecordOperation(databaseName), cancellationToken);
28+
await ConfigureExpiration(documentStore, cancellationToken);
29+
}
3330

34-
if (dbRecord is null)
35-
{
36-
try
37-
{
38-
var databaseRecord = new DatabaseRecord(databaseName);
39-
databaseRecord.Settings.Add("Indexing.Auto.SearchEngineType", "Corax");
40-
databaseRecord.Settings.Add("Indexing.Static.SearchEngineType", "Corax");
41-
42-
await documentStore.Maintenance.Server.SendAsync(new CreateDatabaseOperation(databaseRecord), cancellationToken);
43-
}
44-
catch (ConcurrencyException)
45-
{
46-
// The database was already created before calling CreateDatabaseOperation
47-
}
48-
}
49-
}
31+
async Task CreateDatabase(IDocumentStore documentStore, string databaseName, CancellationToken cancellationToken)
32+
{
33+
var dbRecord = await documentStore.Maintenance.Server.SendAsync(new GetDatabaseRecordOperation(databaseName), cancellationToken);
5034

51-
async Task UpdateDatabaseSettings(IDocumentStore documentStore, string databaseName, CancellationToken cancellationToken)
35+
if (dbRecord is null)
5236
{
53-
var dbRecord = await documentStore.Maintenance.Server.SendAsync(new GetDatabaseRecordOperation(databaseName), cancellationToken);
54-
55-
if (dbRecord is null)
37+
try
5638
{
57-
throw new InvalidOperationException($"Database '{databaseName}' does not exist.");
58-
}
59-
60-
var updated = false;
39+
var databaseRecord = new DatabaseRecord(databaseName);
6140

62-
updated |= dbRecord.Settings.TryAdd("Indexing.Auto.SearchEngineType", "Corax");
63-
updated |= dbRecord.Settings.TryAdd("Indexing.Static.SearchEngineType", "Corax");
41+
SetSearchEngineType(databaseRecord, SearchEngineType.Corax);
6442

65-
if (updated)
43+
await documentStore.Maintenance.Server.SendAsync(new CreateDatabaseOperation(databaseRecord), cancellationToken);
44+
}
45+
catch (ConcurrencyException)
6646
{
67-
await documentStore.Maintenance.ForDatabase(databaseName).SendAsync(new PutDatabaseSettingsOperation(databaseName, dbRecord.Settings), cancellationToken);
68-
await documentStore.Maintenance.Server.SendAsync(new ToggleDatabasesStateOperation(databaseName, true), cancellationToken);
69-
await documentStore.Maintenance.Server.SendAsync(new ToggleDatabasesStateOperation(databaseName, false), cancellationToken);
47+
// The database was already created before calling CreateDatabaseOperation
7048
}
7149
}
50+
}
7251

73-
public static async Task DeleteLegacySagaDetailsIndex(IDocumentStore documentStore, CancellationToken cancellationToken)
52+
async Task UpdateDatabaseSettings(IDocumentStore documentStore, string databaseName, CancellationToken cancellationToken)
53+
{
54+
var databaseRecord = await documentStore.Maintenance.Server.SendAsync(new GetDatabaseRecordOperation(databaseName), cancellationToken) ?? throw new InvalidOperationException($"Database '{databaseName}' does not exist.");
55+
56+
if (!SetSearchEngineType(databaseRecord, SearchEngineType.Corax))
7457
{
75-
// If the SagaDetailsIndex exists but does not have a .Take(50000), then we remove the current SagaDetailsIndex and
76-
// create a new one. If we do not remove the current one, then RavenDB will attempt to do a side-by-side migration.
77-
// Doing a side-by-side migration results in the index never swapping if there is constant ingestion as RavenDB will wait.
78-
// for the index to not be stale before swapping to the new index. Constant ingestion means the index will never be not-stale.
79-
// This needs to stay in place until the next major version as the user could upgrade from an older version of the current
80-
// Major (v5.x.x) which might still have the incorrect index.
81-
var sagaDetailsIndexOperation = new GetIndexOperation("SagaDetailsIndex");
82-
var sagaDetailsIndexDefinition = await documentStore.Maintenance.SendAsync(sagaDetailsIndexOperation, cancellationToken);
83-
if (sagaDetailsIndexDefinition != null && !sagaDetailsIndexDefinition.Reduce.Contains("Take(50000)"))
84-
{
85-
await documentStore.Maintenance.SendAsync(new DeleteIndexOperation("SagaDetailsIndex"), cancellationToken);
86-
}
58+
return;
8759
}
8860

89-
async Task CreateIndexes(IDocumentStore documentStore, CancellationToken cancellationToken)
61+
await documentStore.Maintenance.ForDatabase(databaseName).SendAsync(new PutDatabaseSettingsOperation(databaseName, databaseRecord.Settings), cancellationToken);
62+
await documentStore.Maintenance.Server.SendAsync(new ToggleDatabasesStateOperation(databaseName, true), cancellationToken);
63+
await documentStore.Maintenance.Server.SendAsync(new ToggleDatabasesStateOperation(databaseName, false), cancellationToken);
64+
}
65+
66+
public static async Task DeleteLegacySagaDetailsIndex(IDocumentStore documentStore, CancellationToken cancellationToken)
67+
{
68+
// If the SagaDetailsIndex exists but does not have a .Take(50000), then we remove the current SagaDetailsIndex and
69+
// create a new one. If we do not remove the current one, then RavenDB will attempt to do a side-by-side migration.
70+
// Doing a side-by-side migration results in the index never swapping if there is constant ingestion as RavenDB will wait.
71+
// for the index to not be stale before swapping to the new index. Constant ingestion means the index will never be not-stale.
72+
// This needs to stay in place until the next major version as the user could upgrade from an older version of the current
73+
// Major (v5.x.x) which might still have the incorrect index.
74+
var sagaDetailsIndexOperation = new GetIndexOperation(SagaDetailsIndexName);
75+
var sagaDetailsIndexDefinition = await documentStore.Maintenance.SendAsync(sagaDetailsIndexOperation, cancellationToken);
76+
if (sagaDetailsIndexDefinition != null && !sagaDetailsIndexDefinition.Reduce.Contains("Take(50000)"))
9077
{
91-
await DeleteLegacySagaDetailsIndex(documentStore, cancellationToken);
78+
await documentStore.Maintenance.SendAsync(new DeleteIndexOperation(SagaDetailsIndexName), cancellationToken);
79+
}
80+
}
9281

93-
List<AbstractIndexCreationTask> indexList = [new FailedAuditImportIndex(), new SagaDetailsIndex()];
82+
internal static async Task CreateIndexes(IDocumentStore documentStore, bool enableFreeTextSearch, CancellationToken cancellationToken)
83+
{
84+
await DeleteLegacySagaDetailsIndex(documentStore, cancellationToken);
9485

95-
if (configuration.EnableFullTextSearch)
96-
{
97-
indexList.Add(new MessagesViewIndexWithFullTextSearch());
98-
await documentStore.Maintenance.SendAsync(new DeleteIndexOperation("MessagesViewIndex"), cancellationToken);
99-
}
100-
else
101-
{
102-
indexList.Add(new MessagesViewIndex());
103-
await documentStore.Maintenance.SendAsync(new DeleteIndexOperation("MessagesViewIndexWithFullTextSearch"), cancellationToken);
104-
}
86+
List<AbstractIndexCreationTask> indexList = [new FailedAuditImportIndex(), new SagaDetailsIndex()];
10587

106-
await IndexCreation.CreateIndexesAsync(indexList, documentStore, null, null, cancellationToken);
88+
if (enableFreeTextSearch)
89+
{
90+
indexList.Add(new MessagesViewIndexWithFullTextSearch());
91+
await documentStore.Maintenance.SendAsync(new DeleteIndexOperation(MessagesViewIndexName), cancellationToken);
10792
}
93+
else
94+
{
95+
indexList.Add(new MessagesViewIndex());
96+
await documentStore.Maintenance.SendAsync(new DeleteIndexOperation(MessagesViewIndexWithFulltextSearchName), cancellationToken);
97+
}
98+
99+
await IndexCreation.CreateIndexesAsync(indexList, documentStore, null, null, cancellationToken);
100+
}
108101

109-
async Task ConfigureExpiration(IDocumentStore documentStore, CancellationToken cancellationToken)
102+
async Task ConfigureExpiration(IDocumentStore documentStore, CancellationToken cancellationToken)
103+
{
104+
var expirationConfig = new ExpirationConfiguration
110105
{
111-
var expirationConfig = new ExpirationConfiguration
112-
{
113-
Disabled = false,
114-
DeleteFrequencyInSec = configuration.ExpirationProcessTimerInSeconds
115-
};
106+
Disabled = false,
107+
DeleteFrequencyInSec = configuration.ExpirationProcessTimerInSeconds
108+
};
116109

117-
await documentStore.Maintenance.SendAsync(new ConfigureExpirationOperation(expirationConfig), cancellationToken);
118-
}
110+
await documentStore.Maintenance.SendAsync(new ConfigureExpirationOperation(expirationConfig), cancellationToken);
111+
}
112+
113+
bool SetSearchEngineType(DatabaseRecord database, SearchEngineType searchEngineType)
114+
{
115+
var updated = false;
116+
117+
updated |= database.Settings.TryAdd("Indexing.Auto.SearchEngineType", searchEngineType.ToString());
118+
updated |= database.Settings.TryAdd("Indexing.Static.SearchEngineType", searchEngineType.ToString());
119+
120+
return updated;
119121
}
120-
}
122+
123+
internal const string MessagesViewIndexWithFulltextSearchName = "MessagesViewIndexWithFullTextSearch";
124+
internal const string SagaDetailsIndexName = "SagaDetailsIndex";
125+
internal const string MessagesViewIndexName = "MessagesViewIndex";
126+
}

‎src/ServiceControl.Audit.Persistence.Tests.RavenDB/IndexSetupTests.cs‎

Lines changed: 134 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,134 @@
1+
namespace ServiceControl.Audit.Persistence.Tests;
2+
3+
using System;
4+
using System.Threading;
5+
using System.Threading.Tasks;
6+
using NUnit.Framework;
7+
using Persistence.RavenDB;
8+
using Persistence.RavenDB.Indexes;
9+
using Raven.Client.Documents.Indexes;
10+
using Raven.Client.Documents.Operations.Indexes;
11+
using Raven.Client.Exceptions.Documents.Indexes;
12+
13+
[TestFixture]
14+
class IndexSetupTests : PersistenceTestFixture
15+
{
16+
[Test]
17+
public async Task Corax_should_be_the_default_search_engine_type()
18+
{
19+
var indexes = await configuration.DocumentStore.Maintenance.SendAsync(new GetIndexesOperation(0, int.MaxValue));
20+
21+
foreach (var index in indexes)
22+
{
23+
var indexStats = await configuration.DocumentStore.Maintenance.SendAsync(new GetIndexStatisticsOperation(DatabaseSetup.MessagesViewIndexWithFulltextSearchName));
24+
Assert.That(indexStats.SearchEngineType, Is.EqualTo(SearchEngineType.Corax), $"{index.Name} is not using Corax");
25+
}
26+
}
27+
28+
[Test]
29+
public async Task Free_text_search_index_should_be_used_by_default()
30+
{
31+
var freeTextIndex = await configuration.DocumentStore.Maintenance.SendAsync(new GetIndexOperation(DatabaseSetup.MessagesViewIndexWithFulltextSearchName));
32+
var nonFreeTextIndex = await configuration.DocumentStore.Maintenance.SendAsync(new GetIndexOperation(DatabaseSetup.MessagesViewIndexName));
33+
34+
Assert.That(nonFreeTextIndex, Is.Null);
35+
Assert.That(freeTextIndex, Is.Not.Null);
36+
}
37+
38+
[Test]
39+
public async Task Free_text_search_index_can_be_opted_out_from()
40+
{
41+
await DatabaseSetup.CreateIndexes(configuration.DocumentStore, false, TestTimeoutCancellationToken);
42+
43+
var freeTextIndex = await configuration.DocumentStore.Maintenance.SendAsync(new GetIndexOperation(DatabaseSetup.MessagesViewIndexWithFulltextSearchName));
44+
var nonFreeTextIndex = await configuration.DocumentStore.Maintenance.SendAsync(new GetIndexOperation(DatabaseSetup.MessagesViewIndexName));
45+
46+
Assert.That(freeTextIndex, Is.Null);
47+
Assert.That(nonFreeTextIndex, Is.Not.Null);
48+
}
49+
50+
[Test]
51+
public async Task Indexes_should_be_reset_on_setup()
52+
{
53+
var index = new MessagesViewIndexWithFullTextSearch { Configuration = { ["Indexing.Static.SearchEngineType"] = SearchEngineType.Lucene.ToString() } };
54+
55+
var indexWithCustomConfigStats = await UpdateIndex(index);
56+
57+
Assert.That(indexWithCustomConfigStats.SearchEngineType, Is.EqualTo(SearchEngineType.Lucene));
58+
59+
await DatabaseSetup.CreateIndexes(configuration.DocumentStore, true, TestTimeoutCancellationToken);
60+
61+
await WaitForIndexDefinitionUpdate(indexWithCustomConfigStats);
62+
63+
var indexAfterResetStats = await configuration.DocumentStore.Maintenance.SendAsync(new GetIndexStatisticsOperation(index.IndexName));
64+
65+
Assert.That(indexAfterResetStats.SearchEngineType, Is.EqualTo(SearchEngineType.Corax));
66+
}
67+
68+
[Test]
69+
public async Task Indexes_should_not_be_reset_on_setup_when_locked_as_ignore()
70+
{
71+
var index = new MessagesViewIndexWithFullTextSearch
72+
{
73+
Configuration = { ["Indexing.Static.SearchEngineType"] = SearchEngineType.Lucene.ToString() },
74+
LockMode = IndexLockMode.LockedIgnore
75+
};
76+
77+
var indexStatsBefore = await UpdateIndex(index);
78+
79+
Assert.That(indexStatsBefore.SearchEngineType, Is.EqualTo(SearchEngineType.Lucene));
80+
81+
await DatabaseSetup.CreateIndexes(configuration.DocumentStore, true, TestTimeoutCancellationToken);
82+
83+
// raven will ignore the update since index was locked, so best we can do is wait a bit and check that settings hasn't changed
84+
await Task.Delay(1000);
85+
86+
var indexStatsAfter = await configuration.DocumentStore.Maintenance.SendAsync(new GetIndexStatisticsOperation(index.IndexName));
87+
Assert.That(indexStatsAfter.SearchEngineType, Is.EqualTo(SearchEngineType.Lucene));
88+
}
89+
90+
[Test]
91+
public async Task Indexes_should_not_be_reset_on_setup_when_locked_as_error()
92+
{
93+
var index = new MessagesViewIndexWithFullTextSearch
94+
{
95+
Configuration = { ["Indexing.Static.SearchEngineType"] = SearchEngineType.Lucene.ToString() },
96+
LockMode = IndexLockMode.LockedError
97+
};
98+
99+
await UpdateIndex(index);
100+
101+
Assert.ThrowsAsync<IndexCreationException>(async () => await DatabaseSetup.CreateIndexes(configuration.DocumentStore, true, TestTimeoutCancellationToken));
102+
}
103+
104+
async Task<IndexStats> UpdateIndex(IAbstractIndexCreationTask index)
105+
{
106+
var statsBefore = await configuration.DocumentStore.Maintenance.SendAsync(new GetIndexStatisticsOperation(index.IndexName), TestTimeoutCancellationToken);
107+
108+
await IndexCreation.CreateIndexesAsync([index], configuration.DocumentStore, null, null, TestTimeoutCancellationToken);
109+
110+
return await WaitForIndexDefinitionUpdate(statsBefore);
111+
}
112+
113+
async Task<IndexStats> WaitForIndexDefinitionUpdate(IndexStats oldStats)
114+
{
115+
while (true)
116+
{
117+
try
118+
{
119+
var newStats = await configuration.DocumentStore.Maintenance.SendAsync(new GetIndexStatisticsOperation(oldStats.Name), TestTimeoutCancellationToken);
120+
121+
if (newStats.CreatedTimestamp > oldStats.CreatedTimestamp)
122+
{
123+
return newStats;
124+
}
125+
}
126+
catch (OperationCanceledException)
127+
{
128+
// keep going since we can get this if we query right when the update happens
129+
}
130+
131+
await Task.Delay(TimeSpan.FromMilliseconds(100), TestTimeoutCancellationToken);
132+
}
133+
}
134+
}

0 commit comments

Comments
 (0)