Ensure commits are no larger than 100 changes (#1606)

* don't include the object twice when serializing `MiniLcmCrdtAdapter`

* chunk changes into commits of 100, using the defer commit feature of Harmony

* fix a race condition in a migration test. Prevent hiding exceptions when querying entries

* update harmony

* flush commits server side based on the number of changes or commits

Author: hahn-kev

backend/FwLite/LcmCrdt.Tests/Data/MigrationTests.cs

@@ -57,6 +57,8 @@ private async Task InitDbFromScripts()
     public async Task GetEntries_WorksAfterMigrationFromScriptedDb()
     {
         await InitDbFromScripts();
+        //manually open the connection, this prevents a race condition where the entries query runs before the collations are created.
+        await _asyncScope.ServiceProvider.GetRequiredService<LcmCrdtDbContext>().Database.OpenConnectionAsync();
         var api = _asyncScope.ServiceProvider.GetRequiredService<IMiniLcmApi>();
         var hasEntries = false;
         await foreach (var entry in api.GetEntries(new(Count: 100)))

backend/FwLite/LcmCrdt.Tests/DataModelSnapshotTests.VerifyDbModel.verified.txt

@@ -359,6 +359,7 @@
     Foreign keys: 
       ObjectSnapshot {'CommitId'} -> Commit {'Id'} Required Cascade ToDependent: Snapshots ToPrincipal: Commit
     Indexes: 
+      EntityId
       CommitId, EntityId Unique
     Annotations: 
       DiscriminatorProperty: 

backend/FwLite/LcmCrdt/CrdtMiniLcmApi.cs

@@ -27,6 +27,7 @@ public class CrdtMiniLcmApi(
     CurrentProjectService projectService,
     IMiniLcmCultureProvider cultureProvider,
     MiniLcmValidators validators,
+    LcmCrdtDbContext dbContext,
     IOptions<LcmCrdtConfig> config) : IMiniLcmApi
 {
     private Guid ClientId { get; } = projectService.ProjectData.ClientId;
@@ -60,10 +61,24 @@ private async Task<Commit> AddChange(IChange change)
         return commit;
     }
 
-    private async Task<Commit> AddChanges(IEnumerable<IChange> changes)
+    private async Task AddChanges(IEnumerable<IChange> changes)
     {
-        var commit = await dataModel.AddChanges(ClientId, changes, commitMetadata: NewMetadata());
-        return commit;
+        await AddChanges(changes.Chunk(100));
+    }
+
+    /// <summary>
+    /// use when making a large number of changes at once
+    /// </summary>
+    /// <param name="changeChunks"></param>
+    private async Task AddChanges(IEnumerable<IChange[]> changeChunks)
+    {
+        await using var transaction = await dbContext.Database.BeginTransactionAsync();
+        foreach (var chunk in changeChunks)
+        {
+            await dataModel.AddChanges(ClientId, chunk, commitMetadata: NewMetadata(), deferCommit: true);
+        }
+        await dataModel.FlushDeferredCommits();
+        await transaction.CommitAsync();
     }
 
     public async Task<WritingSystems> GetWritingSystems()
@@ -403,7 +418,7 @@ private async IAsyncEnumerable<Entry> GetEntries(
         var complexFormComparer = cultureProvider.GetCompareInfo(sortWs)
             .AsComplexFormComparer();
         var entries = queryable.AsAsyncEnumerable();
-        await foreach (var entry in entries)
+        await foreach (var entry in EfExtensions.SafeIterate(entries))
         {
             entry.ApplySortOrder(complexFormComparer);
             yield return entry;
@@ -433,12 +448,10 @@ private async IAsyncEnumerable<Entry> GetEntries(
     public async Task BulkCreateEntries(IAsyncEnumerable<Entry> entries)
     {
         var semanticDomains = await SemanticDomains.ToDictionaryAsync(sd => sd.Id, sd => sd);
-        await AddChanges(
-            entries.ToBlockingEnumerable()
-                .SelectMany(entry => CreateEntryChanges(entry, semanticDomains))
-                //force entries to be created first, this avoids issues where references are created before the entry is created
-                .OrderBy(c => c is CreateEntryChange ? 0 : 1)
-        );
+        await AddChanges(entries.ToBlockingEnumerable()
+            .SelectMany(entry => CreateEntryChanges(entry, semanticDomains))
+            //force entries to be created first, this avoids issues where references are created before the entry is created
+            .OrderBy(c => c is CreateEntryChange ? 0 : 1));
     }
 
     private IEnumerable<IChange> CreateEntryChanges(Entry entry, Dictionary<Guid, SemanticDomain> semanticDomains)