Project only the latest snapshot of each entity in a single batch (#25)

* Introduce snapshot scope for better managing what gets projected

* simplify snapshot scope, remove AddIfNew since it didn't seem to make a difference which was used

* fix issue of inserting snapshots which were already created

* Remove snapshot scope and project all latest snapshots in single batch

* add failed sync dump which will export the commits, the entity which the update failed on and the error message

* Don't make assumptions about the order of snapshots when projecting

* Verify new snapshots

* write test to ensure that a word reference gets updated even when it's in a root snapshot that was created during this snapshot update

* only loop over snapshots once when adding them

---------

Co-authored-by: Kevin Hahn <[email protected]>

Author: myieye

src/SIL.Harmony.Sample/Changes/NewWordChange.cs

@@ -4,13 +4,15 @@
 
 namespace SIL.Harmony.Sample.Changes;
 
-public class NewWordChange(Guid entityId, string text, string? note = null) : CreateChange<Word>(entityId), ISelfNamedType<NewWordChange>
+public class NewWordChange(Guid entityId, string text, string? note = null, Guid? antonymId = null) : CreateChange<Word>(entityId), ISelfNamedType<NewWordChange>
 {
     public string Text { get; } = text;
     public string? Note { get; } = note;
+    public Guid? AntonymId { get; } = antonymId;
 
-    public override ValueTask<Word> NewEntity(Commit commit, ChangeContext context)
+    public override async ValueTask<Word> NewEntity(Commit commit, ChangeContext context)
     {
-        return new(new Word { Text = Text, Note = Note, Id = EntityId });
+        var antonymShouldBeNull = AntonymId is null || (await context.IsObjectDeleted(AntonymId.Value));
+        return (new Word { Text = Text, Note = Note, Id = EntityId, AntonymId = antonymShouldBeNull ? null : AntonymId });
     }
 }

src/SIL.Harmony.Sample/Changes/SetTagChange.cs

@@ -0,0 +1,26 @@
+using SIL.Harmony.Changes;
+using SIL.Harmony.Entities;
+using SIL.Harmony.Sample.Models;
+
+namespace SIL.Harmony.Sample.Changes;
+
+public class SetTagChange(Guid entityId, string text) : Change<Tag>(entityId), ISelfNamedType<SetTagChange>
+{
+    public string Text { get; } = text;
+
+    public override ValueTask<Tag> NewEntity(Commit commit, ChangeContext context)
+    {
+        return new(new Tag()
+        {
+            Id = EntityId,
+            Text = Text
+        });
+    }
+
+
+    public override ValueTask ApplyChange(Tag entity, ChangeContext context)
+    {
+        entity.Text = Text;
+        return ValueTask.CompletedTask;
+    }
+}

src/SIL.Harmony.Sample/CrdtSampleKernel.cs

@@ -48,9 +48,11 @@ public static IServiceCollection AddCrdtDataSample(this IServiceCollection servi
                 .Add<AddWordImageChange>()
                 .Add<SetOrderChange<Definition>>()
                 .Add<SetDefinitionPartOfSpeechChange>()
+                .Add<SetTagChange>()
                 .Add<DeleteChange<Word>>()
                 .Add<DeleteChange<Definition>>()
                 .Add<DeleteChange<Example>>()
+                .Add<DeleteChange<Tag>>()
                 ;
             config.ObjectTypeListBuilder.DefaultAdapter()
                 .Add<Word>()
@@ -61,7 +63,11 @@ public static IServiceCollection AddCrdtDataSample(this IServiceCollection servi
                         .HasForeignKey(d => d.WordId)
                         .OnDelete(DeleteBehavior.Cascade);
                 })
-                .Add<Example>();
+                .Add<Example>()
+                .Add<Tag>(builder =>
+                {
+                    builder.HasIndex(tag => tag.Text).IsUnique();
+                });
         });
         return services;
     }

src/SIL.Harmony.Sample/Models/Tag.cs

@@ -0,0 +1,30 @@
+using SIL.Harmony.Entities;
+
+namespace SIL.Harmony.Sample.Models;
+
+public class Tag : IObjectBase<Tag>
+{
+    public required string Text { get; set; }
+
+    public Guid Id { get; init; }
+    public DateTimeOffset? DeletedAt { get; set; }
+
+    public Guid[] GetReferences()
+    {
+        return [];
+    }
+
+    public void RemoveReference(Guid id, Commit commit)
+    {
+    }
+
+    public IObjectBase Copy()
+    {
+        return new Tag
+        {
+            Id = Id,
+            Text = Text,
+            DeletedAt = DeletedAt
+        };
+    }
+}

src/SIL.Harmony.Tests/DataModelReferenceTests.cs

@@ -1,4 +1,5 @@
-using SIL.Harmony.Changes;
+using Microsoft.EntityFrameworkCore;
+using SIL.Harmony.Changes;
 using SIL.Harmony.Sample.Changes;
 using SIL.Harmony.Sample.Models;
 using SIL.Harmony.Tests;
@@ -65,4 +66,22 @@ public async Task DeleteRetroactivelyRemovesRefs()
         var entry = await DataModel.GetLatest<Word>(entityId3);
         entry!.AntonymId.Should().BeNull();
     }
-}
+
+    [Fact]
+    public async Task DeleteDoesNotEffectARootSnapshotCreatedBeforeTheDelete()
+    {
+        var wordId = Guid.NewGuid();
+        var initialWordCommit = await WriteNextChange(new NewWordChange(wordId, "entity1", antonymId: _word1Id), add: false);
+        var deleteWordCommit = await WriteNextChange(DeleteWord(_word1Id), add: false);
+        await AddCommitsViaSync([
+            initialWordCommit,
+            deleteWordCommit
+        ]);
+        var snapshot = await DbContext.Snapshots.SingleAsync(s => s.CommitId == initialWordCommit.Id);
+        var initialWord = (Word) snapshot.Entity;
+        initialWord.AntonymId.Should().Be(_word1Id);
+        snapshot = await DbContext.Snapshots.SingleAsync(s => s.CommitId == deleteWordCommit.Id && s.EntityId == wordId);
+        var wordWithoutRef = (Word) snapshot.Entity;
+        wordWithoutRef.AntonymId.Should().BeNull();
+    }
+}

src/SIL.Harmony.Tests/DataModelSimpleChanges.Writing2ChangesAtOnceWithMergedHistory.verified.txt

@@ -50,29 +50,12 @@
   },
   {
     $type: Commit,
-    Snapshots: [
-      {
-        $type: ObjectSnapshot,
-        Id: Guid_5,
-        TypeName: Word,
-        Entity: {
-          $type: Word,
-          Text: first,
-          Note: a word note,
-          Id: Guid_2
-        },
-        EntityId: Guid_2,
-        EntityIsDeleted: false,
-        CommitId: Guid_6,
-        IsRoot: false
-      }
-    ],
     Hash: Hash_2,
     ParentHash: Hash_1,
     ChangeEntities: [
       {
         $type: ChangeEntity<IChange>,
-        CommitId: Guid_6,
+        CommitId: Guid_5,
         EntityId: Guid_2,
         Change: {
           $type: SetWordNoteChange,
@@ -85,9 +68,9 @@
     CompareKey: {
       $type: ValueTuple<DateTimeOffset, long,
       Item1: DateTimeOffset_2,
-      Item3: Guid_6
+      Item3: Guid_5
     },
-    Id: Guid_6,
+    Id: Guid_5,
     HybridDateTime: {
       $type: HybridDateTime,
       DateTime: DateTimeOffset_2
@@ -103,7 +86,7 @@
     Snapshots: [
       {
         $type: ObjectSnapshot,
-        Id: Guid_7,
+        Id: Guid_6,
         TypeName: Word,
         Entity: {
           $type: Word,
@@ -112,7 +95,7 @@
         },
         EntityId: Guid_2,
         EntityIsDeleted: false,
-        CommitId: Guid_8,
+        CommitId: Guid_7,
         IsRoot: false
       }
     ],
@@ -121,7 +104,7 @@
     ChangeEntities: [
       {
         $type: ChangeEntity<IChange>,
-        CommitId: Guid_8,
+        CommitId: Guid_7,
         EntityId: Guid_2,
         Change: {
           $type: SetWordTextChange,
@@ -134,9 +117,9 @@
     CompareKey: {
       $type: ValueTuple<DateTimeOffset, long,
       Item1: DateTimeOffset_3,
-      Item3: Guid_8
+      Item3: Guid_7
     },
-    Id: Guid_8,
+    Id: Guid_7,
     HybridDateTime: {
       $type: HybridDateTime,
       DateTime: DateTimeOffset_3
@@ -152,7 +135,7 @@
     Snapshots: [
       {
         $type: ObjectSnapshot,
-        Id: Guid_9,
+        Id: Guid_8,
         TypeName: Word,
         Entity: {
           $type: Word,
@@ -162,7 +145,7 @@
         },
         EntityId: Guid_2,
         EntityIsDeleted: false,
-        CommitId: Guid_10,
+        CommitId: Guid_9,
         IsRoot: false
       }
     ],
@@ -171,7 +154,7 @@
     ChangeEntities: [
       {
         $type: ChangeEntity<IChange>,
-        CommitId: Guid_10,
+        CommitId: Guid_9,
         EntityId: Guid_2,
         Change: {
           $type: SetWordTextChange,
@@ -184,9 +167,9 @@
     CompareKey: {
       $type: ValueTuple<DateTimeOffset, long,
       Item1: DateTimeOffset_4,
-      Item3: Guid_10
+      Item3: Guid_9
     },
-    Id: Guid_10,
+    Id: Guid_9,
     HybridDateTime: {
       $type: HybridDateTime,
       DateTime: DateTimeOffset_4

src/SIL.Harmony.Tests/DataModelTestBase.cs

@@ -131,6 +131,21 @@ public IChange SetWord(Guid entityId, string value)
         return new SetWordTextChange(entityId, value);
     }
 
+    public IChange DeleteWord(Guid entityId)
+    {
+        return new DeleteChange<Word>(entityId);
+    }
+
+    public IChange SetTag(Guid entityId, string value)
+    {
+        return new SetTagChange(entityId, value);
+    }
+
+    public IChange DeleteTag(Guid entityId)
+    {
+        return new DeleteChange<Tag>(entityId);
+    }
+
     public IChange NewDefinition(Guid wordId,
         string text,
         string partOfSpeech,

src/SIL.Harmony.Tests/DbContextTests.VerifyModel.verified.txt

@@ -162,6 +162,27 @@
       Relational:TableName: Example
       Relational:ViewName: 
       Relational:ViewSchema: 
+  EntityType: Tag
+    Properties: 
+      Id (Guid) Required PK AfterSave:Throw ValueGenerated.OnAdd
+      DeletedAt (DateTimeOffset?)
+      SnapshotId (no field, Guid?) Shadow FK Index
+      Text (string) Required Index
+    Keys: 
+      Id PK
+    Foreign keys: 
+      Tag {'SnapshotId'} -> ObjectSnapshot {'Id'} Unique SetNull
+    Indexes: 
+      SnapshotId Unique
+      Text Unique
+    Annotations: 
+      DiscriminatorProperty: 
+      Relational:FunctionName: 
+      Relational:Schema: 
+      Relational:SqlQuery: 
+      Relational:TableName: Tag
+      Relational:ViewName: 
+      Relational:ViewSchema: 
   EntityType: Word
     Properties: 
       Id (Guid) Required PK AfterSave:Throw ValueGenerated.OnAdd

src/SIL.Harmony.Tests/SnapshotTests.cs

@@ -74,4 +74,35 @@ await WriteChange(_localClientId,
             ]);
         DbContext.Snapshots.Should().ContainSingle();
     }
+
+    [Fact]
+    public async Task DontAddTheSameSnapshotTwice()
+    {
+        var entityId = Guid.NewGuid();
+        await WriteNextChange(SetWord(entityId, "test root"));
+        await WriteNextChange(SetWord(entityId, "test non root"));
+
+        await AddCommitsViaSync([
+            //the order here is important, the second commit was causing the snapshot for 'test non root' to attempt to be inserted again
+            await WriteNextChange(SetWord(Guid.NewGuid(), "test 1"), add: false),
+            await WriteNextChange(SetWord(entityId, "test 2"), add: false),
+        ]);
+    }
+
+    [Fact]
+    public async Task CanRecreateUniqueConstraintConflictingValueInOneCommit()
+    {
+        var entityId = Guid.NewGuid();
+        await WriteChange(_localClientId,
+            DateTimeOffset.Now,
+            [
+                SetTag(entityId, "tag-1"),
+            ]);
+        await WriteChange(_localClientId,
+            DateTimeOffset.Now,
+            [
+                DeleteTag(entityId),
+                SetTag(Guid.NewGuid(), "tag-1"),
+            ]);
+    }
 }

src/SIL.Harmony/CrdtConfig.cs

@@ -74,6 +74,8 @@ private void JsonTypeModifier(JsonTypeInfo typeInfo)
 
     public bool RemoteResourcesEnabled { get; private set; }
     public string LocalResourceCachePath { get; set; } = Path.GetFullPath("./localResourceCache");
+    public string FailedSyncOutputPath { get; set; } = Path.GetFullPath("./failedSyncs");
+
     public void AddRemoteResourceEntity(string? cachePath = null)
     {
         RemoteResourcesEnabled = true;