create a converter and use in EF to deserialize ChangeEntries which stored json in json (#1602)

Author: hahn-kev

backend/LexData/Entities/CommitEntityConfiguration.cs

@@ -11,6 +11,10 @@ namespace LexData.Entities;
 
 public class CommitEntityConfiguration : IEntityTypeConfiguration<ServerCommit>
 {
+    private static JsonSerializerOptions Options = new JsonSerializerOptions()
+    {
+        Converters = { new ServerJsonChangeConverter() }
+    };
     public void Configure(EntityTypeBuilder<ServerCommit> builder)
     {
         builder.ToTable("CrdtCommits");
@@ -24,12 +28,37 @@ public void Configure(EntityTypeBuilder<ServerCommit> builder)
             json => JsonSerializer.Deserialize<CommitMetadata>(json, (JsonSerializerOptions?)null) ?? new()
         );
         builder.Property(c => c.ChangeEntities).HasConversion(
-            c => JsonSerializer.Serialize(c, (JsonSerializerOptions?)null),
-            json => JsonSerializer.Deserialize<List<ChangeEntity<ServerJsonChange>>>(json, (JsonSerializerOptions?)null) ?? new()
+            c => JsonSerializer.Serialize(c, Options),
+            json => JsonSerializer.Deserialize<List<ChangeEntity<ServerJsonChange>>>(json, Options) ?? new()
         ).HasColumnType("jsonb").IsRequired(false);
     }
 
     private static ServerJsonChange Deserialize(string s) => JsonSerializer.Deserialize<ServerJsonChange>(s)!;
 
     private static string Serialize(ServerJsonChange c) => JsonSerializer.Serialize(c);
+
+    private class ServerJsonChangeConverter: JsonConverter<ServerJsonChange>
+    {
+        public override ServerJsonChange? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions _)
+        {
+            //ignoring JsonSerializerOptions, otherwise we would stack overflow recursively using this converter
+            if (typeToConvert != typeof(ServerJsonChange))
+                throw new SerializationException("type not supported " + typeToConvert.FullName);
+            //special case, this object type used to be encoded as a string in json like this: {"Change": "{\"Prop\": 1}"}
+            if (reader.TokenType == JsonTokenType.String)
+            {
+                var json = reader.GetString();
+                if (json is null) return null;
+                return JsonSerializer.Deserialize<ServerJsonChange>(json);
+            }
+
+            return JsonSerializer.Deserialize<ServerJsonChange>(ref reader);
+        }
+
+        public override void Write(Utf8JsonWriter writer, ServerJsonChange value, JsonSerializerOptions options)
+        {
+            //not passing in options otherwise it would just use this converter again
+            JsonSerializer.Serialize(writer, value);
+        }
+    }
 }

backend/Testing/LexCore/Services/CrdtCommitServiceTests.cs

@@ -3,6 +3,7 @@
 using FluentAssertions.Equivalency;
 using LexBoxApi.Services;
 using LexData;
+using LinqToDB.EntityFrameworkCore;
 using Microsoft.EntityFrameworkCore;
 using Microsoft.Extensions.DependencyInjection;
 using SIL.Harmony.Core;
@@ -61,6 +62,58 @@ private async IAsyncEnumerable<ServerCommit> AsAsync(IEnumerable<ServerCommit> c
         }
     }
 
+    //previously the value of the Change property was serialized twice, this test ensures that we can still
+    //pull those commits out of the database
+    [Fact]
+    public async Task CanQueryOldCommits()
+    {
+        var projectId = await _lexBoxDbContext.Projects.Select(p => p.Id).FirstOrDefaultAsync();
+        var context = _lexBoxDbContext.CreateLinqToDBContext();
+        var table = LinqToDB.DataExtensions.GetTable<ServerCommit>(context);
+        var commitId = Guid.NewGuid();
+        var changeEntity = new ChangeEntity<ServerJsonChange>
+        {
+            Index = 0,
+            CommitId = commitId,
+            EntityId = Guid.NewGuid(),
+            Change = new()
+            {
+                Type = "MyTestType",
+                ExtensionData = new Dictionary<string, JsonElement>()
+                {
+                    ["MyTestProperty"] = JsonSerializer.SerializeToElement("MyTestValue")
+                }
+            }
+        };
+        var changeEntityJson = JsonSerializer.SerializeToNode(changeEntity);
+        changeEntityJson.Should().NotBeNull();
+        //the old format stored json in json, this is emulating that.
+        changeEntityJson["Change"] = changeEntityJson["Change"]?.ToJsonString();
+        var jsonPayload = changeEntityJson.ToJsonString();
+        var inlineSql = $"'[{jsonPayload}]'::jsonb";
+        //insert a new server commit, manually specifying the value for ChangeEntities so it will match the old format.
+        await LinqToDB.LinqExtensions.InsertAsync(table, () => new ServerCommit(commitId)
+        {
+            Id = commitId,
+            ClientId = Guid.NewGuid(),
+            HybridDateTime = new HybridDateTime(DateTimeOffset.UtcNow, 0)
+            {
+                DateTime = DateTimeOffset.UtcNow,
+                Counter = 0
+            },
+            ProjectId = projectId,
+            Metadata = new CommitMetadata(),
+            ChangeEntities = LinqToDB.Sql.Expr<List<ChangeEntity<ServerJsonChange>>>(inlineSql)
+        });
+        var commits = await _lexBoxDbContext.CrdtCommits(projectId).ToArrayAsync();
+        var actualCommit = commits.Should().ContainSingle(c => c.Id == commitId).Subject;
+        actualCommit.ChangeEntities.Should().BeEquivalentTo([changeEntity],
+            options => options
+                .Using<JsonElement>(ctx => ctx.Subject.ToString().Should().Be(ctx.Expectation.ToString()))
+                .WhenTypeIs<JsonElement>()
+            );
+    }
+
 
     [Fact]
     public async Task CanAddCommits()