From 5634d4a76633e645e5e0a06a045b4997289258b9 Mon Sep 17 00:00:00 2001 From: Kevin Hahn Date: Wed, 23 Jul 2025 16:19:30 +0700 Subject: [PATCH 01/22] extract out an ICrdtRepository.cs interface --- src/SIL.Harmony.Tests/RepositoryTests.cs | 2 +- src/SIL.Harmony/DataModel.cs | 4 +-- src/SIL.Harmony/Db/CrdtRepository.cs | 12 +++---- src/SIL.Harmony/Db/ICrdtRepository.cs | 46 ++++++++++++++++++++++++ src/SIL.Harmony/ResourceService.cs | 2 +- src/SIL.Harmony/SnapshotWorker.cs | 8 ++--- 6 files changed, 60 insertions(+), 14 deletions(-) create mode 100644 src/SIL.Harmony/Db/ICrdtRepository.cs diff --git a/src/SIL.Harmony.Tests/RepositoryTests.cs b/src/SIL.Harmony.Tests/RepositoryTests.cs index b3e23ad..103fea3 100644 --- a/src/SIL.Harmony.Tests/RepositoryTests.cs +++ b/src/SIL.Harmony.Tests/RepositoryTests.cs @@ -12,7 +12,7 @@ namespace SIL.Harmony.Tests; public class RepositoryTests : IAsyncLifetime { private readonly ServiceProvider _services; - private readonly CrdtRepository _repository; + private readonly ICrdtRepository _repository; private readonly SampleDbContext _crdtDbContext; public RepositoryTests() diff --git a/src/SIL.Harmony/DataModel.cs b/src/SIL.Harmony/DataModel.cs index 9235389..0b46e31 100644 --- a/src/SIL.Harmony/DataModel.cs +++ b/src/SIL.Harmony/DataModel.cs @@ -194,7 +194,7 @@ ValueTask ISyncable.ShouldSync() return ValueTask.FromResult(true); } - private async Task UpdateSnapshots(CrdtRepository repo, Commit oldestAddedCommit, Commit[] newCommits) + private async Task UpdateSnapshots(ICrdtRepository repo, Commit oldestAddedCommit, Commit[] newCommits) { await repo.DeleteStaleSnapshots(oldestAddedCommit); Dictionary snapshotLookup; @@ -215,7 +215,7 @@ private async Task UpdateSnapshots(CrdtRepository repo, Commit oldestAddedCommit await snapshotWorker.UpdateSnapshots(oldestAddedCommit, newCommits); } - private async Task ValidateCommits(CrdtRepository repo) + private async Task ValidateCommits(ICrdtRepository repo) { Commit? parentCommit = null; await foreach (var commit in repo.CurrentCommits().AsNoTracking().AsAsyncEnumerable()) diff --git a/src/SIL.Harmony/Db/CrdtRepository.cs b/src/SIL.Harmony/Db/CrdtRepository.cs index 768613b..7b53a76 100644 --- a/src/SIL.Harmony/Db/CrdtRepository.cs +++ b/src/SIL.Harmony/Db/CrdtRepository.cs @@ -15,30 +15,30 @@ namespace SIL.Harmony.Db; internal class CrdtRepositoryFactory(IServiceProvider serviceProvider, ICrdtDbContextFactory dbContextFactory) { - public async Task CreateRepository() + public async Task CreateRepository() { return ActivatorUtilities.CreateInstance(serviceProvider, await dbContextFactory.CreateDbContextAsync()); } - public CrdtRepository CreateRepositorySync() + public ICrdtRepository CreateRepositorySync() { return ActivatorUtilities.CreateInstance(serviceProvider, dbContextFactory.CreateDbContext()); } - public async Task Execute(Func> func) + public async Task Execute(Func> func) { await using var repo = await CreateRepository(); return await func(repo); } - public async ValueTask Execute(Func> func) + public async ValueTask Execute(Func> func) { await using var repo = await CreateRepository(); return await func(repo); } } -internal class CrdtRepository : IDisposable, IAsyncDisposable +internal class CrdtRepository : IDisposable, IAsyncDisposable, ICrdtRepository { private static readonly ConcurrentDictionary Locks = new(); @@ -82,7 +82,7 @@ private string DatabaseIdentifier //doesn't really do anything when using a dbcontext factory since it will likely just have been created //but when not using the factory it is still useful - internal void ClearChangeTracker() + public void ClearChangeTracker() { _dbContext.ChangeTracker.Clear(); } diff --git a/src/SIL.Harmony/Db/ICrdtRepository.cs b/src/SIL.Harmony/Db/ICrdtRepository.cs new file mode 100644 index 0000000..6ce73d8 --- /dev/null +++ b/src/SIL.Harmony/Db/ICrdtRepository.cs @@ -0,0 +1,46 @@ +using Microsoft.EntityFrameworkCore.Storage; +using Nito.AsyncEx; +using SIL.Harmony.Resource; + +namespace SIL.Harmony.Db; + +internal interface ICrdtRepository : IAsyncDisposable, IDisposable +{ + AwaitableDisposable Lock(); + void ClearChangeTracker(); + Task BeginTransactionAsync(); + bool IsInTransaction { get; } + Task HasCommit(Guid commitId); + Task<(Commit? oldestChange, Commit[] newCommits)> FilterExistingCommits(ICollection commits); + Task DeleteStaleSnapshots(Commit oldestChange); + Task DeleteSnapshotsAndProjectedTables(); + IQueryable CurrentCommits(); + IQueryable CurrentSnapshots(); + IAsyncEnumerable CurrenSimpleSnapshots(bool includeDeleted = false); + Task<(Dictionary currentSnapshots, Commit[] pendingCommits)> GetCurrentSnapshotsAndPendingCommits(); + Task FindCommitByHash(string hash); + Task FindPreviousCommit(Commit commit); + Task GetCommitsAfter(Commit? commit); + Task FindSnapshot(Guid id, bool tracking = false); + Task GetCurrentSnapshotByObjectId(Guid objectId, bool tracking = false); + Task GetObjectBySnapshotId(Guid snapshotId); + Task GetCurrent(Guid objectId) where T: class; + IQueryable GetCurrentObjects() where T : class; + Task GetCurrentSyncState(); + Task> GetChanges(SyncState remoteState); + Task AddSnapshots(IEnumerable snapshots); + CrdtRepository GetScopedRepository(Commit excludeChangesAfterCommit); + Task AddCommit(Commit commit); + Task AddCommits(IEnumerable commits, bool save = true); + HybridDateTime? GetLatestDateTime(); + Task AddLocalResource(LocalResource localResource); + IAsyncEnumerable LocalResourcesByIds(IEnumerable resourceIds); + IAsyncEnumerable LocalResources(); + + /// + /// primarily for filtering other queries + /// + IQueryable LocalResourceIds(); + + Task GetLocalResource(Guid resourceId); +} \ No newline at end of file diff --git a/src/SIL.Harmony/ResourceService.cs b/src/SIL.Harmony/ResourceService.cs index c4fae85..61e87d0 100644 --- a/src/SIL.Harmony/ResourceService.cs +++ b/src/SIL.Harmony/ResourceService.cs @@ -149,7 +149,7 @@ public async Task DownloadResource(RemoteResource remoteResource, return await DownloadResourceInternal(repo, remoteResource, remoteResourceService); } - private async Task DownloadResourceInternal(CrdtRepository repo, + private async Task DownloadResourceInternal(ICrdtRepository repo, RemoteResource remoteResource, IRemoteResourceService remoteResourceService) { diff --git a/src/SIL.Harmony/SnapshotWorker.cs b/src/SIL.Harmony/SnapshotWorker.cs index 612346f..22827f9 100644 --- a/src/SIL.Harmony/SnapshotWorker.cs +++ b/src/SIL.Harmony/SnapshotWorker.cs @@ -11,7 +11,7 @@ namespace SIL.Harmony; internal class SnapshotWorker { private readonly Dictionary _snapshotLookup; - private readonly CrdtRepository _crdtRepository; + private readonly ICrdtRepository _crdtRepository; private readonly CrdtConfig _crdtConfig; private readonly Dictionary _pendingSnapshots = []; private readonly Dictionary _rootSnapshots = []; @@ -19,7 +19,7 @@ internal class SnapshotWorker private SnapshotWorker(Dictionary snapshots, Dictionary snapshotLookup, - CrdtRepository crdtRepository, + ICrdtRepository crdtRepository, CrdtConfig crdtConfig) { _pendingSnapshots = snapshots; @@ -30,7 +30,7 @@ private SnapshotWorker(Dictionary snapshots, internal static async Task> ApplyCommitsToSnapshots( Dictionary snapshots, - CrdtRepository crdtRepository, + ICrdtRepository crdtRepository, ICollection commits, CrdtConfig crdtConfig) { @@ -44,7 +44,7 @@ internal static async Task> ApplyCommitsToSnaps /// /// internal SnapshotWorker(Dictionary snapshotLookup, - CrdtRepository crdtRepository, + ICrdtRepository crdtRepository, CrdtConfig crdtConfig): this([], snapshotLookup, crdtRepository, crdtConfig) { } From e6ff74cc9b8f6756ea2e0965b4011981c8303e1b Mon Sep 17 00:00:00 2001 From: Kevin Hahn Date: Wed, 23 Jul 2025 16:46:19 +0700 Subject: [PATCH 02/22] create benchmarks project to enable dedicated running on benchmarks --- Directory.Packages.props | 4 +- README.md | 7 +++ harmony.sln | 6 ++ src/SIL.Harmony.Benchmarks/Program.cs | 4 ++ .../SIL.Harmony.Benchmarks.csproj | 18 ++++++ .../Benchmarks/ChangeThroughput.cs | 63 +++++++++++++++++++ .../DataModelPerformanceTests.cs | 60 +----------------- 7 files changed, 102 insertions(+), 60 deletions(-) create mode 100644 src/SIL.Harmony.Benchmarks/Program.cs create mode 100644 src/SIL.Harmony.Benchmarks/SIL.Harmony.Benchmarks.csproj create mode 100644 src/SIL.Harmony.Tests/Benchmarks/ChangeThroughput.cs diff --git a/Directory.Packages.props b/Directory.Packages.props index 4263bab..646a280 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -9,7 +9,7 @@ - + @@ -25,4 +25,4 @@ - + \ No newline at end of file diff --git a/README.md b/README.md index 7b8e061..ceaabd0 100644 --- a/README.md +++ b/README.md @@ -186,3 +186,10 @@ NuGet package versions are calculated from a combination of tags and commit mess * `+semver: minor` or `+semver: feature` - update minor version number, reset patch to 0 (so 2.3.1 would become 2.4.0) * Anything else, including no `+semver` lines at all - update patch version number (so 2.3.1 would become 2.3.2) * If you want to include `+semver` lines, then `+semver: patch` or `+semver: fix` are the standard ways to increment a patch version bump, but the patch version will be bumped regardless as long as there is at least one commit since the most recent tag. + +### Run benchmarks + +docs: https://benchmarkdotnet.org/articles/guides/console-args.html +```bash +dotnet run --project ./src/SIL.Harmony.Benchmarks/SIL.Harmony.Benchmarks.csproj -c Release -- --filter * +``` \ No newline at end of file diff --git a/harmony.sln b/harmony.sln index 8fc9d10..ab8ed43 100644 --- a/harmony.sln +++ b/harmony.sln @@ -21,6 +21,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution src\.editorconfig = src\.editorconfig EndProjectSection EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SIL.Harmony.Benchmarks", "src\SIL.Harmony.Benchmarks\SIL.Harmony.Benchmarks.csproj", "{96FABEBB-5A18-46D5-8530-84B9FFF5442C}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -54,5 +56,9 @@ Global {8EB807F6-C548-4016-856E-2ACCA5603036}.Debug|Any CPU.Build.0 = Debug|Any CPU {8EB807F6-C548-4016-856E-2ACCA5603036}.Release|Any CPU.ActiveCfg = Release|Any CPU {8EB807F6-C548-4016-856E-2ACCA5603036}.Release|Any CPU.Build.0 = Release|Any CPU + {96FABEBB-5A18-46D5-8530-84B9FFF5442C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {96FABEBB-5A18-46D5-8530-84B9FFF5442C}.Debug|Any CPU.Build.0 = Debug|Any CPU + {96FABEBB-5A18-46D5-8530-84B9FFF5442C}.Release|Any CPU.ActiveCfg = Release|Any CPU + {96FABEBB-5A18-46D5-8530-84B9FFF5442C}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection EndGlobal diff --git a/src/SIL.Harmony.Benchmarks/Program.cs b/src/SIL.Harmony.Benchmarks/Program.cs new file mode 100644 index 0000000..72b1f4e --- /dev/null +++ b/src/SIL.Harmony.Benchmarks/Program.cs @@ -0,0 +1,4 @@ +using BenchmarkDotNet.Running; +using SIL.Harmony.Tests.Benchmarks; + +BenchmarkSwitcher.FromAssemblies([typeof(Program).Assembly, typeof(ChangeThroughput).Assembly]).Run(args); \ No newline at end of file diff --git a/src/SIL.Harmony.Benchmarks/SIL.Harmony.Benchmarks.csproj b/src/SIL.Harmony.Benchmarks/SIL.Harmony.Benchmarks.csproj new file mode 100644 index 0000000..34faa54 --- /dev/null +++ b/src/SIL.Harmony.Benchmarks/SIL.Harmony.Benchmarks.csproj @@ -0,0 +1,18 @@ + + + + Exe + net9.0 + enable + enable + + + + + + + + + + + diff --git a/src/SIL.Harmony.Tests/Benchmarks/ChangeThroughput.cs b/src/SIL.Harmony.Tests/Benchmarks/ChangeThroughput.cs new file mode 100644 index 0000000..32b9462 --- /dev/null +++ b/src/SIL.Harmony.Tests/Benchmarks/ChangeThroughput.cs @@ -0,0 +1,63 @@ +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Engines; + +namespace SIL.Harmony.Tests.Benchmarks; + + +// disable warning about waiting for sync code, benchmarkdotnet does not support async code, and it doesn't deadlock when waiting. +#pragma warning disable VSTHRD002 +[SimpleJob(RunStrategy.Throughput, warmupCount: 2)] +public class ChangeThroughput +{ + private DataModelTestBase _templateModel = null!; + private DataModelTestBase _dataModelTestBase = null!; + private DataModelTestBase _emptyDataModel = null!; + + + [GlobalSetup] + public void GlobalSetup() + { + _templateModel = new DataModelTestBase(alwaysValidate: false, performanceTest: true); + DataModelPerformanceTests.BulkInsertChanges(_templateModel, StartingSnapshots).GetAwaiter().GetResult(); + } + + [Params(0, 1000, 10_000)] + public int StartingSnapshots { get; set; } + + [IterationSetup] + public void IterationSetup() + { + _emptyDataModel = new(alwaysValidate: false, performanceTest: true); + _ = _emptyDataModel.WriteNextChange(_emptyDataModel.SetWord(Guid.NewGuid(), "entity1")).Result; + _dataModelTestBase = _templateModel.ForkDatabase(false); + } + + [Benchmark(Baseline = true), BenchmarkCategory("WriteChange")] + public Commit AddSingleChangePerformance() + { + return _emptyDataModel.WriteNextChange(_emptyDataModel.SetWord(Guid.NewGuid(), "entity1")).Result; + } + + [Benchmark, BenchmarkCategory("WriteChange")] + public Commit AddSingleChangeWithManySnapshots() + { + var count = _dataModelTestBase.DbContext.Snapshots.Count(); + // had a bug where there were no snapshots, this means the test was useless, this is slower, but it's better that then a useless test + if (count < (StartingSnapshots - 5)) throw new Exception($"Not enough snapshots, found {count}"); + return _dataModelTestBase.WriteNextChange(_dataModelTestBase.SetWord(Guid.NewGuid(), "entity1")).Result; + } + + [IterationCleanup] + public void IterationCleanup() + { + _emptyDataModel.DisposeAsync().GetAwaiter().GetResult(); + _dataModelTestBase.DisposeAsync().GetAwaiter().GetResult(); + } + + [GlobalCleanup] + public void GlobalCleanup() + { + _templateModel.DisposeAsync().GetAwaiter().GetResult(); + } +} +#pragma warning restore VSTHRD002 \ No newline at end of file diff --git a/src/SIL.Harmony.Tests/DataModelPerformanceTests.cs b/src/SIL.Harmony.Tests/DataModelPerformanceTests.cs index 84398e5..a09e43f 100644 --- a/src/SIL.Harmony.Tests/DataModelPerformanceTests.cs +++ b/src/SIL.Harmony.Tests/DataModelPerformanceTests.cs @@ -11,6 +11,7 @@ using SIL.Harmony.Changes; using SIL.Harmony.Db; using SIL.Harmony.Sample.Changes; +using SIL.Harmony.Tests.Benchmarks; using Xunit.Abstractions; namespace SIL.Harmony.Tests; @@ -25,7 +26,7 @@ public void AddingChangePerformance() Assert.Fail("This test is disabled in debug builds, not reliable"); #endif var summary = - BenchmarkRunner.Run( + BenchmarkRunner.Run( ManualConfig.CreateEmpty() .AddExporter(JsonExporter.FullCompressed) .AddColumnProvider(DefaultColumnProviders.Instance) @@ -177,60 +178,3 @@ public void Flush() } } -// disable warning about waiting for sync code, benchmarkdotnet does not support async code, and it doesn't deadlock when waiting. -#pragma warning disable VSTHRD002 -[SimpleJob(RunStrategy.Throughput, warmupCount: 2)] -public class DataModelPerformanceBenchmarks -{ - private DataModelTestBase _templateModel = null!; - private DataModelTestBase _dataModelTestBase = null!; - private DataModelTestBase _emptyDataModel = null!; - - - [GlobalSetup] - public void GlobalSetup() - { - _templateModel = new DataModelTestBase(alwaysValidate: false, performanceTest: true); - DataModelPerformanceTests.BulkInsertChanges(_templateModel, StartingSnapshots).GetAwaiter().GetResult(); - } - - [Params(0, 1000, 10_000)] - public int StartingSnapshots { get; set; } - - [IterationSetup] - public void IterationSetup() - { - _emptyDataModel = new(alwaysValidate: false, performanceTest: true); - _ = _emptyDataModel.WriteNextChange(_emptyDataModel.SetWord(Guid.NewGuid(), "entity1")).Result; - _dataModelTestBase = _templateModel.ForkDatabase(false); - } - - [Benchmark(Baseline = true), BenchmarkCategory("WriteChange")] - public Commit AddSingleChangePerformance() - { - return _emptyDataModel.WriteNextChange(_emptyDataModel.SetWord(Guid.NewGuid(), "entity1")).Result; - } - - [Benchmark, BenchmarkCategory("WriteChange")] - public Commit AddSingleChangeWithManySnapshots() - { - var count = _dataModelTestBase.DbContext.Snapshots.Count(); - // had a bug where there were no snapshots, this means the test was useless, this is slower, but it's better that then a useless test - if (count < (StartingSnapshots - 5)) throw new Exception($"Not enough snapshots, found {count}"); - return _dataModelTestBase.WriteNextChange(_dataModelTestBase.SetWord(Guid.NewGuid(), "entity1")).Result; - } - - [IterationCleanup] - public void IterationCleanup() - { - _emptyDataModel.DisposeAsync().GetAwaiter().GetResult(); - _dataModelTestBase.DisposeAsync().GetAwaiter().GetResult(); - } - - [GlobalCleanup] - public void GlobalCleanup() - { - _templateModel.DisposeAsync().GetAwaiter().GetResult(); - } -} -#pragma warning restore VSTHRD002 \ No newline at end of file From c2d3c5fadb9aeffcb9d79e7ecb58b0a314600f67 Mon Sep 17 00:00:00 2001 From: Kevin Hahn Date: Wed, 23 Jul 2025 16:49:34 +0700 Subject: [PATCH 03/22] setup git job comparison --- Directory.Packages.props | 1 + src/SIL.Harmony.Benchmarks/MyCustomBenchmark.cs | 15 +++++++++++++++ .../SIL.Harmony.Benchmarks.csproj | 1 + src/SIL.Harmony.Linq2db/Linq2dbKernel.cs | 5 +++++ 4 files changed, 22 insertions(+) create mode 100644 src/SIL.Harmony.Benchmarks/MyCustomBenchmark.cs diff --git a/Directory.Packages.props b/Directory.Packages.props index 646a280..a645b0a 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -3,6 +3,7 @@ true + diff --git a/src/SIL.Harmony.Benchmarks/MyCustomBenchmark.cs b/src/SIL.Harmony.Benchmarks/MyCustomBenchmark.cs new file mode 100644 index 0000000..1feaef9 --- /dev/null +++ b/src/SIL.Harmony.Benchmarks/MyCustomBenchmark.cs @@ -0,0 +1,15 @@ +using BenchmarkDotNet_GitCompare; +using BenchmarkDotNet.Attributes; + +namespace SIL.Harmony.Benchmarks; + +[SimpleJob(id: "now")] +[GitJob(gitReference: "HEAD", id: "before", baseline: true)] +public class MyCustomBenchmark +{ + [Benchmark] + public Task Test() + { + return Task.Delay(10); + } +} \ No newline at end of file diff --git a/src/SIL.Harmony.Benchmarks/SIL.Harmony.Benchmarks.csproj b/src/SIL.Harmony.Benchmarks/SIL.Harmony.Benchmarks.csproj index 34faa54..b7818b3 100644 --- a/src/SIL.Harmony.Benchmarks/SIL.Harmony.Benchmarks.csproj +++ b/src/SIL.Harmony.Benchmarks/SIL.Harmony.Benchmarks.csproj @@ -9,6 +9,7 @@ + diff --git a/src/SIL.Harmony.Linq2db/Linq2dbKernel.cs b/src/SIL.Harmony.Linq2db/Linq2dbKernel.cs index 5c32565..77817e7 100644 --- a/src/SIL.Harmony.Linq2db/Linq2dbKernel.cs +++ b/src/SIL.Harmony.Linq2db/Linq2dbKernel.cs @@ -38,4 +38,9 @@ public static DbContextOptionsBuilder UseLinqToDbCrdt(this DbContextOptionsBuild optionsBuilder.AddCustomOptions(dataOptions => dataOptions.UseLoggerFactory(loggerFactory)); }); } + + public static IServiceCollection AddLinq2DbRepository(this IServiceCollection services) + { + return services; + } } From 04c6e4d0df1f642422d71441f564ed0a231efae5 Mon Sep 17 00:00:00 2001 From: Kevin Hahn Date: Wed, 23 Jul 2025 16:58:26 +0700 Subject: [PATCH 04/22] replace CrdtRepositoryFactory with ICrdtRepositoryFactory --- src/SIL.Harmony.Tests/RepositoryTests.cs | 2 +- src/SIL.Harmony/CrdtKernel.cs | 8 ++--- src/SIL.Harmony/DataModel.cs | 4 +-- src/SIL.Harmony/Db/CrdtRepository.cs | 28 +--------------- src/SIL.Harmony/Db/CrdtRepositoryFactory.cs | 36 +++++++++++++++++++++ src/SIL.Harmony/Db/ICrdtRepository.cs | 2 +- src/SIL.Harmony/ResourceService.cs | 4 +-- 7 files changed, 47 insertions(+), 37 deletions(-) create mode 100644 src/SIL.Harmony/Db/CrdtRepositoryFactory.cs diff --git a/src/SIL.Harmony.Tests/RepositoryTests.cs b/src/SIL.Harmony.Tests/RepositoryTests.cs index 103fea3..4c4eefc 100644 --- a/src/SIL.Harmony.Tests/RepositoryTests.cs +++ b/src/SIL.Harmony.Tests/RepositoryTests.cs @@ -21,7 +21,7 @@ public RepositoryTests() .AddCrdtDataSample(":memory:") .BuildServiceProvider(); - _repository = _services.GetRequiredService().CreateRepositorySync(); + _repository = _services.GetRequiredService().CreateRepositorySync(); _crdtDbContext = _services.GetRequiredService(); } diff --git a/src/SIL.Harmony/CrdtKernel.cs b/src/SIL.Harmony/CrdtKernel.cs index 2d1c95d..f2d8a06 100644 --- a/src/SIL.Harmony/CrdtKernel.cs +++ b/src/SIL.Harmony/CrdtKernel.cs @@ -33,10 +33,10 @@ public static IServiceCollection AddCrdtDataCore(this IServiceCollection service services.AddSingleton(sp => sp.GetRequiredService>().Value.JsonSerializerOptions); services.AddSingleton(TimeProvider.System); services.AddScoped(NewTimeProvider); - services.AddScoped(); + services.AddScoped(); //must use factory method because DataModel constructor is internal services.AddScoped(provider => new DataModel( - provider.GetRequiredService(), + provider.GetRequiredService(), provider.GetRequiredService(), provider.GetRequiredService(), provider.GetRequiredService>(), @@ -44,7 +44,7 @@ public static IServiceCollection AddCrdtDataCore(this IServiceCollection service )); //must use factory method because ResourceService constructor is internal services.AddScoped(provider => new ResourceService( - provider.GetRequiredService(), + provider.GetRequiredService(), provider.GetRequiredService>(), provider.GetRequiredService(), provider.GetRequiredService>() @@ -57,7 +57,7 @@ public static HybridDateTimeProvider NewTimeProvider(IServiceProvider servicePro //todo, if this causes issues getting the order correct, we can update the last date time after the db is created //as long as it's before we get a date time from the provider //todo use IMemoryCache to store the last date time, possibly based on the current project - using var repo = serviceProvider.GetRequiredService().CreateRepositorySync(); + using var repo = serviceProvider.GetRequiredService().CreateRepositorySync(); var hybridDateTime = repo.GetLatestDateTime(); hybridDateTime ??= HybridDateTimeProvider.DefaultLastDateTime; return ActivatorUtilities.CreateInstance(serviceProvider, hybridDateTime); diff --git a/src/SIL.Harmony/DataModel.cs b/src/SIL.Harmony/DataModel.cs index 0b46e31..b5e1b60 100644 --- a/src/SIL.Harmony/DataModel.cs +++ b/src/SIL.Harmony/DataModel.cs @@ -18,14 +18,14 @@ public class DataModel : ISyncable, IAsyncDisposable /// private bool AlwaysValidate => _crdtConfig.Value.AlwaysValidateCommits; - private readonly CrdtRepositoryFactory _crdtRepositoryFactory; + private readonly ICrdtRepositoryFactory _crdtRepositoryFactory; private readonly JsonSerializerOptions _serializerOptions; private readonly IHybridDateTimeProvider _timeProvider; private readonly IOptions _crdtConfig; private readonly ILogger _logger; //constructor must be internal because CrdtRepository is internal - internal DataModel(CrdtRepositoryFactory crdtRepositoryFactory, + internal DataModel(ICrdtRepositoryFactory crdtRepositoryFactory, JsonSerializerOptions serializerOptions, IHybridDateTimeProvider timeProvider, IOptions crdtConfig, diff --git a/src/SIL.Harmony/Db/CrdtRepository.cs b/src/SIL.Harmony/Db/CrdtRepository.cs index 7b53a76..eee307c 100644 --- a/src/SIL.Harmony/Db/CrdtRepository.cs +++ b/src/SIL.Harmony/Db/CrdtRepository.cs @@ -4,7 +4,6 @@ using Microsoft.EntityFrameworkCore.ChangeTracking; using Microsoft.EntityFrameworkCore.Infrastructure; using Microsoft.EntityFrameworkCore.Storage; -using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using Nito.AsyncEx; @@ -13,32 +12,7 @@ namespace SIL.Harmony.Db; -internal class CrdtRepositoryFactory(IServiceProvider serviceProvider, ICrdtDbContextFactory dbContextFactory) -{ - public async Task CreateRepository() - { - return ActivatorUtilities.CreateInstance(serviceProvider, await dbContextFactory.CreateDbContextAsync()); - } - - public ICrdtRepository CreateRepositorySync() - { - return ActivatorUtilities.CreateInstance(serviceProvider, dbContextFactory.CreateDbContext()); - } - - public async Task Execute(Func> func) - { - await using var repo = await CreateRepository(); - return await func(repo); - } - - public async ValueTask Execute(Func> func) - { - await using var repo = await CreateRepository(); - return await func(repo); - } -} - -internal class CrdtRepository : IDisposable, IAsyncDisposable, ICrdtRepository +public class CrdtRepository : IDisposable, IAsyncDisposable, ICrdtRepository { private static readonly ConcurrentDictionary Locks = new(); diff --git a/src/SIL.Harmony/Db/CrdtRepositoryFactory.cs b/src/SIL.Harmony/Db/CrdtRepositoryFactory.cs new file mode 100644 index 0000000..605f8c1 --- /dev/null +++ b/src/SIL.Harmony/Db/CrdtRepositoryFactory.cs @@ -0,0 +1,36 @@ +using Microsoft.Extensions.DependencyInjection; + +namespace SIL.Harmony.Db; + +public interface ICrdtRepositoryFactory +{ + Task CreateRepository(); + ICrdtRepository CreateRepositorySync(); + Task Execute(Func> func); + ValueTask Execute(Func> func); +} + +public class CrdtRepositoryFactory(IServiceProvider serviceProvider, ICrdtDbContextFactory dbContextFactory) : ICrdtRepositoryFactory +{ + public async Task CreateRepository() + { + return ActivatorUtilities.CreateInstance(serviceProvider, await dbContextFactory.CreateDbContextAsync()); + } + + public ICrdtRepository CreateRepositorySync() + { + return ActivatorUtilities.CreateInstance(serviceProvider, dbContextFactory.CreateDbContext()); + } + + public async Task Execute(Func> func) + { + await using var repo = await CreateRepository(); + return await func(repo); + } + + public async ValueTask Execute(Func> func) + { + await using var repo = await CreateRepository(); + return await func(repo); + } +} \ No newline at end of file diff --git a/src/SIL.Harmony/Db/ICrdtRepository.cs b/src/SIL.Harmony/Db/ICrdtRepository.cs index 6ce73d8..40eb650 100644 --- a/src/SIL.Harmony/Db/ICrdtRepository.cs +++ b/src/SIL.Harmony/Db/ICrdtRepository.cs @@ -4,7 +4,7 @@ namespace SIL.Harmony.Db; -internal interface ICrdtRepository : IAsyncDisposable, IDisposable +public interface ICrdtRepository : IAsyncDisposable, IDisposable { AwaitableDisposable Lock(); void ClearChangeTracker(); diff --git a/src/SIL.Harmony/ResourceService.cs b/src/SIL.Harmony/ResourceService.cs index 61e87d0..42a3e77 100644 --- a/src/SIL.Harmony/ResourceService.cs +++ b/src/SIL.Harmony/ResourceService.cs @@ -10,12 +10,12 @@ namespace SIL.Harmony; public class ResourceService { - private readonly CrdtRepositoryFactory _crdtRepositoryFactory; + private readonly ICrdtRepositoryFactory _crdtRepositoryFactory; private readonly IOptions _crdtConfig; private readonly DataModel _dataModel; private readonly ILogger _logger; - internal ResourceService(CrdtRepositoryFactory crdtRepositoryFactory, IOptions crdtConfig, DataModel dataModel, ILogger logger) + internal ResourceService(ICrdtRepositoryFactory crdtRepositoryFactory, IOptions crdtConfig, DataModel dataModel, ILogger logger) { _crdtRepositoryFactory = crdtRepositoryFactory; _crdtConfig = crdtConfig; From 8e08b90f5fa7b440a791f5e31a68999312f6dd17 Mon Sep 17 00:00:00 2001 From: Kevin Hahn Date: Thu, 24 Jul 2025 11:02:36 +0700 Subject: [PATCH 05/22] introduce some more benchmarks and initial version of Linq2Db repo --- .../AddChangeBenchmarks.cs | 48 ++++ .../AddSnapshotsBenchmarks.cs | 67 ++++++ .../MyCustomBenchmark.cs | 15 -- src/SIL.Harmony.Linq2db/Linq2DbCrdtRepo.cs | 227 ++++++++++++++++++ src/SIL.Harmony.Linq2db/Linq2dbKernel.cs | 12 +- src/SIL.Harmony.Tests/DataModelTestBase.cs | 2 + src/SIL.Harmony/Commit.cs | 2 +- src/SIL.Harmony/CrdtKernel.cs | 3 +- src/SIL.Harmony/Db/CrdtRepositoryFactory.cs | 29 ++- 9 files changed, 374 insertions(+), 31 deletions(-) create mode 100644 src/SIL.Harmony.Benchmarks/AddChangeBenchmarks.cs create mode 100644 src/SIL.Harmony.Benchmarks/AddSnapshotsBenchmarks.cs delete mode 100644 src/SIL.Harmony.Benchmarks/MyCustomBenchmark.cs create mode 100644 src/SIL.Harmony.Linq2db/Linq2DbCrdtRepo.cs diff --git a/src/SIL.Harmony.Benchmarks/AddChangeBenchmarks.cs b/src/SIL.Harmony.Benchmarks/AddChangeBenchmarks.cs new file mode 100644 index 0000000..587aa6a --- /dev/null +++ b/src/SIL.Harmony.Benchmarks/AddChangeBenchmarks.cs @@ -0,0 +1,48 @@ +using BenchmarkDotNet_GitCompare; +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Engines; +using SIL.Harmony.Linq2db; +using SIL.Harmony.Tests; + +namespace SIL.Harmony.Benchmarks; + +[SimpleJob(RunStrategy.Throughput)] +// [GitJob(gitReference: "HEAD", id: "before", baseline: true)] +public class AddChangeBenchmarks +{ + private DataModelTestBase _emptyDataModel = null!; + + [Params(200)] + public int ChangeCount { get; set; } + [Params(true, false)] + public bool UseLinq2DbRepo { get; set; } + + [IterationSetup] + public void IterationSetup() + { + _emptyDataModel = new(alwaysValidate: false, performanceTest: true, configure: collection => + { + if (UseLinq2DbRepo) + collection.AddLinq2DbRepository(); + }); + _emptyDataModel.WriteNextChange(_emptyDataModel.SetWord(Guid.NewGuid(), "entity1")).GetAwaiter().GetResult(); + } + + [Benchmark(OperationsPerInvoke = 200)] + public List AddChanges() + { + var commits = new List(); + for (var i = 0; i < ChangeCount; i++) + { + commits.Add(_emptyDataModel.WriteNextChange(_emptyDataModel.SetWord(Guid.NewGuid(), "entity1")).Result); + } + + return commits; + } + + [IterationCleanup] + public void IterationCleanup() + { + _emptyDataModel.DisposeAsync().GetAwaiter().GetResult(); + } +} \ No newline at end of file diff --git a/src/SIL.Harmony.Benchmarks/AddSnapshotsBenchmarks.cs b/src/SIL.Harmony.Benchmarks/AddSnapshotsBenchmarks.cs new file mode 100644 index 0000000..75831a0 --- /dev/null +++ b/src/SIL.Harmony.Benchmarks/AddSnapshotsBenchmarks.cs @@ -0,0 +1,67 @@ +using BenchmarkDotNet_GitCompare; +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Engines; +using Microsoft.Extensions.DependencyInjection; +using SIL.Harmony.Core; +using SIL.Harmony.Db; +using SIL.Harmony.Linq2db; +using SIL.Harmony.Sample.Models; +using SIL.Harmony.Tests; + +namespace SIL.Harmony.Benchmarks; + +[SimpleJob(RunStrategy.Throughput)] +// [GitJob(gitReference: "HEAD", id: "before", baseline: true)] +public class AddSnapshotsBenchmarks +{ + private DataModelTestBase _emptyDataModel = null!; + private ICrdtRepository _repository = null!; + private Commit _commit = null!; + + [Params(1000)] + public int SnapshotCount { get; set; } + + [Params(true, false)] + public bool UseLinq2DbRepo { get; set; } + + [IterationSetup] + public void IterationSetup() + { + _emptyDataModel = new(alwaysValidate: false, + performanceTest: true, + configure: collection => + { + if (UseLinq2DbRepo) + collection.AddLinq2DbRepository(); + }); + var crdtRepositoryFactory = _emptyDataModel.Services.GetRequiredService(); + _repository = crdtRepositoryFactory.CreateRepositorySync(); + _repository.AddCommit(_commit = new Commit(Guid.NewGuid()) + { + ClientId = Guid.NewGuid(), + HybridDateTime = new HybridDateTime(DateTimeOffset.Now, 0), + }).GetAwaiter().GetResult(); + } + + [Benchmark(OperationsPerInvoke = 1000)] + public void AddSnapshots() + { + for (var i = 0; i < SnapshotCount; i++) + { + _repository.AddSnapshots([ + new ObjectSnapshot(new Word() + { + Id = Guid.NewGuid(), + Text = "test", + }, _commit, true) + ]).GetAwaiter().GetResult(); + } + } + + [IterationCleanup] + public void IterationCleanup() + { + _repository.Dispose(); + _emptyDataModel.DisposeAsync().GetAwaiter().GetResult(); + } +} \ No newline at end of file diff --git a/src/SIL.Harmony.Benchmarks/MyCustomBenchmark.cs b/src/SIL.Harmony.Benchmarks/MyCustomBenchmark.cs deleted file mode 100644 index 1feaef9..0000000 --- a/src/SIL.Harmony.Benchmarks/MyCustomBenchmark.cs +++ /dev/null @@ -1,15 +0,0 @@ -using BenchmarkDotNet_GitCompare; -using BenchmarkDotNet.Attributes; - -namespace SIL.Harmony.Benchmarks; - -[SimpleJob(id: "now")] -[GitJob(gitReference: "HEAD", id: "before", baseline: true)] -public class MyCustomBenchmark -{ - [Benchmark] - public Task Test() - { - return Task.Delay(10); - } -} \ No newline at end of file diff --git a/src/SIL.Harmony.Linq2db/Linq2DbCrdtRepo.cs b/src/SIL.Harmony.Linq2db/Linq2DbCrdtRepo.cs new file mode 100644 index 0000000..486a9eb --- /dev/null +++ b/src/SIL.Harmony.Linq2db/Linq2DbCrdtRepo.cs @@ -0,0 +1,227 @@ +using System.Reflection; +using LinqToDB; +using LinqToDB.Data; +using LinqToDB.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Storage; +using Microsoft.Extensions.DependencyInjection; +using Nito.AsyncEx; +using SIL.Harmony.Core; +using SIL.Harmony.Db; +using SIL.Harmony.Resource; + +namespace SIL.Harmony.Linq2db; + +public class Linq2DbCrdtRepoFactory( + IServiceProvider serviceProvider, + ICrdtDbContextFactory dbContextFactory, CrdtRepositoryFactory factory): ICrdtRepositoryFactory +{ + public async Task CreateRepository() + { + return CreateInstance(await dbContextFactory.CreateDbContextAsync()); + } + + public ICrdtRepository CreateRepositorySync() + { + return CreateInstance(dbContextFactory.CreateDbContext()); + } + + private Linq2DbCrdtRepo CreateInstance(ICrdtDbContext dbContext) + { + return ActivatorUtilities.CreateInstance(serviceProvider, factory.CreateInstance(dbContext), dbContext); + } +} + +public class Linq2DbCrdtRepo : ICrdtRepository +{ + private readonly ICrdtRepository _original; + private readonly ICrdtDbContext _dbContext; + + public Linq2DbCrdtRepo(ICrdtRepository original, ICrdtDbContext dbContext) + { + _original = original; + _dbContext = dbContext; + } + + public ValueTask DisposeAsync() + { + return _original.DisposeAsync(); + } + + public void Dispose() + { + _original.Dispose(); + } + + public AwaitableDisposable Lock() + { + return _original.Lock(); + } + + public void ClearChangeTracker() + { + _original.ClearChangeTracker(); + } + + public async Task AddSnapshots(IEnumerable snapshots) + { + var linqToDbTable = _dbContext.Set().ToLinqToDBTable(); + var objectSnapshots = snapshots.ToArray(); + await linqToDbTable.BulkCopyAsync(objectSnapshots); + foreach (var objectSnapshot in objectSnapshots) + { + await InsertOrReplaceAsync(linqToDbTable.DataContext, objectSnapshot.Entity); + } + } + + private static readonly MethodInfo InsertOrReplaceAsyncMethodGeneric = + new Func>(DataExtensions.InsertOrReplaceAsync).Method.GetGenericMethodDefinition(); + + private Task InsertOrReplaceAsync(IDataContext dataContext, IObjectBase entity) + { + var result = InsertOrReplaceAsyncMethodGeneric.MakeGenericMethod(entity.GetType()).Invoke(null, [dataContext, entity, null, null, null, null, TableOptions.None, CancellationToken.None]); + ArgumentNullException.ThrowIfNull(result); + return (Task)result; + } + + public Task AddCommit(Commit commit) + { + return _original.AddCommit(commit); + } + + public Task AddCommits(IEnumerable commits, bool save = true) + { + return _original.AddCommits(commits, save); + } + + public Task BeginTransactionAsync() + { + return _original.BeginTransactionAsync(); + } + + public bool IsInTransaction => _original.IsInTransaction; + + public Task HasCommit(Guid commitId) + { + return _original.HasCommit(commitId); + } + + public Task<(Commit? oldestChange, Commit[] newCommits)> FilterExistingCommits(ICollection commits) + { + return _original.FilterExistingCommits(commits); + } + + public Task DeleteStaleSnapshots(Commit oldestChange) + { + return _original.DeleteStaleSnapshots(oldestChange); + } + + public Task DeleteSnapshotsAndProjectedTables() + { + return _original.DeleteSnapshotsAndProjectedTables(); + } + + public IQueryable CurrentCommits() + { + return _original.CurrentCommits(); + } + + public IQueryable CurrentSnapshots() + { + return _original.CurrentSnapshots(); + } + + public IAsyncEnumerable CurrenSimpleSnapshots(bool includeDeleted = false) + { + return _original.CurrenSimpleSnapshots(includeDeleted); + } + + public Task<(Dictionary currentSnapshots, Commit[] pendingCommits)> GetCurrentSnapshotsAndPendingCommits() + { + return _original.GetCurrentSnapshotsAndPendingCommits(); + } + + public Task FindCommitByHash(string hash) + { + return _original.FindCommitByHash(hash); + } + + public Task FindPreviousCommit(Commit commit) + { + return _original.FindPreviousCommit(commit); + } + + public Task GetCommitsAfter(Commit? commit) + { + return _original.GetCommitsAfter(commit); + } + + public Task FindSnapshot(Guid id, bool tracking = false) + { + return _original.FindSnapshot(id, tracking); + } + + public Task GetCurrentSnapshotByObjectId(Guid objectId, bool tracking = false) + { + return _original.GetCurrentSnapshotByObjectId(objectId, tracking); + } + + public Task GetObjectBySnapshotId(Guid snapshotId) + { + return _original.GetObjectBySnapshotId(snapshotId); + } + + public Task GetCurrent(Guid objectId) where T : class + { + return _original.GetCurrent(objectId); + } + + public IQueryable GetCurrentObjects() where T : class + { + return _original.GetCurrentObjects(); + } + + public Task GetCurrentSyncState() + { + return _original.GetCurrentSyncState(); + } + + public Task> GetChanges(SyncState remoteState) + { + return _original.GetChanges(remoteState); + } + + public CrdtRepository GetScopedRepository(Commit excludeChangesAfterCommit) + { + return _original.GetScopedRepository(excludeChangesAfterCommit); + } + + public HybridDateTime? GetLatestDateTime() + { + return _original.GetLatestDateTime(); + } + + public Task AddLocalResource(LocalResource localResource) + { + return _original.AddLocalResource(localResource); + } + + public IAsyncEnumerable LocalResourcesByIds(IEnumerable resourceIds) + { + return _original.LocalResourcesByIds(resourceIds); + } + + public IAsyncEnumerable LocalResources() + { + return _original.LocalResources(); + } + + public IQueryable LocalResourceIds() + { + return _original.LocalResourceIds(); + } + + public Task GetLocalResource(Guid resourceId) + { + return _original.GetLocalResource(resourceId); + } +} \ No newline at end of file diff --git a/src/SIL.Harmony.Linq2db/Linq2dbKernel.cs b/src/SIL.Harmony.Linq2db/Linq2dbKernel.cs index 77817e7..72a3b08 100644 --- a/src/SIL.Harmony.Linq2db/Linq2dbKernel.cs +++ b/src/SIL.Harmony.Linq2db/Linq2dbKernel.cs @@ -1,10 +1,13 @@ -using SIL.Harmony.Core; +using System.Text.Json; +using SIL.Harmony.Core; using LinqToDB.AspNet.Logging; using LinqToDB.EntityFrameworkCore; using LinqToDB.Mapping; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; using Microsoft.Extensions.Logging; +using SIL.Harmony.Db; namespace SIL.Harmony.Linq2db; @@ -31,6 +34,10 @@ public static DbContextOptionsBuilder UseLinqToDbCrdt(this DbContextOptionsBuild //need to use ticks here because the DateTime is stored as UTC, but the db records it as unspecified .Property(commit => commit.HybridDateTime.DateTime).HasConversionFunc(dt => dt.UtcDateTime, dt => new DateTimeOffset(dt.Ticks, TimeSpan.Zero)) + .Entity().Property(s => s.References).HasConversionFunc( + guids => JsonSerializer.Serialize(guids), + s => JsonSerializer.Deserialize(s) ?? [] + ) .Build(); var loggerFactory = provider.GetService(); @@ -41,6 +48,9 @@ public static DbContextOptionsBuilder UseLinqToDbCrdt(this DbContextOptionsBuild public static IServiceCollection AddLinq2DbRepository(this IServiceCollection services) { + services.RemoveAll(); + services.AddScoped(); + services.AddScoped(); return services; } } diff --git a/src/SIL.Harmony.Tests/DataModelTestBase.cs b/src/SIL.Harmony.Tests/DataModelTestBase.cs index 09e5b2c..c0d4c2d 100644 --- a/src/SIL.Harmony.Tests/DataModelTestBase.cs +++ b/src/SIL.Harmony.Tests/DataModelTestBase.cs @@ -20,6 +20,8 @@ public class DataModelTestBase : IAsyncLifetime public readonly SampleDbContext DbContext; protected readonly MockTimeProvider MockTimeProvider = new(); + public IServiceProvider Services => _services; + public DataModelTestBase(bool saveToDisk = false, bool alwaysValidate = true, Action? configure = null, bool performanceTest = false) : this(saveToDisk ? new SqliteConnection("Data Source=test.db") diff --git a/src/SIL.Harmony/Commit.cs b/src/SIL.Harmony/Commit.cs index fa3a39e..d9f96ef 100644 --- a/src/SIL.Harmony/Commit.cs +++ b/src/SIL.Harmony/Commit.cs @@ -14,7 +14,7 @@ protected Commit(Guid id, string hash, string parentHash, HybridDateTime hybridD ParentHash = parentHash; } - internal Commit(Guid id) : base(id) + public Commit(Guid id) : base(id) { Hash = GenerateHash(NullParentHash); ParentHash = NullParentHash; diff --git a/src/SIL.Harmony/CrdtKernel.cs b/src/SIL.Harmony/CrdtKernel.cs index f2d8a06..946ef3b 100644 --- a/src/SIL.Harmony/CrdtKernel.cs +++ b/src/SIL.Harmony/CrdtKernel.cs @@ -1,6 +1,7 @@ using System.Text.Json; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using SIL.Harmony.Db; @@ -33,7 +34,7 @@ public static IServiceCollection AddCrdtDataCore(this IServiceCollection service services.AddSingleton(sp => sp.GetRequiredService>().Value.JsonSerializerOptions); services.AddSingleton(TimeProvider.System); services.AddScoped(NewTimeProvider); - services.AddScoped(); + services.TryAddScoped(); //must use factory method because DataModel constructor is internal services.AddScoped(provider => new DataModel( provider.GetRequiredService(), diff --git a/src/SIL.Harmony/Db/CrdtRepositoryFactory.cs b/src/SIL.Harmony/Db/CrdtRepositoryFactory.cs index 605f8c1..b499e42 100644 --- a/src/SIL.Harmony/Db/CrdtRepositoryFactory.cs +++ b/src/SIL.Harmony/Db/CrdtRepositoryFactory.cs @@ -6,31 +6,34 @@ public interface ICrdtRepositoryFactory { Task CreateRepository(); ICrdtRepository CreateRepositorySync(); - Task Execute(Func> func); - ValueTask Execute(Func> func); + + public async Task Execute(Func> func) + { + await using var repo = await CreateRepository(); + return await func(repo); + } + + public async ValueTask Execute(Func> func) + { + await using var repo = await CreateRepository(); + return await func(repo); + } } public class CrdtRepositoryFactory(IServiceProvider serviceProvider, ICrdtDbContextFactory dbContextFactory) : ICrdtRepositoryFactory { public async Task CreateRepository() { - return ActivatorUtilities.CreateInstance(serviceProvider, await dbContextFactory.CreateDbContextAsync()); + return CreateInstance(await dbContextFactory.CreateDbContextAsync()); } public ICrdtRepository CreateRepositorySync() { - return ActivatorUtilities.CreateInstance(serviceProvider, dbContextFactory.CreateDbContext()); - } - - public async Task Execute(Func> func) - { - await using var repo = await CreateRepository(); - return await func(repo); + return CreateInstance(dbContextFactory.CreateDbContext()); } - public async ValueTask Execute(Func> func) + public CrdtRepository CreateInstance(ICrdtDbContext dbContext) { - await using var repo = await CreateRepository(); - return await func(repo); + return ActivatorUtilities.CreateInstance(serviceProvider, dbContext); } } \ No newline at end of file From a395f7a29a202d87afb923f63a068036e5ebfc9d Mon Sep 17 00:00:00 2001 From: Kevin Hahn Date: Thu, 24 Jul 2025 11:54:01 +0700 Subject: [PATCH 06/22] make commit hash updates explicit --- src/SIL.Harmony.Linq2db/Linq2DbCrdtRepo.cs | 5 +++++ src/SIL.Harmony/Commit.cs | 4 +++- src/SIL.Harmony/Db/CrdtRepository.cs | 10 ++++++++++ src/SIL.Harmony/Db/ICrdtRepository.cs | 1 + src/SIL.Harmony/SnapshotWorker.cs | 3 ++- 5 files changed, 21 insertions(+), 2 deletions(-) diff --git a/src/SIL.Harmony.Linq2db/Linq2DbCrdtRepo.cs b/src/SIL.Harmony.Linq2db/Linq2DbCrdtRepo.cs index 486a9eb..069a227 100644 --- a/src/SIL.Harmony.Linq2db/Linq2DbCrdtRepo.cs +++ b/src/SIL.Harmony.Linq2db/Linq2DbCrdtRepo.cs @@ -93,6 +93,11 @@ public Task AddCommits(IEnumerable commits, bool save = true) return _original.AddCommits(commits, save); } + public Task UpdateCommitHash(Guid commitId, string hash, string parentHash) + { + return _original.UpdateCommitHash(commitId, hash, parentHash); + } + public Task BeginTransactionAsync() { return _original.BeginTransactionAsync(); diff --git a/src/SIL.Harmony/Commit.cs b/src/SIL.Harmony/Commit.cs index d9f96ef..6e2c877 100644 --- a/src/SIL.Harmony/Commit.cs +++ b/src/SIL.Harmony/Commit.cs @@ -20,10 +20,12 @@ public Commit(Guid id) : base(id) ParentHash = NullParentHash; } - public void SetParentHash(string parentHash) + public bool SetParentHash(string parentHash) { + if (ParentHash == parentHash) return false; Hash = GenerateHash(parentHash); ParentHash = parentHash; + return true; } internal Commit() : this(Guid.NewGuid()) { diff --git a/src/SIL.Harmony/Db/CrdtRepository.cs b/src/SIL.Harmony/Db/CrdtRepository.cs index eee307c..f7c3776 100644 --- a/src/SIL.Harmony/Db/CrdtRepository.cs +++ b/src/SIL.Harmony/Db/CrdtRepository.cs @@ -348,6 +348,16 @@ public async Task AddCommits(IEnumerable commits, bool save = true) if (save) await _dbContext.SaveChangesAsync(); } + public async Task UpdateCommitHash(Guid commitId, string hash, string parentHash) + { + await _dbContext.Commits + .Where(c => c.Id == commitId) + .ExecuteUpdateAsync(s => s + .SetProperty(c => c.Hash, hash) + .SetProperty(c => c.ParentHash, parentHash) + ); + } + public HybridDateTime? GetLatestDateTime() { return Commits diff --git a/src/SIL.Harmony/Db/ICrdtRepository.cs b/src/SIL.Harmony/Db/ICrdtRepository.cs index 40eb650..e1303df 100644 --- a/src/SIL.Harmony/Db/ICrdtRepository.cs +++ b/src/SIL.Harmony/Db/ICrdtRepository.cs @@ -32,6 +32,7 @@ public interface ICrdtRepository : IAsyncDisposable, IDisposable CrdtRepository GetScopedRepository(Commit excludeChangesAfterCommit); Task AddCommit(Commit commit); Task AddCommits(IEnumerable commits, bool save = true); + Task UpdateCommitHash(Guid commitId, string hash, string parentHash); HybridDateTime? GetLatestDateTime(); Task AddLocalResource(LocalResource localResource); IAsyncEnumerable LocalResourcesByIds(IEnumerable resourceIds); diff --git a/src/SIL.Harmony/SnapshotWorker.cs b/src/SIL.Harmony/SnapshotWorker.cs index 22827f9..0e053bd 100644 --- a/src/SIL.Harmony/SnapshotWorker.cs +++ b/src/SIL.Harmony/SnapshotWorker.cs @@ -71,7 +71,8 @@ private async ValueTask ApplyCommitChanges(IEnumerable commits, bool upd if (updateCommitHash && previousCommitHash is not null) { //we're rewriting history, so we need to update the previous commit hash - commit.SetParentHash(previousCommitHash); + if (commit.SetParentHash(previousCommitHash)) + await _crdtRepository.UpdateCommitHash(commit.Id, hash: commit.Hash, parentHash: commit.ParentHash); } previousCommitHash = commit.Hash; From 7cefd25ddfe0fbc739618297825f806466a8df19 Mon Sep 17 00:00:00 2001 From: Kevin Hahn Date: Thu, 24 Jul 2025 14:22:55 +0700 Subject: [PATCH 07/22] add debug logging --- Directory.Packages.props | 1 + src/SIL.Harmony.Tests/DataModelTestBase.cs | 2 ++ src/SIL.Harmony.Tests/SIL.Harmony.Tests.csproj | 1 + 3 files changed, 4 insertions(+) diff --git a/Directory.Packages.props b/Directory.Packages.props index a645b0a..6fe23ad 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -7,6 +7,7 @@ + diff --git a/src/SIL.Harmony.Tests/DataModelTestBase.cs b/src/SIL.Harmony.Tests/DataModelTestBase.cs index c0d4c2d..dcab49d 100644 --- a/src/SIL.Harmony.Tests/DataModelTestBase.cs +++ b/src/SIL.Harmony.Tests/DataModelTestBase.cs @@ -7,6 +7,7 @@ using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Logging; using SIL.Harmony.Db; namespace SIL.Harmony.Tests; @@ -40,6 +41,7 @@ public DataModelTestBase(SqliteConnection connection, bool alwaysValidate = true { builder.UseSqlite(connection, true); }, performanceTest) + .AddLogging(builder => builder.AddDebug()) .Configure(config => config.AlwaysValidateCommits = alwaysValidate) .Replace(ServiceDescriptor.Singleton(MockTimeProvider)); configure?.Invoke(serviceCollection); diff --git a/src/SIL.Harmony.Tests/SIL.Harmony.Tests.csproj b/src/SIL.Harmony.Tests/SIL.Harmony.Tests.csproj index fdfa563..826bf88 100644 --- a/src/SIL.Harmony.Tests/SIL.Harmony.Tests.csproj +++ b/src/SIL.Harmony.Tests/SIL.Harmony.Tests.csproj @@ -14,6 +14,7 @@ runtime; build; native; contentfiles; analyzers; buildtransitive + From 3367d9fc0a47f5bbcdb31daffe58704e6ca245c6 Mon Sep 17 00:00:00 2001 From: Kevin Hahn Date: Thu, 24 Jul 2025 14:23:16 +0700 Subject: [PATCH 08/22] make debugging easier --- src/SIL.Harmony.Sample/CrdtSampleKernel.cs | 3 --- src/SIL.Harmony.Sample/Models/Definition.cs | 5 +++++ src/SIL.Harmony/Db/ObjectSnapshot.cs | 5 +++++ 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/SIL.Harmony.Sample/CrdtSampleKernel.cs b/src/SIL.Harmony.Sample/CrdtSampleKernel.cs index a430f68..148de50 100644 --- a/src/SIL.Harmony.Sample/CrdtSampleKernel.cs +++ b/src/SIL.Harmony.Sample/CrdtSampleKernel.cs @@ -28,9 +28,6 @@ public static IServiceCollection AddCrdtDataSample(this IServiceCollection servi optionsBuilder(builder); builder.EnableDetailedErrors(); builder.EnableSensitiveDataLogging(); -#if DEBUG - builder.LogTo(s => Debug.WriteLine(s)); -#endif }); services.AddCrdtData(config => { diff --git a/src/SIL.Harmony.Sample/Models/Definition.cs b/src/SIL.Harmony.Sample/Models/Definition.cs index 4eef28f..5327961 100644 --- a/src/SIL.Harmony.Sample/Models/Definition.cs +++ b/src/SIL.Harmony.Sample/Models/Definition.cs @@ -39,4 +39,9 @@ public IObjectBase Copy() DeletedAt = DeletedAt }; } + + public override string ToString() + { + return $"{nameof(Text)}: {Text}, {nameof(Id)}: {Id}, {nameof(WordId)}: {WordId}, {nameof(DeletedAt)}: {DeletedAt}"; + } } \ No newline at end of file diff --git a/src/SIL.Harmony/Db/ObjectSnapshot.cs b/src/SIL.Harmony/Db/ObjectSnapshot.cs index b2c31e8..54fbc28 100644 --- a/src/SIL.Harmony/Db/ObjectSnapshot.cs +++ b/src/SIL.Harmony/Db/ObjectSnapshot.cs @@ -77,4 +77,9 @@ public ObjectSnapshot(IObjectBase entity, Commit commit, bool isRoot) : this() public required Commit Commit { get; init; } CommitBase IObjectSnapshot.Commit => Commit; public required bool IsRoot { get; init; } + + public override string ToString() + { + return $"{Id} [{CommitId}] {TypeName} {EntityId} Deleted:{EntityIsDeleted}, Entity: {Entity}"; + } } From 7b35041199c3c70109a8bea9ada6370b28e82b1d Mon Sep 17 00:00:00 2001 From: Kevin Hahn Date: Thu, 24 Jul 2025 14:23:57 +0700 Subject: [PATCH 09/22] add more repo tests to find bugs with the new repo --- src/SIL.Harmony.Linq2db/Linq2DbCrdtRepo.cs | 73 +++++- src/SIL.Harmony.Linq2db/Linq2dbKernel.cs | 4 +- src/SIL.Harmony.Tests/RepositoryTests.cs | 266 ++++++++++++++++++++- 3 files changed, 321 insertions(+), 22 deletions(-) diff --git a/src/SIL.Harmony.Linq2db/Linq2DbCrdtRepo.cs b/src/SIL.Harmony.Linq2db/Linq2DbCrdtRepo.cs index 069a227..267299f 100644 --- a/src/SIL.Harmony.Linq2db/Linq2DbCrdtRepo.cs +++ b/src/SIL.Harmony.Linq2db/Linq2DbCrdtRepo.cs @@ -2,6 +2,7 @@ using LinqToDB; using LinqToDB.Data; using LinqToDB.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore.Storage; using Microsoft.Extensions.DependencyInjection; using Nito.AsyncEx; @@ -13,7 +14,8 @@ namespace SIL.Harmony.Linq2db; public class Linq2DbCrdtRepoFactory( IServiceProvider serviceProvider, - ICrdtDbContextFactory dbContextFactory, CrdtRepositoryFactory factory): ICrdtRepositoryFactory + ICrdtDbContextFactory dbContextFactory, + CrdtRepositoryFactory factory) : ICrdtRepositoryFactory { public async Task CreateRepository() { @@ -27,7 +29,9 @@ public ICrdtRepository CreateRepositorySync() private Linq2DbCrdtRepo CreateInstance(ICrdtDbContext dbContext) { - return ActivatorUtilities.CreateInstance(serviceProvider, factory.CreateInstance(dbContext), dbContext); + return ActivatorUtilities.CreateInstance(serviceProvider, + factory.CreateInstance(dbContext), + dbContext); } } @@ -64,21 +68,71 @@ public void ClearChangeTracker() public async Task AddSnapshots(IEnumerable snapshots) { + //save any pending commit changes + await _dbContext.SaveChangesAsync(); + var projectedEntityIds = new HashSet(); var linqToDbTable = _dbContext.Set().ToLinqToDBTable(); - var objectSnapshots = snapshots.ToArray(); - await linqToDbTable.BulkCopyAsync(objectSnapshots); - foreach (var objectSnapshot in objectSnapshots) + var dataContext = linqToDbTable.DataContext; + foreach (var grouping in + snapshots.GroupBy(s => s.EntityIsDeleted).OrderByDescending(g => g.Key)) //execute deletes first { - await InsertOrReplaceAsync(linqToDbTable.DataContext, objectSnapshot.Entity); + var objectSnapshots = grouping.ToArray(); + + //delete existing snapshots before we bulk recreate them + await _dbContext.Set() + .Where(s => objectSnapshots.Select(s => s.Id).Contains(s.Id)) + .ExecuteDeleteAsync(); + + await linqToDbTable.BulkCopyAsync(objectSnapshots); + + //descending to insert the most recent snapshots first, only keep the last objects by ordering by descending + //then distinct by, but lastly reverse so we insert objects in the order they should be created + foreach (var objectSnapshot in objectSnapshots.DefaultOrderDescending().DistinctBy(s => s.EntityId).Reverse()) + { + //match how ef core works by adding the snapshot to it's parent commit + objectSnapshot.Commit.Snapshots.Add(objectSnapshot); + + //ensure we skip projecting the same entity multiple times + if (!projectedEntityIds.Add(objectSnapshot.EntityId)) continue; + try + { + if (objectSnapshot.EntityIsDeleted) + { + await DeleteAsync(dataContext, objectSnapshot.Entity); + } + else + { + await InsertOrReplaceAsync(dataContext, objectSnapshot.Entity); + } + } + catch (Exception e) + { + throw new Exception("error when projecting snapshot " + objectSnapshot, e); + } + } } } private static readonly MethodInfo InsertOrReplaceAsyncMethodGeneric = - new Func>(DataExtensions.InsertOrReplaceAsync).Method.GetGenericMethodDefinition(); + new Func>( + DataExtensions.InsertOrReplaceAsync).Method.GetGenericMethodDefinition(); private Task InsertOrReplaceAsync(IDataContext dataContext, IObjectBase entity) { - var result = InsertOrReplaceAsyncMethodGeneric.MakeGenericMethod(entity.GetType()).Invoke(null, [dataContext, entity, null, null, null, null, TableOptions.None, CancellationToken.None]); + var result = InsertOrReplaceAsyncMethodGeneric.MakeGenericMethod(entity.GetType()).Invoke(null, + [dataContext, entity, null, null, null, null, TableOptions.NotSet, CancellationToken.None]); + ArgumentNullException.ThrowIfNull(result); + return (Task)result; + } + + private static readonly MethodInfo DeleteAsyncMethodGeneric = + new Func>( + DataExtensions.DeleteAsync).Method.GetGenericMethodDefinition(); + + private Task DeleteAsync(IDataContext dataContext, IObjectBase entity) + { + var result = DeleteAsyncMethodGeneric.MakeGenericMethod(entity.GetType()).Invoke(null, + [dataContext, entity, null, null, null, null, TableOptions.NotSet, CancellationToken.None]); ArgumentNullException.ThrowIfNull(result); return (Task)result; } @@ -140,7 +194,8 @@ public IAsyncEnumerable CurrenSimpleSnapshots(bool includeDelete return _original.CurrenSimpleSnapshots(includeDeleted); } - public Task<(Dictionary currentSnapshots, Commit[] pendingCommits)> GetCurrentSnapshotsAndPendingCommits() + public Task<(Dictionary currentSnapshots, Commit[] pendingCommits)> + GetCurrentSnapshotsAndPendingCommits() { return _original.GetCurrentSnapshotsAndPendingCommits(); } diff --git a/src/SIL.Harmony.Linq2db/Linq2dbKernel.cs b/src/SIL.Harmony.Linq2db/Linq2dbKernel.cs index 72a3b08..7446a84 100644 --- a/src/SIL.Harmony.Linq2db/Linq2dbKernel.cs +++ b/src/SIL.Harmony.Linq2db/Linq2dbKernel.cs @@ -1,4 +1,5 @@ using System.Text.Json; +using LinqToDB; using SIL.Harmony.Core; using LinqToDB.AspNet.Logging; using LinqToDB.EntityFrameworkCore; @@ -25,7 +26,6 @@ public static DbContextOptionsBuilder UseLinqToDbCrdt(this DbContextOptionsBuild mappingSchema = new MappingSchema(); optionsBuilder.AddMappingSchema(mappingSchema); } - new FluentMappingBuilder(mappingSchema).HasAttribute(new ColumnAttribute("DateTime", nameof(Commit.HybridDateTime) + "." + nameof(HybridDateTime.DateTime))) .HasAttribute(new ColumnAttribute(nameof(HybridDateTime.Counter), @@ -35,7 +35,7 @@ public static DbContextOptionsBuilder UseLinqToDbCrdt(this DbContextOptionsBuild .Property(commit => commit.HybridDateTime.DateTime).HasConversionFunc(dt => dt.UtcDateTime, dt => new DateTimeOffset(dt.Ticks, TimeSpan.Zero)) .Entity().Property(s => s.References).HasConversionFunc( - guids => JsonSerializer.Serialize(guids), + guids => JsonSerializer.Serialize(guids).ToUpper(),//uppercase matches EF and is required for querying by refs s => JsonSerializer.Deserialize(s) ?? [] ) .Build(); diff --git a/src/SIL.Harmony.Tests/RepositoryTests.cs b/src/SIL.Harmony.Tests/RepositoryTests.cs index 4c4eefc..7082131 100644 --- a/src/SIL.Harmony.Tests/RepositoryTests.cs +++ b/src/SIL.Harmony.Tests/RepositoryTests.cs @@ -56,9 +56,15 @@ private Commit Commit(Guid id, HybridDateTime hybridDateTime) }; } - private ObjectSnapshot Snapshot(Guid entityId, Guid commitId, HybridDateTime time) + private ObjectSnapshot Snapshot(Guid entityId, Guid commitId, HybridDateTime time, string? text = null) { - return new(new Word { Text = "test", Id = entityId }, Commit(commitId, time), false) { }; + return new(new Word { Text = text ?? "test", Id = entityId }, Commit(commitId, time), false) { }; + } + + private async Task AddSnapshots(IEnumerable snapshots) + { + _crdtDbContext.AddRange(snapshots); + await _crdtDbContext.SaveChangesAsync(); } private Guid[] OrderedIds(int count) @@ -145,20 +151,258 @@ await _repository.AddCommits([ commits.Select(c => c.Id).Should().ContainInConsecutiveOrder(ids); } + [Fact] + public async Task AddSnapshots_Works() + { + var snapshot = Snapshot(Guid.NewGuid(), Guid.NewGuid(), Time(1, 0)); + await _repository.AddCommit(snapshot.Commit); + + await _repository.AddSnapshots([snapshot]); + + _crdtDbContext.ChangeTracker.Clear(); + var actualSnapshot = await _crdtDbContext.Snapshots.SingleAsync(s => s.Id == snapshot.Id); + actualSnapshot.Should().BeEquivalentTo(snapshot, o => o + .Excluding(s => s.Entity.DbObject) + .Excluding(s => s.Commit)); + } + + [Fact] + public async Task AddSnapshots_StoresReferences() + { + var wordId = Guid.NewGuid(); + var snapshot = Snapshot(wordId, Guid.NewGuid(), Time(1, 0)); + await _repository.AddCommit(snapshot.Commit); + await _repository.AddSnapshots([snapshot]); + var commit = Commit(Guid.NewGuid(), Time(2, 0)); + await _repository.AddCommit(commit); + var defSnapshot = new ObjectSnapshot( + new Definition + { + Id = Guid.NewGuid(), + Text = "word", + Order = 0, + PartOfSpeech = "noun", + WordId = wordId + }, + commit, + true + ); + await _repository.AddSnapshots([defSnapshot]); + + _crdtDbContext.ChangeTracker.Clear(); + var actualSnapshot = await _crdtDbContext.Snapshots.SingleAsync(s => s.Id == defSnapshot.Id); + actualSnapshot.Should().BeEquivalentTo(defSnapshot, o => o + .Excluding(s => s.Entity.DbObject) + .Excluding(s => s.Commit)); + } + + [Fact] + public async Task AddSnapshots_HandlesTheSameSnapshotTwice() + { + var snapshot = Snapshot(Guid.NewGuid(), Guid.NewGuid(), Time(1, 0)); + await _repository.AddCommit(snapshot.Commit); + + await _repository.AddSnapshots([snapshot]); + await _repository.AddSnapshots([snapshot]); + _crdtDbContext.Snapshots.Should().ContainSingle(s => s.Id == snapshot.Id); + } + + [Fact] + public async Task AddSnapshots_Works_InsertsIntoProjectedTable() + { + var snapshot = Snapshot(Guid.NewGuid(), Guid.NewGuid(), Time(1, 0)); + await _repository.AddCommit(snapshot.Commit); + + await _repository.AddSnapshots([snapshot]); + + _crdtDbContext.Set().Should().Contain(w => w.Id == snapshot.EntityId); + } + + [Fact] + public async Task AddSnapshots_Works_UpdatesProjectedTable() + { + var entityId = Guid.NewGuid(); + await AddSnapshots([Snapshot(entityId, Guid.NewGuid(), Time(1, 0))]); + + var snapshot = Snapshot(entityId, Guid.NewGuid(), Time(2, 0), text: "updated"); + await _repository.AddCommit(snapshot.Commit); + await _repository.AddSnapshots([snapshot]); + + var word = await _crdtDbContext.Set().FindAsync(entityId); + word.Should().NotBeNull(); + word.Text.Should().Be("updated"); + } + + [Fact] + public async Task AddSnapshots_Works_DeletesFromProjectedTable() + { + var entityId = Guid.NewGuid(); + var firstSnapshot = Snapshot(entityId, Guid.NewGuid(), Time(1, 0)); + await _repository.AddCommit(firstSnapshot.Commit); + await _repository.AddSnapshots([firstSnapshot]); + _crdtDbContext.Set().Should().ContainSingle(w => w.Id == entityId); + + var time = Time(2, 0); + var snapshot = new ObjectSnapshot(new Word + { + Text = "test", + Id = entityId, + //mark as deleted + DeletedAt = time.DateTime + }, + Commit(Guid.NewGuid(), time), + false); + await _repository.AddCommit(snapshot.Commit); + await _repository.AddSnapshots([snapshot]); + + _crdtDbContext.Set().Should().NotContain(w => w.Id == entityId); + } + + [Fact] + public async Task AddSnapshots_DeletesFirst() + { + var firstTagId = Guid.NewGuid(); + var secondTagId = Guid.NewGuid(); + //must be the same text so that the unique constraint is violated if the delete is not executed first + var tagText = "tag"; + await AddSnapshots([ + new ObjectSnapshot(new Tag() + { + Id = firstTagId, + Text = tagText + }, + Commit(Guid.NewGuid(), Time(1, 0)), + true) + ]); + var secondCommit = Commit(Guid.NewGuid(), Time(2, 0)); + await _repository.AddCommit(secondCommit); + + //act + await _repository.AddSnapshots([ + new ObjectSnapshot(new Tag() + { + Id = secondTagId, + Text = tagText + }, + secondCommit, + true), + new ObjectSnapshot(new Tag() + { + Id = firstTagId, + Text = tagText, + DeletedAt = secondCommit.DateTime + }, + secondCommit, + false), + ]); + + //assert + _crdtDbContext.Set().Should().ContainSingle(t => t.Id == secondTagId); + } + + [Fact] + public async Task AddSnapshots_LastSnapshotWins() + { + var wordId = Guid.NewGuid(); + var firstCommit = Commit(Guid.NewGuid(), Time(1, 0)); + var secondCommit = Commit(Guid.NewGuid(), Time(2, 0)); + await _repository.AddCommit(firstCommit); + await _repository.AddCommit(secondCommit); + + await _repository.AddSnapshots([ + new ObjectSnapshot(new Word() + { + Id = wordId, + Text = "first" + }, + firstCommit, + true), + new ObjectSnapshot(new Word() + { + Id = wordId, + Text = "second" + }, + secondCommit, + false) + ]); + + _crdtDbContext.Set().Should().ContainSingle(w => w.Id == wordId).Which.Text.Should().Be("second"); + } + + [Fact] + public async Task AddSnapshots_InsertsInTheCorrectOrder() + { + var wordId = Guid.NewGuid(); + var firstCommit = Commit(Guid.NewGuid(), Time(1, 0)); + var secondCommit = Commit(Guid.NewGuid(), Time(2, 0)); + await _repository.AddCommit(firstCommit); + await _repository.AddCommit(secondCommit); + + await _repository.AddSnapshots([ + new ObjectSnapshot(new Word() + { + Id = wordId, + Text = "first" + }, + firstCommit, + true), + new ObjectSnapshot(new Definition() + { + Id = Guid.NewGuid(), + Text = "second", + WordId = wordId, + Order = 0, + PartOfSpeech = "noun" + }, + secondCommit, + false) + ]); + + _crdtDbContext.Set().Should().ContainSingle(w => w.Id == wordId).Which.Text.Should().Be("first"); + _crdtDbContext.Set().Should().ContainSingle(d => d.WordId == wordId).Which.Text.Should().Be("second"); + } + [Fact] public async Task CurrentSnapshots_Works() { - await _repository.AddSnapshots([Snapshot(Guid.NewGuid(), Guid.NewGuid(), Time(1, 0))]); + await AddSnapshots([Snapshot(Guid.NewGuid(), Guid.NewGuid(), Time(1, 0))]); var snapshots = await _repository.CurrentSnapshots().ToArrayAsync(); snapshots.Should().ContainSingle(); } + [Fact] + public async Task CurrentSnapshots_CanFilterByRefs() + { + var wordId = Guid.NewGuid(); + var snapshot = Snapshot(wordId, Guid.NewGuid(), Time(1, 0)); + await _repository.AddCommit(snapshot.Commit); + await _repository.AddSnapshots([snapshot]); + var commit = Commit(Guid.NewGuid(), Time(2, 0)); + await _repository.AddCommit(commit); + var defSnapshot = new ObjectSnapshot( + new Definition + { + Id = Guid.NewGuid(), + Text = "word", + Order = 0, + PartOfSpeech = "noun", + WordId = wordId + }, + commit, + true + ); + await _repository.AddSnapshots([defSnapshot]); + + var objectSnapshots = await _repository.CurrentSnapshots().Where(s => s.References.Contains(wordId)).ToArrayAsync(); + objectSnapshots.Should().ContainSingle().Which.Id.Should().Be(defSnapshot.Id); + } + [Fact] public async Task CurrentSnapshots_GroupsByEntityIdSortedByTime() { var entityId = Guid.NewGuid(); var expectedTime = Time(2, 0); - await _repository.AddSnapshots([ + await AddSnapshots([ Snapshot(entityId, Guid.NewGuid(), Time(1, 0)), Snapshot(entityId, Guid.NewGuid(), expectedTime), ]); @@ -170,7 +414,7 @@ await _repository.AddSnapshots([ public async Task CurrentSnapshots_GroupsByEntityIdSortedByCount() { var entityId = Guid.NewGuid(); - await _repository.AddSnapshots([ + await AddSnapshots([ Snapshot(entityId, Guid.NewGuid(), Time(1, 0)), Snapshot(entityId, Guid.NewGuid(), Time(1, 1)), ]); @@ -184,7 +428,7 @@ public async Task CurrentSnapshots_GroupsByEntityIdSortedByCommitId() var time = Time(1, 1); var entityId = Guid.NewGuid(); var ids = OrderedIds(2); - await _repository.AddSnapshots([ + await AddSnapshots([ Snapshot(entityId, ids[0], time), Snapshot(entityId, ids[1], time), ]); @@ -201,7 +445,7 @@ public async Task ScopedRepo_CurrentSnapshots_FiltersByCounter() var snapshot1 = Snapshot(entityId, commitIds[0], Time(1, 0)); var snapshot2 = Snapshot(entityId, commitIds[1], Time(2, 0)); var snapshot3 = Snapshot(entityId, commitIds[2], Time(2, 1)); - await _repository.AddSnapshots([ + await AddSnapshots([ snapshot3, snapshot1, snapshot2, @@ -226,7 +470,7 @@ public async Task ScopedRepo_CurrentSnapshots_FiltersByCommitId() var snapshot1 = Snapshot(entityId, commitIds[0], Time(1, 0)); var snapshot2 = Snapshot(entityId, commitIds[1], Time(2, 0)); var snapshot3 = Snapshot(entityId, commitIds[2], Time(2, 0)); - await _repository.AddSnapshots([ + await AddSnapshots([ snapshot3, snapshot1, snapshot2, @@ -250,7 +494,7 @@ public async Task DeleteStaleSnapshots_Works() [Fact] public async Task DeleteStaleSnapshots_DeletesSnapshotsAfterCommitByTime() { - await _repository.AddSnapshots([ + await AddSnapshots([ Snapshot(Guid.NewGuid(), Guid.NewGuid(), Time(1, 0)), Snapshot(Guid.NewGuid(), Guid.NewGuid(), Time(3, 0)), ]); @@ -263,7 +507,7 @@ await _repository.AddSnapshots([ [Fact] public async Task DeleteStaleSnapshots_DeletesSnapshotsAfterCommitByCount() { - await _repository.AddSnapshots([ + await AddSnapshots([ Snapshot(Guid.NewGuid(), Guid.NewGuid(), Time(1, 0)), Snapshot(Guid.NewGuid(), Guid.NewGuid(), Time(1, 2)), ]); @@ -279,7 +523,7 @@ public async Task DeleteStaleSnapshots_DeletesSnapshotsAfterCommitByCommitId() var time = Time(1, 1); var entityId = Guid.NewGuid(); var ids = OrderedIds(3); - await _repository.AddSnapshots([ + await AddSnapshots([ Snapshot(entityId, ids[0], time), Snapshot(entityId, ids[2], time), ]); From b5e9eb649e2a385f19353d7b5660c0ec7f1834cc Mon Sep 17 00:00:00 2001 From: Kevin Hahn Date: Thu, 24 Jul 2025 14:36:24 +0700 Subject: [PATCH 10/22] fix an issue with one test expecting the returned commit to have the created snapshots --- src/SIL.Harmony.Linq2db/Linq2DbCrdtRepo.cs | 12 +++++------- src/SIL.Harmony.Tests/DataModelSimpleChanges.cs | 2 +- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/src/SIL.Harmony.Linq2db/Linq2DbCrdtRepo.cs b/src/SIL.Harmony.Linq2db/Linq2DbCrdtRepo.cs index 267299f..45e417d 100644 --- a/src/SIL.Harmony.Linq2db/Linq2DbCrdtRepo.cs +++ b/src/SIL.Harmony.Linq2db/Linq2DbCrdtRepo.cs @@ -73,8 +73,8 @@ public async Task AddSnapshots(IEnumerable snapshots) var projectedEntityIds = new HashSet(); var linqToDbTable = _dbContext.Set().ToLinqToDBTable(); var dataContext = linqToDbTable.DataContext; - foreach (var grouping in - snapshots.GroupBy(s => s.EntityIsDeleted).OrderByDescending(g => g.Key)) //execute deletes first + foreach (var grouping in snapshots.GroupBy(s => s.EntityIsDeleted) + .OrderByDescending(g => g.Key)) //execute deletes first { var objectSnapshots = grouping.ToArray(); @@ -86,12 +86,10 @@ await _dbContext.Set() await linqToDbTable.BulkCopyAsync(objectSnapshots); //descending to insert the most recent snapshots first, only keep the last objects by ordering by descending - //then distinct by, but lastly reverse so we insert objects in the order they should be created - foreach (var objectSnapshot in objectSnapshots.DefaultOrderDescending().DistinctBy(s => s.EntityId).Reverse()) + //don't want to change the objectSnapshot order to preserve the order of the changes + var snapshotsToProject = objectSnapshots.DefaultOrderDescending().DistinctBy(s => s.EntityId).Select(s => s.Id).ToHashSet(); + foreach (var objectSnapshot in objectSnapshots.IntersectBy(snapshotsToProject, s => s.Id)) { - //match how ef core works by adding the snapshot to it's parent commit - objectSnapshot.Commit.Snapshots.Add(objectSnapshot); - //ensure we skip projecting the same entity multiple times if (!projectedEntityIds.Add(objectSnapshot.EntityId)) continue; try diff --git a/src/SIL.Harmony.Tests/DataModelSimpleChanges.cs b/src/SIL.Harmony.Tests/DataModelSimpleChanges.cs index 284d2cc..7d315eb 100644 --- a/src/SIL.Harmony.Tests/DataModelSimpleChanges.cs +++ b/src/SIL.Harmony.Tests/DataModelSimpleChanges.cs @@ -213,7 +213,7 @@ public async Task ChangesToSnapshotsAreNotSaved() word.Note = "a note"; var commit = await WriteNextChange(SetWord(_entity1Id, "after-change")); - var objectSnapshot = commit.Snapshots.Should().ContainSingle().Subject; + var objectSnapshot = await DbContext.Snapshots.SingleAsync(s => s.CommitId == commit.Id); objectSnapshot.Entity.Is().Text.Should().Be("after-change"); objectSnapshot.Entity.Is().Note.Should().BeNull(); } From 426bf8c0b7a725a586b854f63bf74b6f4b28e1c4 Mon Sep 17 00:00:00 2001 From: Kevin Hahn Date: Thu, 24 Jul 2025 15:01:25 +0700 Subject: [PATCH 11/22] add 2 benchmarks, one for all at once --- .../AddSnapshotsBenchmarks.cs | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/src/SIL.Harmony.Benchmarks/AddSnapshotsBenchmarks.cs b/src/SIL.Harmony.Benchmarks/AddSnapshotsBenchmarks.cs index 75831a0..5de6df5 100644 --- a/src/SIL.Harmony.Benchmarks/AddSnapshotsBenchmarks.cs +++ b/src/SIL.Harmony.Benchmarks/AddSnapshotsBenchmarks.cs @@ -44,7 +44,7 @@ public void IterationSetup() } [Benchmark(OperationsPerInvoke = 1000)] - public void AddSnapshots() + public void AddSnapshotsOneAtATime() { for (var i = 0; i < SnapshotCount; i++) { @@ -58,6 +58,22 @@ public void AddSnapshots() } } + [Benchmark(OperationsPerInvoke = 1000)] + public void AddSnapshotsAllAtOnce() + { + var snapshots = Enumerable.Range(0, SnapshotCount) + .Select(i => new ObjectSnapshot(new Word() + { + Id = Guid.NewGuid(), + Text = "test", + }, + _commit, + true)) + .ToArray(); + + _repository.AddSnapshots(snapshots).GetAwaiter().GetResult(); + } + [IterationCleanup] public void IterationCleanup() { From 2f3cfc6c94dbf0998da8c444ef993c32ff737086 Mon Sep 17 00:00:00 2001 From: Kevin Hahn Date: Thu, 24 Jul 2025 15:03:51 +0700 Subject: [PATCH 12/22] don't use logging during a perf test, add a DataModelTestBase.cs config for using linq2db or not --- src/SIL.Harmony.Benchmarks/AddSnapshotsBenchmarks.cs | 7 +------ src/SIL.Harmony.Sample/CrdtSampleKernel.cs | 4 +++- src/SIL.Harmony.Tests/DataModelTestBase.cs | 10 +++++----- 3 files changed, 9 insertions(+), 12 deletions(-) diff --git a/src/SIL.Harmony.Benchmarks/AddSnapshotsBenchmarks.cs b/src/SIL.Harmony.Benchmarks/AddSnapshotsBenchmarks.cs index 5de6df5..4e794fb 100644 --- a/src/SIL.Harmony.Benchmarks/AddSnapshotsBenchmarks.cs +++ b/src/SIL.Harmony.Benchmarks/AddSnapshotsBenchmarks.cs @@ -28,12 +28,7 @@ public class AddSnapshotsBenchmarks public void IterationSetup() { _emptyDataModel = new(alwaysValidate: false, - performanceTest: true, - configure: collection => - { - if (UseLinq2DbRepo) - collection.AddLinq2DbRepository(); - }); + performanceTest: true, useLinq2DbRepo: UseLinq2DbRepo); var crdtRepositoryFactory = _emptyDataModel.Services.GetRequiredService(); _repository = crdtRepositoryFactory.CreateRepositorySync(); _repository.AddCommit(_commit = new Commit(Guid.NewGuid()) diff --git a/src/SIL.Harmony.Sample/CrdtSampleKernel.cs b/src/SIL.Harmony.Sample/CrdtSampleKernel.cs index 148de50..742e0b4 100644 --- a/src/SIL.Harmony.Sample/CrdtSampleKernel.cs +++ b/src/SIL.Harmony.Sample/CrdtSampleKernel.cs @@ -16,7 +16,7 @@ public static IServiceCollection AddCrdtDataSample(this IServiceCollection servi } public static IServiceCollection AddCrdtDataSample(this IServiceCollection services, - Action optionsBuilder, bool performanceTest = false) + Action optionsBuilder, bool performanceTest = false, bool useLinq2DbRepo = false) { services.AddDbContext((provider, builder) => { @@ -76,6 +76,8 @@ public static IServiceCollection AddCrdtDataSample(this IServiceCollection servi builder.HasIndex(wt => new { wt.WordId, wt.TagId }).IsUnique(); }); }); + if (useLinq2DbRepo) + services.AddLinq2DbRepository(); return services; } } \ No newline at end of file diff --git a/src/SIL.Harmony.Tests/DataModelTestBase.cs b/src/SIL.Harmony.Tests/DataModelTestBase.cs index dcab49d..18e94e3 100644 --- a/src/SIL.Harmony.Tests/DataModelTestBase.cs +++ b/src/SIL.Harmony.Tests/DataModelTestBase.cs @@ -24,9 +24,9 @@ public class DataModelTestBase : IAsyncLifetime public IServiceProvider Services => _services; public DataModelTestBase(bool saveToDisk = false, bool alwaysValidate = true, - Action? configure = null, bool performanceTest = false) : this(saveToDisk + Action? configure = null, bool performanceTest = false, bool useLinq2DbRepo = false) : this(saveToDisk ? new SqliteConnection("Data Source=test.db") - : new SqliteConnection("Data Source=:memory:"), alwaysValidate, configure, performanceTest) + : new SqliteConnection("Data Source=:memory:"), alwaysValidate, configure, performanceTest, useLinq2DbRepo) { } @@ -34,16 +34,16 @@ public DataModelTestBase() : this(new SqliteConnection("Data Source=:memory:")) { } - public DataModelTestBase(SqliteConnection connection, bool alwaysValidate = true, Action? configure = null, bool performanceTest = false) + public DataModelTestBase(SqliteConnection connection, bool alwaysValidate = true, Action? configure = null, bool performanceTest = false, bool useLinq2DbRepo = false) { _performanceTest = performanceTest; var serviceCollection = new ServiceCollection().AddCrdtDataSample(builder => { builder.UseSqlite(connection, true); - }, performanceTest) - .AddLogging(builder => builder.AddDebug()) + }, performanceTest, useLinq2DbRepo) .Configure(config => config.AlwaysValidateCommits = alwaysValidate) .Replace(ServiceDescriptor.Singleton(MockTimeProvider)); + if (!performanceTest) serviceCollection.AddLogging(builder => builder.AddDebug()); configure?.Invoke(serviceCollection); _services = serviceCollection.BuildServiceProvider(); DbContext = _services.GetRequiredService(); From 4a304a13c37ceee825fc0d5ed27a90152f045375 Mon Sep 17 00:00:00 2001 From: Kevin Hahn Date: Thu, 24 Jul 2025 15:16:16 +0700 Subject: [PATCH 13/22] always run repo tests for both versions --- src/SIL.Harmony.Tests/RepositoryTests.cs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/SIL.Harmony.Tests/RepositoryTests.cs b/src/SIL.Harmony.Tests/RepositoryTests.cs index 7082131..95f09ac 100644 --- a/src/SIL.Harmony.Tests/RepositoryTests.cs +++ b/src/SIL.Harmony.Tests/RepositoryTests.cs @@ -9,21 +9,26 @@ namespace SIL.Harmony.Tests; +public class RepositoryTests_Linq2Db() : RepositoryTests(true); + public class RepositoryTests : IAsyncLifetime { private readonly ServiceProvider _services; private readonly ICrdtRepository _repository; private readonly SampleDbContext _crdtDbContext; - public RepositoryTests() + protected RepositoryTests(bool useLinq2DbRepo) { _services = new ServiceCollection() - .AddCrdtDataSample(":memory:") + .AddCrdtDataSample(builder => builder.UseSqlite("Data Source=:memory:"), useLinq2DbRepo: useLinq2DbRepo) .BuildServiceProvider(); _repository = _services.GetRequiredService().CreateRepositorySync(); _crdtDbContext = _services.GetRequiredService(); } + public RepositoryTests() : this(false) + { + } public async Task InitializeAsync() { From 9a9901a49034aa39d155daefd182be4845f7b4ea Mon Sep 17 00:00:00 2001 From: Kevin Hahn Date: Thu, 24 Jul 2025 16:23:40 +0700 Subject: [PATCH 14/22] add dotTrace for easy profiling --- Directory.Packages.props | 9 +++++---- src/SIL.Harmony.Benchmarks/AddChangeBenchmarks.cs | 3 +++ src/SIL.Harmony.Benchmarks/AddSnapshotsBenchmarks.cs | 1 + src/SIL.Harmony.Benchmarks/SIL.Harmony.Benchmarks.csproj | 1 + 4 files changed, 10 insertions(+), 4 deletions(-) diff --git a/Directory.Packages.props b/Directory.Packages.props index 6fe23ad..88b3656 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -3,10 +3,11 @@ true + - - - + + + @@ -25,6 +26,6 @@ - + \ No newline at end of file diff --git a/src/SIL.Harmony.Benchmarks/AddChangeBenchmarks.cs b/src/SIL.Harmony.Benchmarks/AddChangeBenchmarks.cs index 587aa6a..fefa16d 100644 --- a/src/SIL.Harmony.Benchmarks/AddChangeBenchmarks.cs +++ b/src/SIL.Harmony.Benchmarks/AddChangeBenchmarks.cs @@ -1,5 +1,6 @@ using BenchmarkDotNet_GitCompare; using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Diagnostics.dotTrace; using BenchmarkDotNet.Engines; using SIL.Harmony.Linq2db; using SIL.Harmony.Tests; @@ -7,6 +8,8 @@ namespace SIL.Harmony.Benchmarks; [SimpleJob(RunStrategy.Throughput)] +[MemoryDiagnoser] +// [DotTraceDiagnoser] // [GitJob(gitReference: "HEAD", id: "before", baseline: true)] public class AddChangeBenchmarks { diff --git a/src/SIL.Harmony.Benchmarks/AddSnapshotsBenchmarks.cs b/src/SIL.Harmony.Benchmarks/AddSnapshotsBenchmarks.cs index 4e794fb..e8c6574 100644 --- a/src/SIL.Harmony.Benchmarks/AddSnapshotsBenchmarks.cs +++ b/src/SIL.Harmony.Benchmarks/AddSnapshotsBenchmarks.cs @@ -11,6 +11,7 @@ namespace SIL.Harmony.Benchmarks; [SimpleJob(RunStrategy.Throughput)] +[MemoryDiagnoser] // [GitJob(gitReference: "HEAD", id: "before", baseline: true)] public class AddSnapshotsBenchmarks { diff --git a/src/SIL.Harmony.Benchmarks/SIL.Harmony.Benchmarks.csproj b/src/SIL.Harmony.Benchmarks/SIL.Harmony.Benchmarks.csproj index b7818b3..21b070d 100644 --- a/src/SIL.Harmony.Benchmarks/SIL.Harmony.Benchmarks.csproj +++ b/src/SIL.Harmony.Benchmarks/SIL.Harmony.Benchmarks.csproj @@ -9,6 +9,7 @@ + From 53a46ed8910a63c65c8e72d357d8270cbc7b3dde Mon Sep 17 00:00:00 2001 From: Kevin Hahn Date: Thu, 24 Jul 2025 17:03:39 +0700 Subject: [PATCH 15/22] use new package version --- Directory.Packages.props | 2 +- src/SIL.Harmony.Linq2db/Linq2dbKernel.cs | 3 +-- src/SIL.Harmony.Linq2db/SIL.Harmony.Linq2db.csproj | 2 +- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/Directory.Packages.props b/Directory.Packages.props index 88b3656..f2232f0 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -5,6 +5,7 @@ + @@ -25,7 +26,6 @@ - \ No newline at end of file diff --git a/src/SIL.Harmony.Linq2db/Linq2dbKernel.cs b/src/SIL.Harmony.Linq2db/Linq2dbKernel.cs index 7446a84..adc18fa 100644 --- a/src/SIL.Harmony.Linq2db/Linq2dbKernel.cs +++ b/src/SIL.Harmony.Linq2db/Linq2dbKernel.cs @@ -1,8 +1,7 @@ using System.Text.Json; -using LinqToDB; using SIL.Harmony.Core; -using LinqToDB.AspNet.Logging; using LinqToDB.EntityFrameworkCore; +using LinqToDB.Extensions.Logging; using LinqToDB.Mapping; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.DependencyInjection; diff --git a/src/SIL.Harmony.Linq2db/SIL.Harmony.Linq2db.csproj b/src/SIL.Harmony.Linq2db/SIL.Harmony.Linq2db.csproj index d530422..1453659 100644 --- a/src/SIL.Harmony.Linq2db/SIL.Harmony.Linq2db.csproj +++ b/src/SIL.Harmony.Linq2db/SIL.Harmony.Linq2db.csproj @@ -10,8 +10,8 @@ - + From 7c3b8d2d954e52d9ca9875057830759cfc50c436 Mon Sep 17 00:00:00 2001 From: Kevin Hahn Date: Thu, 24 Jul 2025 17:07:44 +0700 Subject: [PATCH 16/22] remove extra check projecting entities --- src/SIL.Harmony.Linq2db/Linq2DbCrdtRepo.cs | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/SIL.Harmony.Linq2db/Linq2DbCrdtRepo.cs b/src/SIL.Harmony.Linq2db/Linq2DbCrdtRepo.cs index 45e417d..d9b5fe0 100644 --- a/src/SIL.Harmony.Linq2db/Linq2DbCrdtRepo.cs +++ b/src/SIL.Harmony.Linq2db/Linq2DbCrdtRepo.cs @@ -70,7 +70,6 @@ public async Task AddSnapshots(IEnumerable snapshots) { //save any pending commit changes await _dbContext.SaveChangesAsync(); - var projectedEntityIds = new HashSet(); var linqToDbTable = _dbContext.Set().ToLinqToDBTable(); var dataContext = linqToDbTable.DataContext; foreach (var grouping in snapshots.GroupBy(s => s.EntityIsDeleted) @@ -90,8 +89,6 @@ await _dbContext.Set() var snapshotsToProject = objectSnapshots.DefaultOrderDescending().DistinctBy(s => s.EntityId).Select(s => s.Id).ToHashSet(); foreach (var objectSnapshot in objectSnapshots.IntersectBy(snapshotsToProject, s => s.Id)) { - //ensure we skip projecting the same entity multiple times - if (!projectedEntityIds.Add(objectSnapshot.EntityId)) continue; try { if (objectSnapshot.EntityIsDeleted) From e29dd8e2b33233944dec84bbab50e428b5e1784d Mon Sep 17 00:00:00 2001 From: Kevin Hahn Date: Thu, 24 Jul 2025 17:18:11 +0700 Subject: [PATCH 17/22] add a second benchmark for all changes --- src/SIL.Harmony.Benchmarks/AddChangeBenchmarks.cs | 15 ++++++++++----- src/SIL.Harmony.Tests/DataModelTestBase.cs | 2 +- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/src/SIL.Harmony.Benchmarks/AddChangeBenchmarks.cs b/src/SIL.Harmony.Benchmarks/AddChangeBenchmarks.cs index fefa16d..1a25b6f 100644 --- a/src/SIL.Harmony.Benchmarks/AddChangeBenchmarks.cs +++ b/src/SIL.Harmony.Benchmarks/AddChangeBenchmarks.cs @@ -14,6 +14,7 @@ namespace SIL.Harmony.Benchmarks; public class AddChangeBenchmarks { private DataModelTestBase _emptyDataModel = null!; + private Guid _clientId = Guid.NewGuid(); [Params(200)] public int ChangeCount { get; set; } @@ -23,11 +24,7 @@ public class AddChangeBenchmarks [IterationSetup] public void IterationSetup() { - _emptyDataModel = new(alwaysValidate: false, performanceTest: true, configure: collection => - { - if (UseLinq2DbRepo) - collection.AddLinq2DbRepository(); - }); + _emptyDataModel = new(alwaysValidate: false, performanceTest: true, useLinq2DbRepo: UseLinq2DbRepo); _emptyDataModel.WriteNextChange(_emptyDataModel.SetWord(Guid.NewGuid(), "entity1")).GetAwaiter().GetResult(); } @@ -43,6 +40,14 @@ public List AddChanges() return commits; } + [Benchmark(OperationsPerInvoke = 200)] + public Commit AddChangesAllAtOnce() + { + return _emptyDataModel.WriteChange(_clientId, DateTimeOffset.Now, Enumerable.Range(0, ChangeCount) + .Select(i => + _emptyDataModel.SetWord(Guid.NewGuid(), "entity1"))).Result; + } + [IterationCleanup] public void IterationCleanup() { diff --git a/src/SIL.Harmony.Tests/DataModelTestBase.cs b/src/SIL.Harmony.Tests/DataModelTestBase.cs index 18e94e3..e5797a5 100644 --- a/src/SIL.Harmony.Tests/DataModelTestBase.cs +++ b/src/SIL.Harmony.Tests/DataModelTestBase.cs @@ -101,7 +101,7 @@ protected async ValueTask WriteChange(Guid clientId, return await WriteChange(clientId, dateTime, [change], add); } - protected async ValueTask WriteChange(Guid clientId, + public async ValueTask WriteChange(Guid clientId, DateTimeOffset dateTime, IEnumerable changes, bool add = true) From 9ad734c48aaa4dd6264d2975c90ff1110e320564 Mon Sep 17 00:00:00 2001 From: Kevin Hahn Date: Thu, 28 Aug 2025 16:44:53 +0700 Subject: [PATCH 18/22] disable linq2db logging when running as part of a perf test --- src/SIL.Harmony.Linq2db/Linq2dbKernel.cs | 11 +++++++---- src/SIL.Harmony.Sample/CrdtSampleKernel.cs | 2 +- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/SIL.Harmony.Linq2db/Linq2dbKernel.cs b/src/SIL.Harmony.Linq2db/Linq2dbKernel.cs index adc18fa..557a05f 100644 --- a/src/SIL.Harmony.Linq2db/Linq2dbKernel.cs +++ b/src/SIL.Harmony.Linq2db/Linq2dbKernel.cs @@ -13,7 +13,7 @@ namespace SIL.Harmony.Linq2db; public static class Linq2dbKernel { - public static DbContextOptionsBuilder UseLinqToDbCrdt(this DbContextOptionsBuilder builder, IServiceProvider provider) + public static DbContextOptionsBuilder UseLinqToDbCrdt(this DbContextOptionsBuilder builder, IServiceProvider provider, bool useLogging = true) { LinqToDBForEFTools.Initialize(); return builder.UseLinqToDB(optionsBuilder => @@ -39,9 +39,12 @@ public static DbContextOptionsBuilder UseLinqToDbCrdt(this DbContextOptionsBuild ) .Build(); - var loggerFactory = provider.GetService(); - if (loggerFactory is not null) - optionsBuilder.AddCustomOptions(dataOptions => dataOptions.UseLoggerFactory(loggerFactory)); + if (useLogging) + { + var loggerFactory = provider.GetService(); + if (loggerFactory is not null) + optionsBuilder.AddCustomOptions(dataOptions => dataOptions.UseLoggerFactory(loggerFactory)); + } }); } diff --git a/src/SIL.Harmony.Sample/CrdtSampleKernel.cs b/src/SIL.Harmony.Sample/CrdtSampleKernel.cs index 742e0b4..ee21d26 100644 --- a/src/SIL.Harmony.Sample/CrdtSampleKernel.cs +++ b/src/SIL.Harmony.Sample/CrdtSampleKernel.cs @@ -24,7 +24,7 @@ public static IServiceCollection AddCrdtDataSample(this IServiceCollection servi //this can show up as the second instance using the JsonSerializerOptions from the first container //only needed for testing scenarios builder.EnableServiceProviderCaching(performanceTest); - builder.UseLinqToDbCrdt(provider); + builder.UseLinqToDbCrdt(provider, !performanceTest); optionsBuilder(builder); builder.EnableDetailedErrors(); builder.EnableSensitiveDataLogging(); From d7414c33491126252073962806407289e6f84d03 Mon Sep 17 00:00:00 2001 From: Kevin Hahn Date: Thu, 28 Aug 2025 17:21:15 +0700 Subject: [PATCH 19/22] increase change count in add change benchmarks --- Directory.Packages.props | 1 + src/SIL.Harmony.Benchmarks/AddChangeBenchmarks.cs | 14 +++++++------- .../SIL.Harmony.Benchmarks.csproj | 1 + 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/Directory.Packages.props b/Directory.Packages.props index f2232f0..9a977b7 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -3,6 +3,7 @@ true + diff --git a/src/SIL.Harmony.Benchmarks/AddChangeBenchmarks.cs b/src/SIL.Harmony.Benchmarks/AddChangeBenchmarks.cs index 1a25b6f..23a302d 100644 --- a/src/SIL.Harmony.Benchmarks/AddChangeBenchmarks.cs +++ b/src/SIL.Harmony.Benchmarks/AddChangeBenchmarks.cs @@ -1,5 +1,6 @@ using BenchmarkDotNet_GitCompare; using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Diagnostics.dotMemory; using BenchmarkDotNet.Diagnostics.dotTrace; using BenchmarkDotNet.Engines; using SIL.Harmony.Linq2db; @@ -9,15 +10,14 @@ namespace SIL.Harmony.Benchmarks; [SimpleJob(RunStrategy.Throughput)] [MemoryDiagnoser] +// [DotMemoryDiagnoser] // [DotTraceDiagnoser] // [GitJob(gitReference: "HEAD", id: "before", baseline: true)] public class AddChangeBenchmarks { private DataModelTestBase _emptyDataModel = null!; private Guid _clientId = Guid.NewGuid(); - - [Params(200)] - public int ChangeCount { get; set; } + public const int ActualChangeCount = 2000; [Params(true, false)] public bool UseLinq2DbRepo { get; set; } @@ -28,11 +28,11 @@ public void IterationSetup() _emptyDataModel.WriteNextChange(_emptyDataModel.SetWord(Guid.NewGuid(), "entity1")).GetAwaiter().GetResult(); } - [Benchmark(OperationsPerInvoke = 200)] + [Benchmark(OperationsPerInvoke = ActualChangeCount)] public List AddChanges() { var commits = new List(); - for (var i = 0; i < ChangeCount; i++) + for (var i = 0; i < ActualChangeCount; i++) { commits.Add(_emptyDataModel.WriteNextChange(_emptyDataModel.SetWord(Guid.NewGuid(), "entity1")).Result); } @@ -40,10 +40,10 @@ public List AddChanges() return commits; } - [Benchmark(OperationsPerInvoke = 200)] + [Benchmark(OperationsPerInvoke = ActualChangeCount)] public Commit AddChangesAllAtOnce() { - return _emptyDataModel.WriteChange(_clientId, DateTimeOffset.Now, Enumerable.Range(0, ChangeCount) + return _emptyDataModel.WriteChange(_clientId, DateTimeOffset.Now, Enumerable.Range(0, ActualChangeCount) .Select(i => _emptyDataModel.SetWord(Guid.NewGuid(), "entity1"))).Result; } diff --git a/src/SIL.Harmony.Benchmarks/SIL.Harmony.Benchmarks.csproj b/src/SIL.Harmony.Benchmarks/SIL.Harmony.Benchmarks.csproj index 21b070d..4dd49f2 100644 --- a/src/SIL.Harmony.Benchmarks/SIL.Harmony.Benchmarks.csproj +++ b/src/SIL.Harmony.Benchmarks/SIL.Harmony.Benchmarks.csproj @@ -9,6 +9,7 @@ + From c593cc01048d71dcb4f10e48f6ec37c03d724482 Mon Sep 17 00:00:00 2001 From: Kevin Hahn Date: Thu, 28 Aug 2025 17:21:52 +0700 Subject: [PATCH 20/22] add test for `GetCurrentSnapshotByObjectId` and change it to use a compiled query for better performance --- .../DbContextTests.VerifyModel.verified.txt | 2 +- src/SIL.Harmony.Tests/RepositoryTests.cs | 18 ++++++++++++++++++ src/SIL.Harmony/Db/CrdtRepository.cs | 11 +++++++++++ 3 files changed, 30 insertions(+), 1 deletion(-) diff --git a/src/SIL.Harmony.Tests/DbContextTests.VerifyModel.verified.txt b/src/SIL.Harmony.Tests/DbContextTests.VerifyModel.verified.txt index 7ad2c2d..71edd8b 100644 --- a/src/SIL.Harmony.Tests/DbContextTests.VerifyModel.verified.txt +++ b/src/SIL.Harmony.Tests/DbContextTests.VerifyModel.verified.txt @@ -237,4 +237,4 @@ Relational:ViewName: Relational:ViewSchema: Annotations: - ProductVersion: 9.0.4 \ No newline at end of file + ProductVersion: 9.0.7 \ No newline at end of file diff --git a/src/SIL.Harmony.Tests/RepositoryTests.cs b/src/SIL.Harmony.Tests/RepositoryTests.cs index 95f09ac..d356def 100644 --- a/src/SIL.Harmony.Tests/RepositoryTests.cs +++ b/src/SIL.Harmony.Tests/RepositoryTests.cs @@ -4,6 +4,7 @@ using SIL.Harmony.Tests.Mocks; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; using SIL.Harmony.Changes; using SIL.Harmony.Sample.Changes; @@ -21,6 +22,7 @@ protected RepositoryTests(bool useLinq2DbRepo) { _services = new ServiceCollection() .AddCrdtDataSample(builder => builder.UseSqlite("Data Source=:memory:"), useLinq2DbRepo: useLinq2DbRepo) + .AddLogging(c => c.AddDebug()) .BuildServiceProvider(); _repository = _services.GetRequiredService().CreateRepositorySync(); @@ -592,4 +594,20 @@ public async Task FindPreviousCommit_ReturnsNullForFirstCommit() var previousCommit = await _repository.FindPreviousCommit(commit1); previousCommit.Should().BeNull(); } + + [Fact] + public async Task GetCurrentSnapshotByObjectId_Works() + { + var entityId = Guid.NewGuid(); + var snapshot = Snapshot(entityId, Guid.NewGuid(), Time(1, 0)); + await _repository.AddCommit(snapshot.Commit); + await _repository.AddSnapshots([snapshot]); + + _crdtDbContext.ChangeTracker.Clear(); + var actualSnapshot = await _repository.GetCurrentSnapshotByObjectId(entityId); + actualSnapshot.Should().BeEquivalentTo(snapshot, + o => o + .Excluding(s => s.Entity.DbObject) + .Excluding(s => s.Commit)); + } } diff --git a/src/SIL.Harmony/Db/CrdtRepository.cs b/src/SIL.Harmony/Db/CrdtRepository.cs index f7c3776..6c14450 100644 --- a/src/SIL.Harmony/Db/CrdtRepository.cs +++ b/src/SIL.Harmony/Db/CrdtRepository.cs @@ -222,8 +222,19 @@ public async Task GetCommitsAfter(Commit? commit) .SingleOrDefaultAsync(s => s.Id == id); } + private readonly Func> GetCurrentSnapshotByObjectIdQuery = + EF.CompileAsyncQuery((DbContext dbContext, Guid objectId) => + dbContext.Set() + .AsTracking(QueryTrackingBehavior.TrackAll) + .Include(s => s.Commit) + .OrderBy(c => c.Commit.HybridDateTime.DateTime) + .ThenBy(c => c.Commit.HybridDateTime.Counter) + .ThenBy(c => c.Commit.Id) + .LastOrDefault(s => s.EntityId == objectId)); + public async Task GetCurrentSnapshotByObjectId(Guid objectId, bool tracking = false) { + if (tracking) return await GetCurrentSnapshotByObjectIdQuery(_dbContext.ChangeTracker.Context, objectId); return await Snapshots .AsTracking(tracking) .Include(s => s.Commit) From cd78ec9bb6a98f8c2cb65bc945acedd77e731150 Mon Sep 17 00:00:00 2001 From: Kevin Hahn Date: Fri, 29 Aug 2025 09:10:43 +0700 Subject: [PATCH 21/22] lower dotnet version used for benchmarks --- src/SIL.Harmony.Benchmarks/SIL.Harmony.Benchmarks.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/SIL.Harmony.Benchmarks/SIL.Harmony.Benchmarks.csproj b/src/SIL.Harmony.Benchmarks/SIL.Harmony.Benchmarks.csproj index 4dd49f2..db0c41b 100644 --- a/src/SIL.Harmony.Benchmarks/SIL.Harmony.Benchmarks.csproj +++ b/src/SIL.Harmony.Benchmarks/SIL.Harmony.Benchmarks.csproj @@ -2,7 +2,7 @@ Exe - net9.0 + net8.0 enable enable From 733b6225a4bc83833f7697d3b7d8e702672e2c18 Mon Sep 17 00:00:00 2001 From: Kevin Hahn Date: Fri, 29 Aug 2025 09:19:37 +0700 Subject: [PATCH 22/22] resolve merge conflicts --- src/SIL.Harmony.Linq2db/Linq2DbCrdtRepo.cs | 10 +++++-- src/SIL.Harmony/Db/CrdtRepository.cs | 32 +-------------------- src/SIL.Harmony/Db/CrdtRepositoryFactory.cs | 8 +++++- src/SIL.Harmony/Db/ICrdtRepository.cs | 3 +- 4 files changed, 18 insertions(+), 35 deletions(-) diff --git a/src/SIL.Harmony.Linq2db/Linq2DbCrdtRepo.cs b/src/SIL.Harmony.Linq2db/Linq2DbCrdtRepo.cs index d9b5fe0..7846510 100644 --- a/src/SIL.Harmony.Linq2db/Linq2DbCrdtRepo.cs +++ b/src/SIL.Harmony.Linq2db/Linq2DbCrdtRepo.cs @@ -245,9 +245,10 @@ public Task> GetChanges(SyncState remoteState) return _original.GetChanges(remoteState); } - public CrdtRepository GetScopedRepository(Commit excludeChangesAfterCommit) + public ICrdtRepository GetScopedRepository(Commit excludeChangesAfterCommit) { - return _original.GetScopedRepository(excludeChangesAfterCommit); + var inner = _original.GetScopedRepository(excludeChangesAfterCommit); + return new Linq2DbCrdtRepo(inner, _dbContext); } public HybridDateTime? GetLatestDateTime() @@ -260,6 +261,11 @@ public Task AddLocalResource(LocalResource localResource) return _original.AddLocalResource(localResource); } + public Task DeleteLocalResource(Guid id) + { + return _original.DeleteLocalResource(id); + } + public IAsyncEnumerable LocalResourcesByIds(IEnumerable resourceIds) { return _original.LocalResourcesByIds(resourceIds); diff --git a/src/SIL.Harmony/Db/CrdtRepository.cs b/src/SIL.Harmony/Db/CrdtRepository.cs index f4f4433..9ff5c31 100644 --- a/src/SIL.Harmony/Db/CrdtRepository.cs +++ b/src/SIL.Harmony/Db/CrdtRepository.cs @@ -12,36 +12,6 @@ namespace SIL.Harmony.Db; -internal class CrdtRepositoryFactory(IServiceProvider serviceProvider, ICrdtDbContextFactory dbContextFactory) -{ - public async Task CreateRepository() - { - return ActivatorUtilities.CreateInstance(serviceProvider, await dbContextFactory.CreateDbContextAsync()); - } - - public ICrdtRepository CreateRepositorySync() - { - return ActivatorUtilities.CreateInstance(serviceProvider, dbContextFactory.CreateDbContext()); - } - - public async Task Execute(Func> func) - { - await using var repo = await CreateRepository(); - return await func(repo); - } - public async Task Execute(Func func) - { - await using var repo = await CreateRepository(); - await func(repo); - } - - public async ValueTask Execute(Func> func) - { - await using var repo = await CreateRepository(); - return await func(repo); - } -} - internal class CrdtRepository : IDisposable, IAsyncDisposable, ICrdtRepository { private static readonly ConcurrentDictionary Locks = new(); @@ -372,7 +342,7 @@ private async ValueTask ProjectSnapshot(ObjectSnapshot objectSnapshot) return entity is not null ? _dbContext.Entry(entity) : null; } - public CrdtRepository GetScopedRepository(Commit excludeChangesAfterCommit) + public ICrdtRepository GetScopedRepository(Commit excludeChangesAfterCommit) { return new CrdtRepository(_dbContext, _crdtConfig, _logger, excludeChangesAfterCommit); } diff --git a/src/SIL.Harmony/Db/CrdtRepositoryFactory.cs b/src/SIL.Harmony/Db/CrdtRepositoryFactory.cs index b499e42..f081ca7 100644 --- a/src/SIL.Harmony/Db/CrdtRepositoryFactory.cs +++ b/src/SIL.Harmony/Db/CrdtRepositoryFactory.cs @@ -13,6 +13,12 @@ public async Task Execute(Func> func) return await func(repo); } + public async Task Execute(Func func) + { + await using var repo = await CreateRepository(); + await func(repo); + } + public async ValueTask Execute(Func> func) { await using var repo = await CreateRepository(); @@ -32,7 +38,7 @@ public ICrdtRepository CreateRepositorySync() return CreateInstance(dbContextFactory.CreateDbContext()); } - public CrdtRepository CreateInstance(ICrdtDbContext dbContext) + public ICrdtRepository CreateInstance(ICrdtDbContext dbContext) { return ActivatorUtilities.CreateInstance(serviceProvider, dbContext); } diff --git a/src/SIL.Harmony/Db/ICrdtRepository.cs b/src/SIL.Harmony/Db/ICrdtRepository.cs index e1303df..6db87da 100644 --- a/src/SIL.Harmony/Db/ICrdtRepository.cs +++ b/src/SIL.Harmony/Db/ICrdtRepository.cs @@ -29,7 +29,7 @@ public interface ICrdtRepository : IAsyncDisposable, IDisposable Task GetCurrentSyncState(); Task> GetChanges(SyncState remoteState); Task AddSnapshots(IEnumerable snapshots); - CrdtRepository GetScopedRepository(Commit excludeChangesAfterCommit); + ICrdtRepository GetScopedRepository(Commit excludeChangesAfterCommit); Task AddCommit(Commit commit); Task AddCommits(IEnumerable commits, bool save = true); Task UpdateCommitHash(Guid commitId, string hash, string parentHash); @@ -44,4 +44,5 @@ public interface ICrdtRepository : IAsyncDisposable, IDisposable IQueryable LocalResourceIds(); Task GetLocalResource(Guid resourceId); + Task DeleteLocalResource(Guid id); } \ No newline at end of file