diff --git a/Directory.Packages.props b/Directory.Packages.props index 4263bab..9a977b7 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -3,13 +3,18 @@ true - - - + + + + + + + + - + @@ -22,7 +27,6 @@ - - + - + \ No newline at end of file diff --git a/README.md b/README.md index 7b8e061..ceaabd0 100644 --- a/README.md +++ b/README.md @@ -186,3 +186,10 @@ NuGet package versions are calculated from a combination of tags and commit mess * `+semver: minor` or `+semver: feature` - update minor version number, reset patch to 0 (so 2.3.1 would become 2.4.0) * Anything else, including no `+semver` lines at all - update patch version number (so 2.3.1 would become 2.3.2) * If you want to include `+semver` lines, then `+semver: patch` or `+semver: fix` are the standard ways to increment a patch version bump, but the patch version will be bumped regardless as long as there is at least one commit since the most recent tag. + +### Run benchmarks + +docs: https://benchmarkdotnet.org/articles/guides/console-args.html +```bash +dotnet run --project ./src/SIL.Harmony.Benchmarks/SIL.Harmony.Benchmarks.csproj -c Release -- --filter * +``` \ No newline at end of file diff --git a/harmony.sln b/harmony.sln index 8fc9d10..ab8ed43 100644 --- a/harmony.sln +++ b/harmony.sln @@ -21,6 +21,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution src\.editorconfig = src\.editorconfig EndProjectSection EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SIL.Harmony.Benchmarks", "src\SIL.Harmony.Benchmarks\SIL.Harmony.Benchmarks.csproj", "{96FABEBB-5A18-46D5-8530-84B9FFF5442C}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -54,5 +56,9 @@ Global {8EB807F6-C548-4016-856E-2ACCA5603036}.Debug|Any CPU.Build.0 = Debug|Any CPU {8EB807F6-C548-4016-856E-2ACCA5603036}.Release|Any CPU.ActiveCfg = Release|Any CPU {8EB807F6-C548-4016-856E-2ACCA5603036}.Release|Any CPU.Build.0 = Release|Any CPU + {96FABEBB-5A18-46D5-8530-84B9FFF5442C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {96FABEBB-5A18-46D5-8530-84B9FFF5442C}.Debug|Any CPU.Build.0 = Debug|Any CPU + {96FABEBB-5A18-46D5-8530-84B9FFF5442C}.Release|Any CPU.ActiveCfg = Release|Any CPU + {96FABEBB-5A18-46D5-8530-84B9FFF5442C}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection EndGlobal diff --git a/src/SIL.Harmony.Benchmarks/AddChangeBenchmarks.cs b/src/SIL.Harmony.Benchmarks/AddChangeBenchmarks.cs new file mode 100644 index 0000000..23a302d --- /dev/null +++ b/src/SIL.Harmony.Benchmarks/AddChangeBenchmarks.cs @@ -0,0 +1,56 @@ +using BenchmarkDotNet_GitCompare; +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Diagnostics.dotMemory; +using BenchmarkDotNet.Diagnostics.dotTrace; +using BenchmarkDotNet.Engines; +using SIL.Harmony.Linq2db; +using SIL.Harmony.Tests; + +namespace SIL.Harmony.Benchmarks; + +[SimpleJob(RunStrategy.Throughput)] +[MemoryDiagnoser] +// [DotMemoryDiagnoser] +// [DotTraceDiagnoser] +// [GitJob(gitReference: "HEAD", id: "before", baseline: true)] +public class AddChangeBenchmarks +{ + private DataModelTestBase _emptyDataModel = null!; + private Guid _clientId = Guid.NewGuid(); + public const int ActualChangeCount = 2000; + [Params(true, false)] + public bool UseLinq2DbRepo { get; set; } + + [IterationSetup] + public void IterationSetup() + { + _emptyDataModel = new(alwaysValidate: false, performanceTest: true, useLinq2DbRepo: UseLinq2DbRepo); + _emptyDataModel.WriteNextChange(_emptyDataModel.SetWord(Guid.NewGuid(), "entity1")).GetAwaiter().GetResult(); + } + + [Benchmark(OperationsPerInvoke = ActualChangeCount)] + public List AddChanges() + { + var commits = new List(); + for (var i = 0; i < ActualChangeCount; i++) + { + commits.Add(_emptyDataModel.WriteNextChange(_emptyDataModel.SetWord(Guid.NewGuid(), "entity1")).Result); + } + + return commits; + } + + [Benchmark(OperationsPerInvoke = ActualChangeCount)] + public Commit AddChangesAllAtOnce() + { + return _emptyDataModel.WriteChange(_clientId, DateTimeOffset.Now, Enumerable.Range(0, ActualChangeCount) + .Select(i => + _emptyDataModel.SetWord(Guid.NewGuid(), "entity1"))).Result; + } + + [IterationCleanup] + public void IterationCleanup() + { + _emptyDataModel.DisposeAsync().GetAwaiter().GetResult(); + } +} \ No newline at end of file diff --git a/src/SIL.Harmony.Benchmarks/AddSnapshotsBenchmarks.cs b/src/SIL.Harmony.Benchmarks/AddSnapshotsBenchmarks.cs new file mode 100644 index 0000000..e8c6574 --- /dev/null +++ b/src/SIL.Harmony.Benchmarks/AddSnapshotsBenchmarks.cs @@ -0,0 +1,79 @@ +using BenchmarkDotNet_GitCompare; +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Engines; +using Microsoft.Extensions.DependencyInjection; +using SIL.Harmony.Core; +using SIL.Harmony.Db; +using SIL.Harmony.Linq2db; +using SIL.Harmony.Sample.Models; +using SIL.Harmony.Tests; + +namespace SIL.Harmony.Benchmarks; + +[SimpleJob(RunStrategy.Throughput)] +[MemoryDiagnoser] +// [GitJob(gitReference: "HEAD", id: "before", baseline: true)] +public class AddSnapshotsBenchmarks +{ + private DataModelTestBase _emptyDataModel = null!; + private ICrdtRepository _repository = null!; + private Commit _commit = null!; + + [Params(1000)] + public int SnapshotCount { get; set; } + + [Params(true, false)] + public bool UseLinq2DbRepo { get; set; } + + [IterationSetup] + public void IterationSetup() + { + _emptyDataModel = new(alwaysValidate: false, + performanceTest: true, useLinq2DbRepo: UseLinq2DbRepo); + var crdtRepositoryFactory = _emptyDataModel.Services.GetRequiredService(); + _repository = crdtRepositoryFactory.CreateRepositorySync(); + _repository.AddCommit(_commit = new Commit(Guid.NewGuid()) + { + ClientId = Guid.NewGuid(), + HybridDateTime = new HybridDateTime(DateTimeOffset.Now, 0), + }).GetAwaiter().GetResult(); + } + + [Benchmark(OperationsPerInvoke = 1000)] + public void AddSnapshotsOneAtATime() + { + for (var i = 0; i < SnapshotCount; i++) + { + _repository.AddSnapshots([ + new ObjectSnapshot(new Word() + { + Id = Guid.NewGuid(), + Text = "test", + }, _commit, true) + ]).GetAwaiter().GetResult(); + } + } + + [Benchmark(OperationsPerInvoke = 1000)] + public void AddSnapshotsAllAtOnce() + { + var snapshots = Enumerable.Range(0, SnapshotCount) + .Select(i => new ObjectSnapshot(new Word() + { + Id = Guid.NewGuid(), + Text = "test", + }, + _commit, + true)) + .ToArray(); + + _repository.AddSnapshots(snapshots).GetAwaiter().GetResult(); + } + + [IterationCleanup] + public void IterationCleanup() + { + _repository.Dispose(); + _emptyDataModel.DisposeAsync().GetAwaiter().GetResult(); + } +} \ No newline at end of file diff --git a/src/SIL.Harmony.Benchmarks/Program.cs b/src/SIL.Harmony.Benchmarks/Program.cs new file mode 100644 index 0000000..72b1f4e --- /dev/null +++ b/src/SIL.Harmony.Benchmarks/Program.cs @@ -0,0 +1,4 @@ +using BenchmarkDotNet.Running; +using SIL.Harmony.Tests.Benchmarks; + +BenchmarkSwitcher.FromAssemblies([typeof(Program).Assembly, typeof(ChangeThroughput).Assembly]).Run(args); \ No newline at end of file diff --git a/src/SIL.Harmony.Benchmarks/SIL.Harmony.Benchmarks.csproj b/src/SIL.Harmony.Benchmarks/SIL.Harmony.Benchmarks.csproj new file mode 100644 index 0000000..db0c41b --- /dev/null +++ b/src/SIL.Harmony.Benchmarks/SIL.Harmony.Benchmarks.csproj @@ -0,0 +1,21 @@ + + + + Exe + net8.0 + enable + enable + + + + + + + + + + + + + + diff --git a/src/SIL.Harmony.Linq2db/Linq2DbCrdtRepo.cs b/src/SIL.Harmony.Linq2db/Linq2DbCrdtRepo.cs new file mode 100644 index 0000000..7846510 --- /dev/null +++ b/src/SIL.Harmony.Linq2db/Linq2DbCrdtRepo.cs @@ -0,0 +1,288 @@ +using System.Reflection; +using LinqToDB; +using LinqToDB.Data; +using LinqToDB.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Storage; +using Microsoft.Extensions.DependencyInjection; +using Nito.AsyncEx; +using SIL.Harmony.Core; +using SIL.Harmony.Db; +using SIL.Harmony.Resource; + +namespace SIL.Harmony.Linq2db; + +public class Linq2DbCrdtRepoFactory( + IServiceProvider serviceProvider, + ICrdtDbContextFactory dbContextFactory, + CrdtRepositoryFactory factory) : ICrdtRepositoryFactory +{ + public async Task CreateRepository() + { + return CreateInstance(await dbContextFactory.CreateDbContextAsync()); + } + + public ICrdtRepository CreateRepositorySync() + { + return CreateInstance(dbContextFactory.CreateDbContext()); + } + + private Linq2DbCrdtRepo CreateInstance(ICrdtDbContext dbContext) + { + return ActivatorUtilities.CreateInstance(serviceProvider, + factory.CreateInstance(dbContext), + dbContext); + } +} + +public class Linq2DbCrdtRepo : ICrdtRepository +{ + private readonly ICrdtRepository _original; + private readonly ICrdtDbContext _dbContext; + + public Linq2DbCrdtRepo(ICrdtRepository original, ICrdtDbContext dbContext) + { + _original = original; + _dbContext = dbContext; + } + + public ValueTask DisposeAsync() + { + return _original.DisposeAsync(); + } + + public void Dispose() + { + _original.Dispose(); + } + + public AwaitableDisposable Lock() + { + return _original.Lock(); + } + + public void ClearChangeTracker() + { + _original.ClearChangeTracker(); + } + + public async Task AddSnapshots(IEnumerable snapshots) + { + //save any pending commit changes + await _dbContext.SaveChangesAsync(); + var linqToDbTable = _dbContext.Set().ToLinqToDBTable(); + var dataContext = linqToDbTable.DataContext; + foreach (var grouping in snapshots.GroupBy(s => s.EntityIsDeleted) + .OrderByDescending(g => g.Key)) //execute deletes first + { + var objectSnapshots = grouping.ToArray(); + + //delete existing snapshots before we bulk recreate them + await _dbContext.Set() + .Where(s => objectSnapshots.Select(s => s.Id).Contains(s.Id)) + .ExecuteDeleteAsync(); + + await linqToDbTable.BulkCopyAsync(objectSnapshots); + + //descending to insert the most recent snapshots first, only keep the last objects by ordering by descending + //don't want to change the objectSnapshot order to preserve the order of the changes + var snapshotsToProject = objectSnapshots.DefaultOrderDescending().DistinctBy(s => s.EntityId).Select(s => s.Id).ToHashSet(); + foreach (var objectSnapshot in objectSnapshots.IntersectBy(snapshotsToProject, s => s.Id)) + { + try + { + if (objectSnapshot.EntityIsDeleted) + { + await DeleteAsync(dataContext, objectSnapshot.Entity); + } + else + { + await InsertOrReplaceAsync(dataContext, objectSnapshot.Entity); + } + } + catch (Exception e) + { + throw new Exception("error when projecting snapshot " + objectSnapshot, e); + } + } + } + } + + private static readonly MethodInfo InsertOrReplaceAsyncMethodGeneric = + new Func>( + DataExtensions.InsertOrReplaceAsync).Method.GetGenericMethodDefinition(); + + private Task InsertOrReplaceAsync(IDataContext dataContext, IObjectBase entity) + { + var result = InsertOrReplaceAsyncMethodGeneric.MakeGenericMethod(entity.GetType()).Invoke(null, + [dataContext, entity, null, null, null, null, TableOptions.NotSet, CancellationToken.None]); + ArgumentNullException.ThrowIfNull(result); + return (Task)result; + } + + private static readonly MethodInfo DeleteAsyncMethodGeneric = + new Func>( + DataExtensions.DeleteAsync).Method.GetGenericMethodDefinition(); + + private Task DeleteAsync(IDataContext dataContext, IObjectBase entity) + { + var result = DeleteAsyncMethodGeneric.MakeGenericMethod(entity.GetType()).Invoke(null, + [dataContext, entity, null, null, null, null, TableOptions.NotSet, CancellationToken.None]); + ArgumentNullException.ThrowIfNull(result); + return (Task)result; + } + + public Task AddCommit(Commit commit) + { + return _original.AddCommit(commit); + } + + public Task AddCommits(IEnumerable commits, bool save = true) + { + return _original.AddCommits(commits, save); + } + + public Task UpdateCommitHash(Guid commitId, string hash, string parentHash) + { + return _original.UpdateCommitHash(commitId, hash, parentHash); + } + + public Task BeginTransactionAsync() + { + return _original.BeginTransactionAsync(); + } + + public bool IsInTransaction => _original.IsInTransaction; + + public Task HasCommit(Guid commitId) + { + return _original.HasCommit(commitId); + } + + public Task<(Commit? oldestChange, Commit[] newCommits)> FilterExistingCommits(ICollection commits) + { + return _original.FilterExistingCommits(commits); + } + + public Task DeleteStaleSnapshots(Commit oldestChange) + { + return _original.DeleteStaleSnapshots(oldestChange); + } + + public Task DeleteSnapshotsAndProjectedTables() + { + return _original.DeleteSnapshotsAndProjectedTables(); + } + + public IQueryable CurrentCommits() + { + return _original.CurrentCommits(); + } + + public IQueryable CurrentSnapshots() + { + return _original.CurrentSnapshots(); + } + + public IAsyncEnumerable CurrenSimpleSnapshots(bool includeDeleted = false) + { + return _original.CurrenSimpleSnapshots(includeDeleted); + } + + public Task<(Dictionary currentSnapshots, Commit[] pendingCommits)> + GetCurrentSnapshotsAndPendingCommits() + { + return _original.GetCurrentSnapshotsAndPendingCommits(); + } + + public Task FindCommitByHash(string hash) + { + return _original.FindCommitByHash(hash); + } + + public Task FindPreviousCommit(Commit commit) + { + return _original.FindPreviousCommit(commit); + } + + public Task GetCommitsAfter(Commit? commit) + { + return _original.GetCommitsAfter(commit); + } + + public Task FindSnapshot(Guid id, bool tracking = false) + { + return _original.FindSnapshot(id, tracking); + } + + public Task GetCurrentSnapshotByObjectId(Guid objectId, bool tracking = false) + { + return _original.GetCurrentSnapshotByObjectId(objectId, tracking); + } + + public Task GetObjectBySnapshotId(Guid snapshotId) + { + return _original.GetObjectBySnapshotId(snapshotId); + } + + public Task GetCurrent(Guid objectId) where T : class + { + return _original.GetCurrent(objectId); + } + + public IQueryable GetCurrentObjects() where T : class + { + return _original.GetCurrentObjects(); + } + + public Task GetCurrentSyncState() + { + return _original.GetCurrentSyncState(); + } + + public Task> GetChanges(SyncState remoteState) + { + return _original.GetChanges(remoteState); + } + + public ICrdtRepository GetScopedRepository(Commit excludeChangesAfterCommit) + { + var inner = _original.GetScopedRepository(excludeChangesAfterCommit); + return new Linq2DbCrdtRepo(inner, _dbContext); + } + + public HybridDateTime? GetLatestDateTime() + { + return _original.GetLatestDateTime(); + } + + public Task AddLocalResource(LocalResource localResource) + { + return _original.AddLocalResource(localResource); + } + + public Task DeleteLocalResource(Guid id) + { + return _original.DeleteLocalResource(id); + } + + public IAsyncEnumerable LocalResourcesByIds(IEnumerable resourceIds) + { + return _original.LocalResourcesByIds(resourceIds); + } + + public IAsyncEnumerable LocalResources() + { + return _original.LocalResources(); + } + + public IQueryable LocalResourceIds() + { + return _original.LocalResourceIds(); + } + + public Task GetLocalResource(Guid resourceId) + { + return _original.GetLocalResource(resourceId); + } +} \ No newline at end of file diff --git a/src/SIL.Harmony.Linq2db/Linq2dbKernel.cs b/src/SIL.Harmony.Linq2db/Linq2dbKernel.cs index 5c32565..557a05f 100644 --- a/src/SIL.Harmony.Linq2db/Linq2dbKernel.cs +++ b/src/SIL.Harmony.Linq2db/Linq2dbKernel.cs @@ -1,16 +1,19 @@ -using SIL.Harmony.Core; -using LinqToDB.AspNet.Logging; +using System.Text.Json; +using SIL.Harmony.Core; using LinqToDB.EntityFrameworkCore; +using LinqToDB.Extensions.Logging; using LinqToDB.Mapping; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; using Microsoft.Extensions.Logging; +using SIL.Harmony.Db; namespace SIL.Harmony.Linq2db; public static class Linq2dbKernel { - public static DbContextOptionsBuilder UseLinqToDbCrdt(this DbContextOptionsBuilder builder, IServiceProvider provider) + public static DbContextOptionsBuilder UseLinqToDbCrdt(this DbContextOptionsBuilder builder, IServiceProvider provider, bool useLogging = true) { LinqToDBForEFTools.Initialize(); return builder.UseLinqToDB(optionsBuilder => @@ -22,7 +25,6 @@ public static DbContextOptionsBuilder UseLinqToDbCrdt(this DbContextOptionsBuild mappingSchema = new MappingSchema(); optionsBuilder.AddMappingSchema(mappingSchema); } - new FluentMappingBuilder(mappingSchema).HasAttribute(new ColumnAttribute("DateTime", nameof(Commit.HybridDateTime) + "." + nameof(HybridDateTime.DateTime))) .HasAttribute(new ColumnAttribute(nameof(HybridDateTime.Counter), @@ -31,11 +33,26 @@ public static DbContextOptionsBuilder UseLinqToDbCrdt(this DbContextOptionsBuild //need to use ticks here because the DateTime is stored as UTC, but the db records it as unspecified .Property(commit => commit.HybridDateTime.DateTime).HasConversionFunc(dt => dt.UtcDateTime, dt => new DateTimeOffset(dt.Ticks, TimeSpan.Zero)) + .Entity().Property(s => s.References).HasConversionFunc( + guids => JsonSerializer.Serialize(guids).ToUpper(),//uppercase matches EF and is required for querying by refs + s => JsonSerializer.Deserialize(s) ?? [] + ) .Build(); - var loggerFactory = provider.GetService(); - if (loggerFactory is not null) - optionsBuilder.AddCustomOptions(dataOptions => dataOptions.UseLoggerFactory(loggerFactory)); + if (useLogging) + { + var loggerFactory = provider.GetService(); + if (loggerFactory is not null) + optionsBuilder.AddCustomOptions(dataOptions => dataOptions.UseLoggerFactory(loggerFactory)); + } }); } + + public static IServiceCollection AddLinq2DbRepository(this IServiceCollection services) + { + services.RemoveAll(); + services.AddScoped(); + services.AddScoped(); + return services; + } } diff --git a/src/SIL.Harmony.Linq2db/SIL.Harmony.Linq2db.csproj b/src/SIL.Harmony.Linq2db/SIL.Harmony.Linq2db.csproj index d530422..1453659 100644 --- a/src/SIL.Harmony.Linq2db/SIL.Harmony.Linq2db.csproj +++ b/src/SIL.Harmony.Linq2db/SIL.Harmony.Linq2db.csproj @@ -10,8 +10,8 @@ - + diff --git a/src/SIL.Harmony.Sample/CrdtSampleKernel.cs b/src/SIL.Harmony.Sample/CrdtSampleKernel.cs index a430f68..ee21d26 100644 --- a/src/SIL.Harmony.Sample/CrdtSampleKernel.cs +++ b/src/SIL.Harmony.Sample/CrdtSampleKernel.cs @@ -16,7 +16,7 @@ public static IServiceCollection AddCrdtDataSample(this IServiceCollection servi } public static IServiceCollection AddCrdtDataSample(this IServiceCollection services, - Action optionsBuilder, bool performanceTest = false) + Action optionsBuilder, bool performanceTest = false, bool useLinq2DbRepo = false) { services.AddDbContext((provider, builder) => { @@ -24,13 +24,10 @@ public static IServiceCollection AddCrdtDataSample(this IServiceCollection servi //this can show up as the second instance using the JsonSerializerOptions from the first container //only needed for testing scenarios builder.EnableServiceProviderCaching(performanceTest); - builder.UseLinqToDbCrdt(provider); + builder.UseLinqToDbCrdt(provider, !performanceTest); optionsBuilder(builder); builder.EnableDetailedErrors(); builder.EnableSensitiveDataLogging(); -#if DEBUG - builder.LogTo(s => Debug.WriteLine(s)); -#endif }); services.AddCrdtData(config => { @@ -79,6 +76,8 @@ public static IServiceCollection AddCrdtDataSample(this IServiceCollection servi builder.HasIndex(wt => new { wt.WordId, wt.TagId }).IsUnique(); }); }); + if (useLinq2DbRepo) + services.AddLinq2DbRepository(); return services; } } \ No newline at end of file diff --git a/src/SIL.Harmony.Sample/Models/Definition.cs b/src/SIL.Harmony.Sample/Models/Definition.cs index 4eef28f..5327961 100644 --- a/src/SIL.Harmony.Sample/Models/Definition.cs +++ b/src/SIL.Harmony.Sample/Models/Definition.cs @@ -39,4 +39,9 @@ public IObjectBase Copy() DeletedAt = DeletedAt }; } + + public override string ToString() + { + return $"{nameof(Text)}: {Text}, {nameof(Id)}: {Id}, {nameof(WordId)}: {WordId}, {nameof(DeletedAt)}: {DeletedAt}"; + } } \ No newline at end of file diff --git a/src/SIL.Harmony.Tests/Benchmarks/ChangeThroughput.cs b/src/SIL.Harmony.Tests/Benchmarks/ChangeThroughput.cs new file mode 100644 index 0000000..32b9462 --- /dev/null +++ b/src/SIL.Harmony.Tests/Benchmarks/ChangeThroughput.cs @@ -0,0 +1,63 @@ +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Engines; + +namespace SIL.Harmony.Tests.Benchmarks; + + +// disable warning about waiting for sync code, benchmarkdotnet does not support async code, and it doesn't deadlock when waiting. +#pragma warning disable VSTHRD002 +[SimpleJob(RunStrategy.Throughput, warmupCount: 2)] +public class ChangeThroughput +{ + private DataModelTestBase _templateModel = null!; + private DataModelTestBase _dataModelTestBase = null!; + private DataModelTestBase _emptyDataModel = null!; + + + [GlobalSetup] + public void GlobalSetup() + { + _templateModel = new DataModelTestBase(alwaysValidate: false, performanceTest: true); + DataModelPerformanceTests.BulkInsertChanges(_templateModel, StartingSnapshots).GetAwaiter().GetResult(); + } + + [Params(0, 1000, 10_000)] + public int StartingSnapshots { get; set; } + + [IterationSetup] + public void IterationSetup() + { + _emptyDataModel = new(alwaysValidate: false, performanceTest: true); + _ = _emptyDataModel.WriteNextChange(_emptyDataModel.SetWord(Guid.NewGuid(), "entity1")).Result; + _dataModelTestBase = _templateModel.ForkDatabase(false); + } + + [Benchmark(Baseline = true), BenchmarkCategory("WriteChange")] + public Commit AddSingleChangePerformance() + { + return _emptyDataModel.WriteNextChange(_emptyDataModel.SetWord(Guid.NewGuid(), "entity1")).Result; + } + + [Benchmark, BenchmarkCategory("WriteChange")] + public Commit AddSingleChangeWithManySnapshots() + { + var count = _dataModelTestBase.DbContext.Snapshots.Count(); + // had a bug where there were no snapshots, this means the test was useless, this is slower, but it's better that then a useless test + if (count < (StartingSnapshots - 5)) throw new Exception($"Not enough snapshots, found {count}"); + return _dataModelTestBase.WriteNextChange(_dataModelTestBase.SetWord(Guid.NewGuid(), "entity1")).Result; + } + + [IterationCleanup] + public void IterationCleanup() + { + _emptyDataModel.DisposeAsync().GetAwaiter().GetResult(); + _dataModelTestBase.DisposeAsync().GetAwaiter().GetResult(); + } + + [GlobalCleanup] + public void GlobalCleanup() + { + _templateModel.DisposeAsync().GetAwaiter().GetResult(); + } +} +#pragma warning restore VSTHRD002 \ No newline at end of file diff --git a/src/SIL.Harmony.Tests/DataModelPerformanceTests.cs b/src/SIL.Harmony.Tests/DataModelPerformanceTests.cs index 84398e5..a09e43f 100644 --- a/src/SIL.Harmony.Tests/DataModelPerformanceTests.cs +++ b/src/SIL.Harmony.Tests/DataModelPerformanceTests.cs @@ -11,6 +11,7 @@ using SIL.Harmony.Changes; using SIL.Harmony.Db; using SIL.Harmony.Sample.Changes; +using SIL.Harmony.Tests.Benchmarks; using Xunit.Abstractions; namespace SIL.Harmony.Tests; @@ -25,7 +26,7 @@ public void AddingChangePerformance() Assert.Fail("This test is disabled in debug builds, not reliable"); #endif var summary = - BenchmarkRunner.Run( + BenchmarkRunner.Run( ManualConfig.CreateEmpty() .AddExporter(JsonExporter.FullCompressed) .AddColumnProvider(DefaultColumnProviders.Instance) @@ -177,60 +178,3 @@ public void Flush() } } -// disable warning about waiting for sync code, benchmarkdotnet does not support async code, and it doesn't deadlock when waiting. -#pragma warning disable VSTHRD002 -[SimpleJob(RunStrategy.Throughput, warmupCount: 2)] -public class DataModelPerformanceBenchmarks -{ - private DataModelTestBase _templateModel = null!; - private DataModelTestBase _dataModelTestBase = null!; - private DataModelTestBase _emptyDataModel = null!; - - - [GlobalSetup] - public void GlobalSetup() - { - _templateModel = new DataModelTestBase(alwaysValidate: false, performanceTest: true); - DataModelPerformanceTests.BulkInsertChanges(_templateModel, StartingSnapshots).GetAwaiter().GetResult(); - } - - [Params(0, 1000, 10_000)] - public int StartingSnapshots { get; set; } - - [IterationSetup] - public void IterationSetup() - { - _emptyDataModel = new(alwaysValidate: false, performanceTest: true); - _ = _emptyDataModel.WriteNextChange(_emptyDataModel.SetWord(Guid.NewGuid(), "entity1")).Result; - _dataModelTestBase = _templateModel.ForkDatabase(false); - } - - [Benchmark(Baseline = true), BenchmarkCategory("WriteChange")] - public Commit AddSingleChangePerformance() - { - return _emptyDataModel.WriteNextChange(_emptyDataModel.SetWord(Guid.NewGuid(), "entity1")).Result; - } - - [Benchmark, BenchmarkCategory("WriteChange")] - public Commit AddSingleChangeWithManySnapshots() - { - var count = _dataModelTestBase.DbContext.Snapshots.Count(); - // had a bug where there were no snapshots, this means the test was useless, this is slower, but it's better that then a useless test - if (count < (StartingSnapshots - 5)) throw new Exception($"Not enough snapshots, found {count}"); - return _dataModelTestBase.WriteNextChange(_dataModelTestBase.SetWord(Guid.NewGuid(), "entity1")).Result; - } - - [IterationCleanup] - public void IterationCleanup() - { - _emptyDataModel.DisposeAsync().GetAwaiter().GetResult(); - _dataModelTestBase.DisposeAsync().GetAwaiter().GetResult(); - } - - [GlobalCleanup] - public void GlobalCleanup() - { - _templateModel.DisposeAsync().GetAwaiter().GetResult(); - } -} -#pragma warning restore VSTHRD002 \ No newline at end of file diff --git a/src/SIL.Harmony.Tests/DataModelSimpleChanges.cs b/src/SIL.Harmony.Tests/DataModelSimpleChanges.cs index 284d2cc..7d315eb 100644 --- a/src/SIL.Harmony.Tests/DataModelSimpleChanges.cs +++ b/src/SIL.Harmony.Tests/DataModelSimpleChanges.cs @@ -213,7 +213,7 @@ public async Task ChangesToSnapshotsAreNotSaved() word.Note = "a note"; var commit = await WriteNextChange(SetWord(_entity1Id, "after-change")); - var objectSnapshot = commit.Snapshots.Should().ContainSingle().Subject; + var objectSnapshot = await DbContext.Snapshots.SingleAsync(s => s.CommitId == commit.Id); objectSnapshot.Entity.Is().Text.Should().Be("after-change"); objectSnapshot.Entity.Is().Note.Should().BeNull(); } diff --git a/src/SIL.Harmony.Tests/DataModelTestBase.cs b/src/SIL.Harmony.Tests/DataModelTestBase.cs index 09e5b2c..e5797a5 100644 --- a/src/SIL.Harmony.Tests/DataModelTestBase.cs +++ b/src/SIL.Harmony.Tests/DataModelTestBase.cs @@ -7,6 +7,7 @@ using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Logging; using SIL.Harmony.Db; namespace SIL.Harmony.Tests; @@ -20,10 +21,12 @@ public class DataModelTestBase : IAsyncLifetime public readonly SampleDbContext DbContext; protected readonly MockTimeProvider MockTimeProvider = new(); + public IServiceProvider Services => _services; + public DataModelTestBase(bool saveToDisk = false, bool alwaysValidate = true, - Action? configure = null, bool performanceTest = false) : this(saveToDisk + Action? configure = null, bool performanceTest = false, bool useLinq2DbRepo = false) : this(saveToDisk ? new SqliteConnection("Data Source=test.db") - : new SqliteConnection("Data Source=:memory:"), alwaysValidate, configure, performanceTest) + : new SqliteConnection("Data Source=:memory:"), alwaysValidate, configure, performanceTest, useLinq2DbRepo) { } @@ -31,15 +34,16 @@ public DataModelTestBase() : this(new SqliteConnection("Data Source=:memory:")) { } - public DataModelTestBase(SqliteConnection connection, bool alwaysValidate = true, Action? configure = null, bool performanceTest = false) + public DataModelTestBase(SqliteConnection connection, bool alwaysValidate = true, Action? configure = null, bool performanceTest = false, bool useLinq2DbRepo = false) { _performanceTest = performanceTest; var serviceCollection = new ServiceCollection().AddCrdtDataSample(builder => { builder.UseSqlite(connection, true); - }, performanceTest) + }, performanceTest, useLinq2DbRepo) .Configure(config => config.AlwaysValidateCommits = alwaysValidate) .Replace(ServiceDescriptor.Singleton(MockTimeProvider)); + if (!performanceTest) serviceCollection.AddLogging(builder => builder.AddDebug()); configure?.Invoke(serviceCollection); _services = serviceCollection.BuildServiceProvider(); DbContext = _services.GetRequiredService(); @@ -97,7 +101,7 @@ protected async ValueTask WriteChange(Guid clientId, return await WriteChange(clientId, dateTime, [change], add); } - protected async ValueTask WriteChange(Guid clientId, + public async ValueTask WriteChange(Guid clientId, DateTimeOffset dateTime, IEnumerable changes, bool add = true) diff --git a/src/SIL.Harmony.Tests/DbContextTests.VerifyModel.verified.txt b/src/SIL.Harmony.Tests/DbContextTests.VerifyModel.verified.txt index 7ad2c2d..71edd8b 100644 --- a/src/SIL.Harmony.Tests/DbContextTests.VerifyModel.verified.txt +++ b/src/SIL.Harmony.Tests/DbContextTests.VerifyModel.verified.txt @@ -237,4 +237,4 @@ Relational:ViewName: Relational:ViewSchema: Annotations: - ProductVersion: 9.0.4 \ No newline at end of file + ProductVersion: 9.0.7 \ No newline at end of file diff --git a/src/SIL.Harmony.Tests/RepositoryTests.cs b/src/SIL.Harmony.Tests/RepositoryTests.cs index b3e23ad..d356def 100644 --- a/src/SIL.Harmony.Tests/RepositoryTests.cs +++ b/src/SIL.Harmony.Tests/RepositoryTests.cs @@ -4,26 +4,33 @@ using SIL.Harmony.Tests.Mocks; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; using SIL.Harmony.Changes; using SIL.Harmony.Sample.Changes; namespace SIL.Harmony.Tests; +public class RepositoryTests_Linq2Db() : RepositoryTests(true); + public class RepositoryTests : IAsyncLifetime { private readonly ServiceProvider _services; - private readonly CrdtRepository _repository; + private readonly ICrdtRepository _repository; private readonly SampleDbContext _crdtDbContext; - public RepositoryTests() + protected RepositoryTests(bool useLinq2DbRepo) { _services = new ServiceCollection() - .AddCrdtDataSample(":memory:") + .AddCrdtDataSample(builder => builder.UseSqlite("Data Source=:memory:"), useLinq2DbRepo: useLinq2DbRepo) + .AddLogging(c => c.AddDebug()) .BuildServiceProvider(); - _repository = _services.GetRequiredService().CreateRepositorySync(); + _repository = _services.GetRequiredService().CreateRepositorySync(); _crdtDbContext = _services.GetRequiredService(); } + public RepositoryTests() : this(false) + { + } public async Task InitializeAsync() { @@ -56,9 +63,15 @@ private Commit Commit(Guid id, HybridDateTime hybridDateTime) }; } - private ObjectSnapshot Snapshot(Guid entityId, Guid commitId, HybridDateTime time) + private ObjectSnapshot Snapshot(Guid entityId, Guid commitId, HybridDateTime time, string? text = null) + { + return new(new Word { Text = text ?? "test", Id = entityId }, Commit(commitId, time), false) { }; + } + + private async Task AddSnapshots(IEnumerable snapshots) { - return new(new Word { Text = "test", Id = entityId }, Commit(commitId, time), false) { }; + _crdtDbContext.AddRange(snapshots); + await _crdtDbContext.SaveChangesAsync(); } private Guid[] OrderedIds(int count) @@ -145,20 +158,258 @@ await _repository.AddCommits([ commits.Select(c => c.Id).Should().ContainInConsecutiveOrder(ids); } + [Fact] + public async Task AddSnapshots_Works() + { + var snapshot = Snapshot(Guid.NewGuid(), Guid.NewGuid(), Time(1, 0)); + await _repository.AddCommit(snapshot.Commit); + + await _repository.AddSnapshots([snapshot]); + + _crdtDbContext.ChangeTracker.Clear(); + var actualSnapshot = await _crdtDbContext.Snapshots.SingleAsync(s => s.Id == snapshot.Id); + actualSnapshot.Should().BeEquivalentTo(snapshot, o => o + .Excluding(s => s.Entity.DbObject) + .Excluding(s => s.Commit)); + } + + [Fact] + public async Task AddSnapshots_StoresReferences() + { + var wordId = Guid.NewGuid(); + var snapshot = Snapshot(wordId, Guid.NewGuid(), Time(1, 0)); + await _repository.AddCommit(snapshot.Commit); + await _repository.AddSnapshots([snapshot]); + var commit = Commit(Guid.NewGuid(), Time(2, 0)); + await _repository.AddCommit(commit); + var defSnapshot = new ObjectSnapshot( + new Definition + { + Id = Guid.NewGuid(), + Text = "word", + Order = 0, + PartOfSpeech = "noun", + WordId = wordId + }, + commit, + true + ); + await _repository.AddSnapshots([defSnapshot]); + + _crdtDbContext.ChangeTracker.Clear(); + var actualSnapshot = await _crdtDbContext.Snapshots.SingleAsync(s => s.Id == defSnapshot.Id); + actualSnapshot.Should().BeEquivalentTo(defSnapshot, o => o + .Excluding(s => s.Entity.DbObject) + .Excluding(s => s.Commit)); + } + + [Fact] + public async Task AddSnapshots_HandlesTheSameSnapshotTwice() + { + var snapshot = Snapshot(Guid.NewGuid(), Guid.NewGuid(), Time(1, 0)); + await _repository.AddCommit(snapshot.Commit); + + await _repository.AddSnapshots([snapshot]); + await _repository.AddSnapshots([snapshot]); + _crdtDbContext.Snapshots.Should().ContainSingle(s => s.Id == snapshot.Id); + } + + [Fact] + public async Task AddSnapshots_Works_InsertsIntoProjectedTable() + { + var snapshot = Snapshot(Guid.NewGuid(), Guid.NewGuid(), Time(1, 0)); + await _repository.AddCommit(snapshot.Commit); + + await _repository.AddSnapshots([snapshot]); + + _crdtDbContext.Set().Should().Contain(w => w.Id == snapshot.EntityId); + } + + [Fact] + public async Task AddSnapshots_Works_UpdatesProjectedTable() + { + var entityId = Guid.NewGuid(); + await AddSnapshots([Snapshot(entityId, Guid.NewGuid(), Time(1, 0))]); + + var snapshot = Snapshot(entityId, Guid.NewGuid(), Time(2, 0), text: "updated"); + await _repository.AddCommit(snapshot.Commit); + await _repository.AddSnapshots([snapshot]); + + var word = await _crdtDbContext.Set().FindAsync(entityId); + word.Should().NotBeNull(); + word.Text.Should().Be("updated"); + } + + [Fact] + public async Task AddSnapshots_Works_DeletesFromProjectedTable() + { + var entityId = Guid.NewGuid(); + var firstSnapshot = Snapshot(entityId, Guid.NewGuid(), Time(1, 0)); + await _repository.AddCommit(firstSnapshot.Commit); + await _repository.AddSnapshots([firstSnapshot]); + _crdtDbContext.Set().Should().ContainSingle(w => w.Id == entityId); + + var time = Time(2, 0); + var snapshot = new ObjectSnapshot(new Word + { + Text = "test", + Id = entityId, + //mark as deleted + DeletedAt = time.DateTime + }, + Commit(Guid.NewGuid(), time), + false); + await _repository.AddCommit(snapshot.Commit); + await _repository.AddSnapshots([snapshot]); + + _crdtDbContext.Set().Should().NotContain(w => w.Id == entityId); + } + + [Fact] + public async Task AddSnapshots_DeletesFirst() + { + var firstTagId = Guid.NewGuid(); + var secondTagId = Guid.NewGuid(); + //must be the same text so that the unique constraint is violated if the delete is not executed first + var tagText = "tag"; + await AddSnapshots([ + new ObjectSnapshot(new Tag() + { + Id = firstTagId, + Text = tagText + }, + Commit(Guid.NewGuid(), Time(1, 0)), + true) + ]); + var secondCommit = Commit(Guid.NewGuid(), Time(2, 0)); + await _repository.AddCommit(secondCommit); + + //act + await _repository.AddSnapshots([ + new ObjectSnapshot(new Tag() + { + Id = secondTagId, + Text = tagText + }, + secondCommit, + true), + new ObjectSnapshot(new Tag() + { + Id = firstTagId, + Text = tagText, + DeletedAt = secondCommit.DateTime + }, + secondCommit, + false), + ]); + + //assert + _crdtDbContext.Set().Should().ContainSingle(t => t.Id == secondTagId); + } + + [Fact] + public async Task AddSnapshots_LastSnapshotWins() + { + var wordId = Guid.NewGuid(); + var firstCommit = Commit(Guid.NewGuid(), Time(1, 0)); + var secondCommit = Commit(Guid.NewGuid(), Time(2, 0)); + await _repository.AddCommit(firstCommit); + await _repository.AddCommit(secondCommit); + + await _repository.AddSnapshots([ + new ObjectSnapshot(new Word() + { + Id = wordId, + Text = "first" + }, + firstCommit, + true), + new ObjectSnapshot(new Word() + { + Id = wordId, + Text = "second" + }, + secondCommit, + false) + ]); + + _crdtDbContext.Set().Should().ContainSingle(w => w.Id == wordId).Which.Text.Should().Be("second"); + } + + [Fact] + public async Task AddSnapshots_InsertsInTheCorrectOrder() + { + var wordId = Guid.NewGuid(); + var firstCommit = Commit(Guid.NewGuid(), Time(1, 0)); + var secondCommit = Commit(Guid.NewGuid(), Time(2, 0)); + await _repository.AddCommit(firstCommit); + await _repository.AddCommit(secondCommit); + + await _repository.AddSnapshots([ + new ObjectSnapshot(new Word() + { + Id = wordId, + Text = "first" + }, + firstCommit, + true), + new ObjectSnapshot(new Definition() + { + Id = Guid.NewGuid(), + Text = "second", + WordId = wordId, + Order = 0, + PartOfSpeech = "noun" + }, + secondCommit, + false) + ]); + + _crdtDbContext.Set().Should().ContainSingle(w => w.Id == wordId).Which.Text.Should().Be("first"); + _crdtDbContext.Set().Should().ContainSingle(d => d.WordId == wordId).Which.Text.Should().Be("second"); + } + [Fact] public async Task CurrentSnapshots_Works() { - await _repository.AddSnapshots([Snapshot(Guid.NewGuid(), Guid.NewGuid(), Time(1, 0))]); + await AddSnapshots([Snapshot(Guid.NewGuid(), Guid.NewGuid(), Time(1, 0))]); var snapshots = await _repository.CurrentSnapshots().ToArrayAsync(); snapshots.Should().ContainSingle(); } + [Fact] + public async Task CurrentSnapshots_CanFilterByRefs() + { + var wordId = Guid.NewGuid(); + var snapshot = Snapshot(wordId, Guid.NewGuid(), Time(1, 0)); + await _repository.AddCommit(snapshot.Commit); + await _repository.AddSnapshots([snapshot]); + var commit = Commit(Guid.NewGuid(), Time(2, 0)); + await _repository.AddCommit(commit); + var defSnapshot = new ObjectSnapshot( + new Definition + { + Id = Guid.NewGuid(), + Text = "word", + Order = 0, + PartOfSpeech = "noun", + WordId = wordId + }, + commit, + true + ); + await _repository.AddSnapshots([defSnapshot]); + + var objectSnapshots = await _repository.CurrentSnapshots().Where(s => s.References.Contains(wordId)).ToArrayAsync(); + objectSnapshots.Should().ContainSingle().Which.Id.Should().Be(defSnapshot.Id); + } + [Fact] public async Task CurrentSnapshots_GroupsByEntityIdSortedByTime() { var entityId = Guid.NewGuid(); var expectedTime = Time(2, 0); - await _repository.AddSnapshots([ + await AddSnapshots([ Snapshot(entityId, Guid.NewGuid(), Time(1, 0)), Snapshot(entityId, Guid.NewGuid(), expectedTime), ]); @@ -170,7 +421,7 @@ await _repository.AddSnapshots([ public async Task CurrentSnapshots_GroupsByEntityIdSortedByCount() { var entityId = Guid.NewGuid(); - await _repository.AddSnapshots([ + await AddSnapshots([ Snapshot(entityId, Guid.NewGuid(), Time(1, 0)), Snapshot(entityId, Guid.NewGuid(), Time(1, 1)), ]); @@ -184,7 +435,7 @@ public async Task CurrentSnapshots_GroupsByEntityIdSortedByCommitId() var time = Time(1, 1); var entityId = Guid.NewGuid(); var ids = OrderedIds(2); - await _repository.AddSnapshots([ + await AddSnapshots([ Snapshot(entityId, ids[0], time), Snapshot(entityId, ids[1], time), ]); @@ -201,7 +452,7 @@ public async Task ScopedRepo_CurrentSnapshots_FiltersByCounter() var snapshot1 = Snapshot(entityId, commitIds[0], Time(1, 0)); var snapshot2 = Snapshot(entityId, commitIds[1], Time(2, 0)); var snapshot3 = Snapshot(entityId, commitIds[2], Time(2, 1)); - await _repository.AddSnapshots([ + await AddSnapshots([ snapshot3, snapshot1, snapshot2, @@ -226,7 +477,7 @@ public async Task ScopedRepo_CurrentSnapshots_FiltersByCommitId() var snapshot1 = Snapshot(entityId, commitIds[0], Time(1, 0)); var snapshot2 = Snapshot(entityId, commitIds[1], Time(2, 0)); var snapshot3 = Snapshot(entityId, commitIds[2], Time(2, 0)); - await _repository.AddSnapshots([ + await AddSnapshots([ snapshot3, snapshot1, snapshot2, @@ -250,7 +501,7 @@ public async Task DeleteStaleSnapshots_Works() [Fact] public async Task DeleteStaleSnapshots_DeletesSnapshotsAfterCommitByTime() { - await _repository.AddSnapshots([ + await AddSnapshots([ Snapshot(Guid.NewGuid(), Guid.NewGuid(), Time(1, 0)), Snapshot(Guid.NewGuid(), Guid.NewGuid(), Time(3, 0)), ]); @@ -263,7 +514,7 @@ await _repository.AddSnapshots([ [Fact] public async Task DeleteStaleSnapshots_DeletesSnapshotsAfterCommitByCount() { - await _repository.AddSnapshots([ + await AddSnapshots([ Snapshot(Guid.NewGuid(), Guid.NewGuid(), Time(1, 0)), Snapshot(Guid.NewGuid(), Guid.NewGuid(), Time(1, 2)), ]); @@ -279,7 +530,7 @@ public async Task DeleteStaleSnapshots_DeletesSnapshotsAfterCommitByCommitId() var time = Time(1, 1); var entityId = Guid.NewGuid(); var ids = OrderedIds(3); - await _repository.AddSnapshots([ + await AddSnapshots([ Snapshot(entityId, ids[0], time), Snapshot(entityId, ids[2], time), ]); @@ -343,4 +594,20 @@ public async Task FindPreviousCommit_ReturnsNullForFirstCommit() var previousCommit = await _repository.FindPreviousCommit(commit1); previousCommit.Should().BeNull(); } + + [Fact] + public async Task GetCurrentSnapshotByObjectId_Works() + { + var entityId = Guid.NewGuid(); + var snapshot = Snapshot(entityId, Guid.NewGuid(), Time(1, 0)); + await _repository.AddCommit(snapshot.Commit); + await _repository.AddSnapshots([snapshot]); + + _crdtDbContext.ChangeTracker.Clear(); + var actualSnapshot = await _repository.GetCurrentSnapshotByObjectId(entityId); + actualSnapshot.Should().BeEquivalentTo(snapshot, + o => o + .Excluding(s => s.Entity.DbObject) + .Excluding(s => s.Commit)); + } } diff --git a/src/SIL.Harmony.Tests/SIL.Harmony.Tests.csproj b/src/SIL.Harmony.Tests/SIL.Harmony.Tests.csproj index fdfa563..826bf88 100644 --- a/src/SIL.Harmony.Tests/SIL.Harmony.Tests.csproj +++ b/src/SIL.Harmony.Tests/SIL.Harmony.Tests.csproj @@ -14,6 +14,7 @@ runtime; build; native; contentfiles; analyzers; buildtransitive + diff --git a/src/SIL.Harmony/Commit.cs b/src/SIL.Harmony/Commit.cs index fa3a39e..6e2c877 100644 --- a/src/SIL.Harmony/Commit.cs +++ b/src/SIL.Harmony/Commit.cs @@ -14,16 +14,18 @@ protected Commit(Guid id, string hash, string parentHash, HybridDateTime hybridD ParentHash = parentHash; } - internal Commit(Guid id) : base(id) + public Commit(Guid id) : base(id) { Hash = GenerateHash(NullParentHash); ParentHash = NullParentHash; } - public void SetParentHash(string parentHash) + public bool SetParentHash(string parentHash) { + if (ParentHash == parentHash) return false; Hash = GenerateHash(parentHash); ParentHash = parentHash; + return true; } internal Commit() : this(Guid.NewGuid()) { diff --git a/src/SIL.Harmony/CrdtKernel.cs b/src/SIL.Harmony/CrdtKernel.cs index 2d1c95d..946ef3b 100644 --- a/src/SIL.Harmony/CrdtKernel.cs +++ b/src/SIL.Harmony/CrdtKernel.cs @@ -1,6 +1,7 @@ using System.Text.Json; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using SIL.Harmony.Db; @@ -33,10 +34,10 @@ public static IServiceCollection AddCrdtDataCore(this IServiceCollection service services.AddSingleton(sp => sp.GetRequiredService>().Value.JsonSerializerOptions); services.AddSingleton(TimeProvider.System); services.AddScoped(NewTimeProvider); - services.AddScoped(); + services.TryAddScoped(); //must use factory method because DataModel constructor is internal services.AddScoped(provider => new DataModel( - provider.GetRequiredService(), + provider.GetRequiredService(), provider.GetRequiredService(), provider.GetRequiredService(), provider.GetRequiredService>(), @@ -44,7 +45,7 @@ public static IServiceCollection AddCrdtDataCore(this IServiceCollection service )); //must use factory method because ResourceService constructor is internal services.AddScoped(provider => new ResourceService( - provider.GetRequiredService(), + provider.GetRequiredService(), provider.GetRequiredService>(), provider.GetRequiredService(), provider.GetRequiredService>() @@ -57,7 +58,7 @@ public static HybridDateTimeProvider NewTimeProvider(IServiceProvider servicePro //todo, if this causes issues getting the order correct, we can update the last date time after the db is created //as long as it's before we get a date time from the provider //todo use IMemoryCache to store the last date time, possibly based on the current project - using var repo = serviceProvider.GetRequiredService().CreateRepositorySync(); + using var repo = serviceProvider.GetRequiredService().CreateRepositorySync(); var hybridDateTime = repo.GetLatestDateTime(); hybridDateTime ??= HybridDateTimeProvider.DefaultLastDateTime; return ActivatorUtilities.CreateInstance(serviceProvider, hybridDateTime); diff --git a/src/SIL.Harmony/DataModel.cs b/src/SIL.Harmony/DataModel.cs index 9235389..b5e1b60 100644 --- a/src/SIL.Harmony/DataModel.cs +++ b/src/SIL.Harmony/DataModel.cs @@ -18,14 +18,14 @@ public class DataModel : ISyncable, IAsyncDisposable /// private bool AlwaysValidate => _crdtConfig.Value.AlwaysValidateCommits; - private readonly CrdtRepositoryFactory _crdtRepositoryFactory; + private readonly ICrdtRepositoryFactory _crdtRepositoryFactory; private readonly JsonSerializerOptions _serializerOptions; private readonly IHybridDateTimeProvider _timeProvider; private readonly IOptions _crdtConfig; private readonly ILogger _logger; //constructor must be internal because CrdtRepository is internal - internal DataModel(CrdtRepositoryFactory crdtRepositoryFactory, + internal DataModel(ICrdtRepositoryFactory crdtRepositoryFactory, JsonSerializerOptions serializerOptions, IHybridDateTimeProvider timeProvider, IOptions crdtConfig, @@ -194,7 +194,7 @@ ValueTask ISyncable.ShouldSync() return ValueTask.FromResult(true); } - private async Task UpdateSnapshots(CrdtRepository repo, Commit oldestAddedCommit, Commit[] newCommits) + private async Task UpdateSnapshots(ICrdtRepository repo, Commit oldestAddedCommit, Commit[] newCommits) { await repo.DeleteStaleSnapshots(oldestAddedCommit); Dictionary snapshotLookup; @@ -215,7 +215,7 @@ private async Task UpdateSnapshots(CrdtRepository repo, Commit oldestAddedCommit await snapshotWorker.UpdateSnapshots(oldestAddedCommit, newCommits); } - private async Task ValidateCommits(CrdtRepository repo) + private async Task ValidateCommits(ICrdtRepository repo) { Commit? parentCommit = null; await foreach (var commit in repo.CurrentCommits().AsNoTracking().AsAsyncEnumerable()) diff --git a/src/SIL.Harmony/Db/CrdtRepository.cs b/src/SIL.Harmony/Db/CrdtRepository.cs index a7145cb..9ff5c31 100644 --- a/src/SIL.Harmony/Db/CrdtRepository.cs +++ b/src/SIL.Harmony/Db/CrdtRepository.cs @@ -4,7 +4,6 @@ using Microsoft.EntityFrameworkCore.ChangeTracking; using Microsoft.EntityFrameworkCore.Infrastructure; using Microsoft.EntityFrameworkCore.Storage; -using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using Nito.AsyncEx; @@ -13,37 +12,7 @@ namespace SIL.Harmony.Db; -internal class CrdtRepositoryFactory(IServiceProvider serviceProvider, ICrdtDbContextFactory dbContextFactory) -{ - public async Task CreateRepository() - { - return ActivatorUtilities.CreateInstance(serviceProvider, await dbContextFactory.CreateDbContextAsync()); - } - - public CrdtRepository CreateRepositorySync() - { - return ActivatorUtilities.CreateInstance(serviceProvider, dbContextFactory.CreateDbContext()); - } - - public async Task Execute(Func> func) - { - await using var repo = await CreateRepository(); - return await func(repo); - } - public async Task Execute(Func func) - { - await using var repo = await CreateRepository(); - await func(repo); - } - - public async ValueTask Execute(Func> func) - { - await using var repo = await CreateRepository(); - return await func(repo); - } -} - -internal class CrdtRepository : IDisposable, IAsyncDisposable +internal class CrdtRepository : IDisposable, IAsyncDisposable, ICrdtRepository { private static readonly ConcurrentDictionary Locks = new(); @@ -87,7 +56,7 @@ private string DatabaseIdentifier //doesn't really do anything when using a dbcontext factory since it will likely just have been created //but when not using the factory it is still useful - internal void ClearChangeTracker() + public void ClearChangeTracker() { _dbContext.ChangeTracker.Clear(); } @@ -253,8 +222,19 @@ public async Task GetCommitsAfter(Commit? commit) .SingleOrDefaultAsync(s => s.Id == id); } + private readonly Func> GetCurrentSnapshotByObjectIdQuery = + EF.CompileAsyncQuery((DbContext dbContext, Guid objectId) => + dbContext.Set() + .AsTracking(QueryTrackingBehavior.TrackAll) + .Include(s => s.Commit) + .OrderBy(c => c.Commit.HybridDateTime.DateTime) + .ThenBy(c => c.Commit.HybridDateTime.Counter) + .ThenBy(c => c.Commit.Id) + .LastOrDefault(s => s.EntityId == objectId)); + public async Task GetCurrentSnapshotByObjectId(Guid objectId, bool tracking = false) { + if (tracking) return await GetCurrentSnapshotByObjectIdQuery(_dbContext.ChangeTracker.Context, objectId); return await Snapshots .AsTracking(tracking) .Include(s => s.Commit) @@ -362,7 +342,7 @@ private async ValueTask ProjectSnapshot(ObjectSnapshot objectSnapshot) return entity is not null ? _dbContext.Entry(entity) : null; } - public CrdtRepository GetScopedRepository(Commit excludeChangesAfterCommit) + public ICrdtRepository GetScopedRepository(Commit excludeChangesAfterCommit) { return new CrdtRepository(_dbContext, _crdtConfig, _logger, excludeChangesAfterCommit); } @@ -379,6 +359,16 @@ public async Task AddCommits(IEnumerable commits, bool save = true) if (save) await _dbContext.SaveChangesAsync(); } + public async Task UpdateCommitHash(Guid commitId, string hash, string parentHash) + { + await _dbContext.Commits + .Where(c => c.Id == commitId) + .ExecuteUpdateAsync(s => s + .SetProperty(c => c.Hash, hash) + .SetProperty(c => c.ParentHash, parentHash) + ); + } + public HybridDateTime? GetLatestDateTime() { return Commits diff --git a/src/SIL.Harmony/Db/CrdtRepositoryFactory.cs b/src/SIL.Harmony/Db/CrdtRepositoryFactory.cs new file mode 100644 index 0000000..f081ca7 --- /dev/null +++ b/src/SIL.Harmony/Db/CrdtRepositoryFactory.cs @@ -0,0 +1,45 @@ +using Microsoft.Extensions.DependencyInjection; + +namespace SIL.Harmony.Db; + +public interface ICrdtRepositoryFactory +{ + Task CreateRepository(); + ICrdtRepository CreateRepositorySync(); + + public async Task Execute(Func> func) + { + await using var repo = await CreateRepository(); + return await func(repo); + } + + public async Task Execute(Func func) + { + await using var repo = await CreateRepository(); + await func(repo); + } + + public async ValueTask Execute(Func> func) + { + await using var repo = await CreateRepository(); + return await func(repo); + } +} + +public class CrdtRepositoryFactory(IServiceProvider serviceProvider, ICrdtDbContextFactory dbContextFactory) : ICrdtRepositoryFactory +{ + public async Task CreateRepository() + { + return CreateInstance(await dbContextFactory.CreateDbContextAsync()); + } + + public ICrdtRepository CreateRepositorySync() + { + return CreateInstance(dbContextFactory.CreateDbContext()); + } + + public ICrdtRepository CreateInstance(ICrdtDbContext dbContext) + { + return ActivatorUtilities.CreateInstance(serviceProvider, dbContext); + } +} \ No newline at end of file diff --git a/src/SIL.Harmony/Db/ICrdtRepository.cs b/src/SIL.Harmony/Db/ICrdtRepository.cs new file mode 100644 index 0000000..6db87da --- /dev/null +++ b/src/SIL.Harmony/Db/ICrdtRepository.cs @@ -0,0 +1,48 @@ +using Microsoft.EntityFrameworkCore.Storage; +using Nito.AsyncEx; +using SIL.Harmony.Resource; + +namespace SIL.Harmony.Db; + +public interface ICrdtRepository : IAsyncDisposable, IDisposable +{ + AwaitableDisposable Lock(); + void ClearChangeTracker(); + Task BeginTransactionAsync(); + bool IsInTransaction { get; } + Task HasCommit(Guid commitId); + Task<(Commit? oldestChange, Commit[] newCommits)> FilterExistingCommits(ICollection commits); + Task DeleteStaleSnapshots(Commit oldestChange); + Task DeleteSnapshotsAndProjectedTables(); + IQueryable CurrentCommits(); + IQueryable CurrentSnapshots(); + IAsyncEnumerable CurrenSimpleSnapshots(bool includeDeleted = false); + Task<(Dictionary currentSnapshots, Commit[] pendingCommits)> GetCurrentSnapshotsAndPendingCommits(); + Task FindCommitByHash(string hash); + Task FindPreviousCommit(Commit commit); + Task GetCommitsAfter(Commit? commit); + Task FindSnapshot(Guid id, bool tracking = false); + Task GetCurrentSnapshotByObjectId(Guid objectId, bool tracking = false); + Task GetObjectBySnapshotId(Guid snapshotId); + Task GetCurrent(Guid objectId) where T: class; + IQueryable GetCurrentObjects() where T : class; + Task GetCurrentSyncState(); + Task> GetChanges(SyncState remoteState); + Task AddSnapshots(IEnumerable snapshots); + ICrdtRepository GetScopedRepository(Commit excludeChangesAfterCommit); + Task AddCommit(Commit commit); + Task AddCommits(IEnumerable commits, bool save = true); + Task UpdateCommitHash(Guid commitId, string hash, string parentHash); + HybridDateTime? GetLatestDateTime(); + Task AddLocalResource(LocalResource localResource); + IAsyncEnumerable LocalResourcesByIds(IEnumerable resourceIds); + IAsyncEnumerable LocalResources(); + + /// + /// primarily for filtering other queries + /// + IQueryable LocalResourceIds(); + + Task GetLocalResource(Guid resourceId); + Task DeleteLocalResource(Guid id); +} \ No newline at end of file diff --git a/src/SIL.Harmony/Db/ObjectSnapshot.cs b/src/SIL.Harmony/Db/ObjectSnapshot.cs index b2c31e8..54fbc28 100644 --- a/src/SIL.Harmony/Db/ObjectSnapshot.cs +++ b/src/SIL.Harmony/Db/ObjectSnapshot.cs @@ -77,4 +77,9 @@ public ObjectSnapshot(IObjectBase entity, Commit commit, bool isRoot) : this() public required Commit Commit { get; init; } CommitBase IObjectSnapshot.Commit => Commit; public required bool IsRoot { get; init; } + + public override string ToString() + { + return $"{Id} [{CommitId}] {TypeName} {EntityId} Deleted:{EntityIsDeleted}, Entity: {Entity}"; + } } diff --git a/src/SIL.Harmony/ResourceService.cs b/src/SIL.Harmony/ResourceService.cs index 344765c..3539f72 100644 --- a/src/SIL.Harmony/ResourceService.cs +++ b/src/SIL.Harmony/ResourceService.cs @@ -10,12 +10,12 @@ namespace SIL.Harmony; public class ResourceService { - private readonly CrdtRepositoryFactory _crdtRepositoryFactory; + private readonly ICrdtRepositoryFactory _crdtRepositoryFactory; private readonly IOptions _crdtConfig; private readonly DataModel _dataModel; private readonly ILogger _logger; - internal ResourceService(CrdtRepositoryFactory crdtRepositoryFactory, IOptions crdtConfig, DataModel dataModel, ILogger logger) + internal ResourceService(ICrdtRepositoryFactory crdtRepositoryFactory, IOptions crdtConfig, DataModel dataModel, ILogger logger) { _crdtRepositoryFactory = crdtRepositoryFactory; _crdtConfig = crdtConfig; @@ -164,7 +164,7 @@ public async Task DownloadResource(RemoteResource remoteResource, return await DownloadResourceInternal(repo, remoteResource, remoteResourceService); } - private async Task DownloadResourceInternal(CrdtRepository repo, + private async Task DownloadResourceInternal(ICrdtRepository repo, RemoteResource remoteResource, IRemoteResourceService remoteResourceService) { diff --git a/src/SIL.Harmony/SnapshotWorker.cs b/src/SIL.Harmony/SnapshotWorker.cs index 612346f..0e053bd 100644 --- a/src/SIL.Harmony/SnapshotWorker.cs +++ b/src/SIL.Harmony/SnapshotWorker.cs @@ -11,7 +11,7 @@ namespace SIL.Harmony; internal class SnapshotWorker { private readonly Dictionary _snapshotLookup; - private readonly CrdtRepository _crdtRepository; + private readonly ICrdtRepository _crdtRepository; private readonly CrdtConfig _crdtConfig; private readonly Dictionary _pendingSnapshots = []; private readonly Dictionary _rootSnapshots = []; @@ -19,7 +19,7 @@ internal class SnapshotWorker private SnapshotWorker(Dictionary snapshots, Dictionary snapshotLookup, - CrdtRepository crdtRepository, + ICrdtRepository crdtRepository, CrdtConfig crdtConfig) { _pendingSnapshots = snapshots; @@ -30,7 +30,7 @@ private SnapshotWorker(Dictionary snapshots, internal static async Task> ApplyCommitsToSnapshots( Dictionary snapshots, - CrdtRepository crdtRepository, + ICrdtRepository crdtRepository, ICollection commits, CrdtConfig crdtConfig) { @@ -44,7 +44,7 @@ internal static async Task> ApplyCommitsToSnaps /// /// internal SnapshotWorker(Dictionary snapshotLookup, - CrdtRepository crdtRepository, + ICrdtRepository crdtRepository, CrdtConfig crdtConfig): this([], snapshotLookup, crdtRepository, crdtConfig) { } @@ -71,7 +71,8 @@ private async ValueTask ApplyCommitChanges(IEnumerable commits, bool upd if (updateCommitHash && previousCommitHash is not null) { //we're rewriting history, so we need to update the previous commit hash - commit.SetParentHash(previousCommitHash); + if (commit.SetParentHash(previousCommitHash)) + await _crdtRepository.UpdateCommitHash(commit.Id, hash: commit.Hash, parentHash: commit.ParentHash); } previousCommitHash = commit.Hash;