V16/feature/move last synced id to db

src/Umbraco.Core/DependencyInjection/UmbracoBuilder.cs

@@ -17,6 +17,7 @@
 using Umbraco.Cms.Core.DynamicRoot;
 using Umbraco.Cms.Core.Editors;
 using Umbraco.Cms.Core.Events;
+using Umbraco.Cms.Core.Factories;
 using Umbraco.Cms.Core.Features;
 using Umbraco.Cms.Core.Handlers;
 using Umbraco.Cms.Core.Hosting;
@@ -344,6 +345,8 @@
                 factory.GetRequiredService<ILogger<LocalizedTextService>>()));
             Services.AddUnique<IRepositoryCacheVersionService, RepositoryCacheVersionService>();
             Services.AddUnique<ILongRunningOperationService, LongRunningOperationService>();
+            Services.AddUnique<ILastSyncedManager, LastSyncedManager>();
+            Services.AddUnique<IMachineInfoFactory, MachineInfoFactory>();
 
             Services.AddUnique<IEntityXmlSerializer, EntityXmlSerializer>();
 

src/Umbraco.Core/Factories/IMachineInfoFactory.cs

@@ -0,0 +1,13 @@
+namespace Umbraco.Cms.Core.Factories;
+
+/// <summary>
+/// Fetches information of the host machine.
+/// </summary>
+public interface IMachineInfoFactory
+{
+    /// <summary>
+    /// Fetches the name of the Host Machine for identification.
+    /// </summary>
+    /// <returns>A name of the host machine.</returns>
+    public string GetMachineIdentifier();
+}

src/Umbraco.Core/Factories/MachineInfoFactory.cs

@@ -0,0 +1,8 @@
+namespace Umbraco.Cms.Core.Factories;
+
+internal sealed class MachineInfoFactory : IMachineInfoFactory
+{
+
+    /// <inheritdoc />
+    public string GetMachineIdentifier() => Environment.MachineName;
+}

src/Umbraco.Core/Persistence/Repositories/ILastSyncedRepository.cs

@@ -0,0 +1,38 @@
+namespace Umbraco.Cms.Core.Persistence.Repositories;
+
+/// <summary>
+/// Handles saving and pruning of the LastSynced database table.
+/// </summary>
+public interface ILastSyncedRepository
+{
+    /// <summary>
+    /// Fetches the last synced internal ID from the database.
+    /// </summary>
+    /// <returns>The Internal ID from the database.</returns>
+    Task<int?> GetInternalIdAsync();
+
+    /// <summary>
+    /// Fetches the last synced external ID from the database.
+    /// </summary>
+    /// <returns>The External ID from the database.</returns>
+    Task<int?> GetExternalIdAsync();
+
+    /// <summary>
+    /// Saves the last synced Internal ID to the Database.
+    /// </summary>
+    /// <param name="id">The last synced internal ID.</param>
+    Task SaveInternalIdAsync(int id);
+
+    /// <summary>
+    /// Saves the last synced External ID to the Database.
+    /// </summary>
+    /// <param name="id">The last synced external ID.</param>
+    Task SaveExternalIdAsync(int id);
+
+    /// <summary>
+    /// Deletes entries older than the set parameter. This method also removes any entries where both
+    /// IDs are higher than the lowest synced CacheInstruction ID.
+    /// </summary>
+    /// <param name="pruneDate">Any date entries in the DB before this parameter, will be removed from the Database.</param>
+    Task DeleteEntriesOlderThanAsync(DateTime pruneDate);
+}

src/Umbraco.Core/Sync/ILastSyncedManager.cs

@@ -0,0 +1,38 @@
+namespace Umbraco.Cms.Core.Sync;
+
+/// <summary>
+/// Handles saving and pruning of the LastSynced database table.
+/// </summary>
+public interface ILastSyncedManager
+{
+    /// <summary>
+    /// Fetches the last synced internal ID from the database.
+    /// </summary>
+    /// <returns>The Internal ID from the database.</returns>
+    Task<int?> GetLastSyncedInternalAsync();
+
+    /// <summary>
+    /// Fetches the last synced external ID from the database.
+    /// </summary>
+    /// <returns>The External ID from the database.</returns>
+    Task<int?> GetLastSyncedExternalAsync();
+
+    /// <summary>
+    /// Saves the last synced Internal ID to the Database.
+    /// </summary>
+    /// <param name="id">The last synced internal ID.</param>
+    Task SaveLastSyncedInternalAsync(int id);
+
+    /// <summary>
+    /// Saves the last synced External ID to the Database.
+    /// </summary>
+    /// <param name="id">The last synced external ID.</param>
+    Task SaveLastSyncedExternalAsync(int id);
+
+    /// <summary>
+    /// Deletes entries older than the set parameter. This method also removes any entries where both
+    /// IDs are higher than the lowest synced CacheInstruction ID.
+    /// </summary>
+    /// <param name="pruneDate">Any date entries in the DB before this parameter, will be removed from the Database.</param>
+    Task DeleteOlderThanAsync(DateTime date);
+}

src/Umbraco.Core/Sync/LastSyncedManager.cs

@@ -0,0 +1,71 @@
+using Umbraco.Cms.Core.Persistence.Repositories;
+using Umbraco.Cms.Core.Scoping;
+
+namespace Umbraco.Cms.Core.Sync;
+
+/// <inheritdoc/>
+internal sealed class LastSyncedManager : ILastSyncedManager
+{
+    private readonly ILastSyncedRepository _lastSyncedRepository;
+    private readonly ICoreScopeProvider _coreScopeProvider;
+
+    public LastSyncedManager(ILastSyncedRepository lastSyncedRepository, ICoreScopeProvider coreScopeProvider)
+    {
+        _lastSyncedRepository = lastSyncedRepository;
+        _coreScopeProvider = coreScopeProvider;
+    }
+
+    /// <inheritdoc/>
+    public async Task<int?> GetLastSyncedInternalAsync()
+    {
+        using ICoreScope scope = _coreScopeProvider.CreateCoreScope();
+        int? internalId = await _lastSyncedRepository.GetInternalIdAsync();
+        scope.Complete();
+
+        return internalId;
+    }
+
+    /// <inheritdoc/>
+    public async Task<int?> GetLastSyncedExternalAsync()
+    {
+        using ICoreScope scope = _coreScopeProvider.CreateCoreScope();
+        int? externalId = await _lastSyncedRepository.GetExternalIdAsync();
+        scope.Complete();
+
+        return externalId;
+    }
+
+    /// <inheritdoc/>
+    public async Task SaveLastSyncedInternalAsync(int id)
+    {
+        if (id < 0)
+        {
+            throw new ArgumentException("Invalid last synced id. Must be non-negative.");
+        }
+
+        using ICoreScope scope = _coreScopeProvider.CreateCoreScope();
+        await _lastSyncedRepository.SaveInternalIdAsync(id);
+        scope.Complete();
+    }
+
+    /// <inheritdoc/>
+    public async Task SaveLastSyncedExternalAsync(int id)
+    {
+        if (id < 0)
+        {
+            throw new ArgumentException("Invalid last synced id. Must be non-negative.");
+        }
+
+        using ICoreScope scope = _coreScopeProvider.CreateCoreScope();
+        await _lastSyncedRepository.SaveExternalIdAsync(id);
+        scope.Complete();
+    }
+
+    /// <inheritdoc/>
+    public async Task DeleteOlderThanAsync(DateTime date)
+    {
+        using ICoreScope scope = _coreScopeProvider.CreateCoreScope();
+        await _lastSyncedRepository.DeleteEntriesOlderThanAsync(date);
+        scope.Complete();
+    }
+}

src/Umbraco.Infrastructure/BackgroundJobs/Jobs/CacheInstructionsPruningJob.cs

@@ -2,7 +2,7 @@
 using Umbraco.Cms.Core.Configuration.Models;
 using Umbraco.Cms.Core.Persistence.Repositories;
 using Umbraco.Cms.Core.Scoping;
-using Umbraco.Cms.Infrastructure.Scoping;
+using Umbraco.Cms.Core.Sync;
 
 namespace Umbraco.Cms.Infrastructure.BackgroundJobs.Jobs;
 
@@ -15,6 +15,7 @@ public class CacheInstructionsPruningJob : IRecurringBackgroundJob
     private readonly ICacheInstructionRepository _cacheInstructionRepository;
     private readonly ICoreScopeProvider _scopeProvider;
     private readonly TimeProvider _timeProvider;
+    private readonly ILastSyncedManager _lastSyncedManager;
 
     /// <summary>
     /// Initializes a new instance of the <see cref="CacheInstructionsPruningJob"/> class.
@@ -27,13 +28,15 @@ public CacheInstructionsPruningJob(
         IOptions<GlobalSettings> globalSettings,
         ICacheInstructionRepository cacheInstructionRepository,
         ICoreScopeProvider scopeProvider,
-        TimeProvider timeProvider)
+        TimeProvider timeProvider,
+        ILastSyncedManager lastSyncedManager)
     {
         _globalSettings = globalSettings;
         _cacheInstructionRepository = cacheInstructionRepository;
         _scopeProvider = scopeProvider;
         _timeProvider = timeProvider;
         Period = globalSettings.Value.DatabaseServerMessenger.TimeBetweenPruneOperations;
+        _lastSyncedManager = lastSyncedManager;
     }
 
     /// <inheritdoc />
@@ -53,6 +56,7 @@ public Task RunJobAsync()
         using (ICoreScope scope = _scopeProvider.CreateCoreScope())
         {
             _cacheInstructionRepository.DeleteInstructionsOlderThan(pruneDate.DateTime);
+            _lastSyncedManager.DeleteOlderThanAsync(pruneDate.DateTime).GetAwaiter().GetResult();
             scope.Complete();
         }
 

src/Umbraco.Infrastructure/DependencyInjection/UmbracoBuilder.Repositories.cs

@@ -84,6 +84,7 @@ internal static IUmbracoBuilder AddRepositories(this IUmbracoBuilder builder)
         builder.Services.AddUnique<IPublishStatusRepository, PublishStatusRepository>();
         builder.Services.AddUnique<IRepositoryCacheVersionRepository, RepositoryCacheVersionRepository>();
         builder.Services.AddUnique<ILongRunningOperationRepository, LongRunningOperationRepository>();
+        builder.Services.AddUnique<ILastSyncedRepository, LastSyncedRepository>();
 
         return builder;
     }

src/Umbraco.Infrastructure/Migrations/Upgrade/V_16_2_0/AddLastSyncedTable.cs

@@ -0,0 +1,21 @@
+using Umbraco.Cms.Infrastructure.Persistence.Dtos;
+
+namespace Umbraco.Cms.Infrastructure.Migrations.Upgrade.V_16_2_0;
+
+public class AddLastSyncedTable : AsyncMigrationBase
+{
+    public AddLastSyncedTable(IMigrationContext context)
+        : base(context)
+    {
+    }
+
+    protected override Task MigrateAsync()
+    {
+        if (TableExists(LastSyncedDto.TableName) is false)
+        {
+            Create.Table<LastSyncedDto>().Do();
+        }
+
+        return Task.CompletedTask;
+    }
+}

src/Umbraco.Infrastructure/Persistence/Dtos/LastSyncedDto.cs

@@ -0,0 +1,29 @@
+using NPoco;
+using Umbraco.Cms.Infrastructure.Persistence.DatabaseAnnotations;
+using Constants = Umbraco.Cms.Core.Constants;
+
+namespace Umbraco.Cms.Infrastructure.Persistence.Dtos;
+
+
+[TableName(TableName)]
+[PrimaryKey("machineId", AutoIncrement = false)]
+[ExplicitColumns]
+public class LastSyncedDto
+{
+    internal const string TableName = Constants.DatabaseSchema.Tables.LastSynced;
+
+    [Column("machineId")]
+    [PrimaryKeyColumn(Name = "PK_lastSyncedMachineId", AutoIncrement = false, Clustered = true)]
+    public required string MachineId { get; set; }
+
+    [Column("lastSyncedInternalId")]
+    [NullSetting(NullSetting = NullSettings.Null)]
+    public int? LastSyncedInternalId { get; set; }
+
+    [Column("lastSyncedExternalId")]
+    [NullSetting(NullSetting = NullSettings.Null)]
+    public int? LastSyncedExternalId { get; set; }
+
+    [Column("lastSyncedDate")]
+    public DateTime LastSyncedDate { get; set; }
+}

src/Umbraco.Infrastructure/Persistence/Repositories/Implement/LastSyncedRepository.cs

@@ -0,0 +1,103 @@
+using NPoco;
+using Umbraco.Cms.Core.Cache;
+using Umbraco.Cms.Core.Factories;
+using Umbraco.Cms.Core.Persistence.Repositories;
+using Umbraco.Cms.Infrastructure.Persistence.Dtos;
+using Umbraco.Cms.Infrastructure.Scoping;
+using Umbraco.Extensions;
+
+namespace Umbraco.Cms.Infrastructure.Persistence.Repositories.Implement;
+
+/// <inheritdoc cref="ILastSyncedRepository"/>
+public class LastSyncedRepository : RepositoryBase, ILastSyncedRepository
+{
+    private readonly IMachineInfoFactory  _machineInfoFactory;
+
+    public LastSyncedRepository(IScopeAccessor scopeAccessor, AppCaches appCaches, IMachineInfoFactory machineInfoFactory)
+        : base(scopeAccessor, appCaches)
+    {
+        _machineInfoFactory = machineInfoFactory;
+    }
+
+
+    /// <inheritdoc />
+    public async Task<int?> GetInternalIdAsync()
+    {
+        string machineName = _machineInfoFactory.GetMachineIdentifier();
+
+        Sql<ISqlContext> sql = Database.SqlContext.Sql()
+            .Select<LastSyncedDto>(x => x.LastSyncedInternalId)
+            .From<LastSyncedDto>()
+            .Where<LastSyncedDto>(x => x.MachineId == machineName);
+
+        return await Database.ExecuteScalarAsync<int?>(sql);
+    }
+
+    /// <inheritdoc />
+    public async Task<int?> GetExternalIdAsync()
+    {
+        string machineName = _machineInfoFactory.GetMachineIdentifier();
+
+        Sql<ISqlContext> sql = Database.SqlContext.Sql()
+            .Select<LastSyncedDto>(x => x.LastSyncedExternalId)
+            .From<LastSyncedDto>()
+            .Where<LastSyncedDto>(x => x.MachineId == machineName);
+
+        return await Database.ExecuteScalarAsync<int?>(sql);
+    }
+
+    /// <inheritdoc />
+    public async Task SaveInternalIdAsync(int id)
+    {
+        LastSyncedDto dto = new LastSyncedDto()
+        {
+            MachineId = _machineInfoFactory.GetMachineIdentifier(),
+            LastSyncedInternalId = id,
+            LastSyncedDate = DateTime.Now,
+        };
+
+        await Database.InsertOrUpdateAsync(
+            dto,
+            "SET lastSyncedInternalId=@LastSyncedInternalId, lastSyncedDate=@LastSyncedDate WHERE machineId=@MachineId",
+            new
+            {
+                dto.LastSyncedInternalId,
+                dto.LastSyncedDate,
+                dto.MachineId,
+            });
+    }
+
+    /// <inheritdoc />
+    public async Task SaveExternalIdAsync(int id)
+    {
+        LastSyncedDto dto = new LastSyncedDto()
+        {
+            MachineId = _machineInfoFactory.GetMachineIdentifier(),
+            LastSyncedExternalId = id,
+            LastSyncedDate = DateTime.Now,
+        };
+
+        await Database.InsertOrUpdateAsync(
+            dto,
+            "SET lastSyncedExternalId=@LastSyncedExternalId, lastSyncedDate=@LastSyncedDate WHERE machineId=@MachineId",
+            new
+            {
+                dto.LastSyncedExternalId,
+                dto.LastSyncedDate,
+                dto.MachineId,
+            });
+    }
+
+    /// <inheritdoc />
+    public async Task DeleteEntriesOlderThanAsync(DateTime pruneDate)
+    {
+        var maxId = Database.ExecuteScalar<int>($"SELECT MAX(Id) FROM umbracoCacheInstruction;");
+
+        Sql sql =
+            new Sql().Append(
+                @"DELETE FROM umbracoLastSynced WHERE lastSyncedDate < @pruneDate OR lastSyncedInternalId > @maxId AND lastSyncedExternalId > @maxId;",
+                new { pruneDate, maxId });
+
+        await Database.ExecuteAsync(sql);
+    }
+}