Load Balancing: Implement distributed background jobs (#20397)

* Start work

* Introduce dto

* Start making repository

* Add migrations

* Implement fetchable first job

* Fix up to also finish tasks

* Refactor jobs to distributed background jobs

* Filter jobs correctly on LastRun

* Hardcode delay

* Add settings to configure delay and period

* Fix formatting

* Add default data

* Add update on startup, which will update periods on startup

* Refactor service to return job directly

* Update src/Umbraco.Infrastructure/Services/Implement/DistributedJobService.cs

Co-authored-by: Copilot <[email protected]>

* Update src/Umbraco.Infrastructure/BackgroundJobs/DistributedBackgroundJobHostedService.cs

Co-authored-by: Copilot <[email protected]>

* Update src/Umbraco.Infrastructure/Migrations/Install/DatabaseDataCreator.cs

Co-authored-by: Copilot <[email protected]>

* Update src/Umbraco.Infrastructure/Migrations/Install/DatabaseDataCreator.cs

Co-authored-by: Copilot <[email protected]>

* Update src/Umbraco.Infrastructure/BackgroundJobs/DistributedBackgroundJobHostedService.cs

Co-authored-by: Copilot <[email protected]>

* Remove unused

* Move jobs and make internal

* make OpenIddictCleanupJob.cs public, as it is used elsewhere

* Minor docstring changes

* Update src/Umbraco.Core/Persistence/Constants-Locks.cs

Co-authored-by: Mole <[email protected]>

* ´Throw correct exceptions

* Update xml doc

* Remove business logic from repository

* Remove more business logic from repository into service

* Remove adding jobs from migration

* fix creation

* Rename to ExecuteAsync

---------

Co-authored-by: Copilot <[email protected]>
Co-authored-by: mole <[email protected]>

Author: 3 people

src/Umbraco.Cms.Api.Common/DependencyInjection/UmbracoBuilderAuthExtensions.cs

@@ -9,8 +9,8 @@
 using Umbraco.Cms.Core;
 using Umbraco.Cms.Core.Configuration.Models;
 using Umbraco.Cms.Core.DependencyInjection;
-using Umbraco.Cms.Infrastructure.BackgroundJobs.Jobs;
-using Umbraco.Extensions;
+using Umbraco.Cms.Infrastructure.BackgroundJobs;
+using Umbraco.Cms.Infrastructure.BackgroundJobs.Jobs.DistributedJobs;
 
 namespace Umbraco.Cms.Api.Common.DependencyInjection;
 
@@ -139,7 +139,7 @@ private static void ConfigureOpenIddict(IUmbracoBuilder builder)
                 });
             });
 
-        builder.Services.AddRecurringBackgroundJob<OpenIddictCleanupJob>();
+        builder.Services.AddSingleton<IDistributedBackgroundJob, OpenIddictCleanupJob>();
         builder.Services.ConfigureOptions<ConfigureOpenIddict>();
     }
 }

src/Umbraco.Cms.Api.Management/DependencyInjection/UmbracoBuilder.BackOffice.cs

@@ -30,7 +30,7 @@ public static IUmbracoBuilder
         .AddMembersIdentity()
         .AddUmbracoProfiler()
         .AddMvcAndRazor(configureMvc)
-        .AddRecurringBackgroundJobs()
+        .AddBackgroundJobs()
         .AddUmbracoHybridCache()
         .AddDistributedCache()
         .AddCoreNotifications();

src/Umbraco.Core/Configuration/Models/DistributedJobSettings.cs

@@ -0,0 +1,25 @@
+using System.ComponentModel;
+
+namespace Umbraco.Cms.Core.Configuration.Models;
+
+/// <summary>
+/// Settings for distributed jobs.
+/// </summary>
+[UmbracoOptions(Constants.Configuration.ConfigDistributedJobs)]
+public class DistributedJobSettings
+{
+    internal const string StaticPeriod = "00:00:10";
+    internal const string StaticDelay = "00:01:00";
+
+    /// <summary>
+    ///     Gets or sets a value for the period of checking if there are any runnable distributed jobs.
+    /// </summary>
+    [DefaultValue(StaticPeriod)]
+    public TimeSpan Period { get; set; } = TimeSpan.Parse(StaticPeriod);
+
+    /// <summary>
+    ///     Gets or sets a value for the delay of when to start checking for distributed jobs.
+    /// </summary>
+    [DefaultValue(StaticDelay)]
+    public TimeSpan Delay { get; set; } = TimeSpan.Parse(StaticDelay);
+}

src/Umbraco.Core/Constants-Configuration.cs

@@ -65,6 +65,7 @@ public static class Configuration
         public const string ConfigWebhook = ConfigPrefix + "Webhook";
         public const string ConfigWebhookPayloadType = ConfigWebhook + ":PayloadType";
         public const string ConfigCache = ConfigPrefix + "Cache";
+        public const string ConfigDistributedJobs = ConfigPrefix + "DistributedJobs";
 
         public static class NamedOptions
         {

src/Umbraco.Core/DependencyInjection/UmbracoBuilder.Configuration.cs

@@ -87,7 +87,8 @@ public static IUmbracoBuilder AddConfiguration(this IUmbracoBuilder builder)
             .AddUmbracoOptions<DataTypesSettings>()
             .AddUmbracoOptions<WebhookSettings>()
             .AddUmbracoOptions<CacheSettings>()
-            .AddUmbracoOptions<SystemDateMigrationSettings>();
+            .AddUmbracoOptions<SystemDateMigrationSettings>()
+            .AddUmbracoOptions<DistributedJobSettings>();
 
         // Configure connection string and ensure it's updated when the configuration changes
         builder.Services.AddSingleton<IConfigureOptions<ConnectionStrings>, ConfigureConnectionStrings>();

src/Umbraco.Core/Persistence/Constants-DatabaseSchema.cs

@@ -103,8 +103,8 @@ public static class Tables
             public const string Webhook2Headers = Webhook + "2Headers";
             public const string WebhookLog = Webhook + "Log";
             public const string WebhookRequest = Webhook + "Request";
-
             public const string LongRunningOperation = TableNamePrefix + "LongRunningOperation";
+            public const string DistributedJob = TableNamePrefix + "DistributedJob";
         }
     }
 }

src/Umbraco.Core/Persistence/Constants-Locks.cs

@@ -90,5 +90,10 @@ public static class Locks
         ///     All document URLs.
         /// </summary>
         public const int DocumentUrls = -345;
+
+        /// <summary>
+        ///     All distributed jobs.
+        /// </summary>
+        public const int DistributedJobs = -347;
     }
 }

src/Umbraco.Infrastructure/BackgroundJobs/DistributedBackgroundJobHostedService.cs

@@ -0,0 +1,103 @@
+using System.Diagnostics;
+using Microsoft.Extensions.Hosting;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
+using Umbraco.Cms.Core;
+using Umbraco.Cms.Core.Configuration.Models;
+using Umbraco.Cms.Core.Services;
+using Umbraco.Cms.Infrastructure.Services;
+
+namespace Umbraco.Cms.Infrastructure.BackgroundJobs;
+
+/// <summary>
+///    A hosted service that checks for any runnable distributed background jobs on a timer.
+/// </summary>
+public class DistributedBackgroundJobHostedService : BackgroundService
+{
+    private readonly ILogger<DistributedBackgroundJobHostedService> _logger;
+    private readonly IRuntimeState _runtimeState;
+    private readonly IDistributedJobService _distributedJobService;
+    private DistributedJobSettings _distributedJobSettings;
+
+    /// <summary>
+    /// Initializes a new instance of the <see cref="DistributedBackgroundJobHostedService"/> class.
+    /// </summary>
+    /// <param name="logger"></param>
+    /// <param name="runtimeState"></param>
+    /// <param name="distributedJobService"></param>
+    /// <param name="distributedJobSettings"></param>
+    public DistributedBackgroundJobHostedService(
+        ILogger<DistributedBackgroundJobHostedService> logger,
+        IRuntimeState runtimeState,
+        IDistributedJobService distributedJobService,
+        IOptionsMonitor<DistributedJobSettings> distributedJobSettings)
+    {
+        _logger = logger;
+        _runtimeState = runtimeState;
+        _distributedJobService = distributedJobService;
+        _distributedJobSettings = distributedJobSettings.CurrentValue;
+        distributedJobSettings.OnChange(options =>
+        {
+            _distributedJobSettings = options;
+        });
+    }
+
+    /// <inheritdoc />
+    protected override async Task ExecuteAsync(CancellationToken stoppingToken)
+    {
+        await Task.Delay(_distributedJobSettings.Delay, stoppingToken);
+
+        while (_runtimeState.Level != RuntimeLevel.Run)
+        {
+            await Task.Delay(_distributedJobSettings.Delay, stoppingToken);
+        }
+
+        // Update all jobs, periods might have changed when restarting.
+        await _distributedJobService.EnsureJobsAsync();
+
+        using PeriodicTimer timer = new(_distributedJobSettings.Period);
+
+        try
+        {
+            while (await timer.WaitForNextTickAsync(stoppingToken))
+            {
+                await RunRunnableJob();
+            }
+        }
+        catch (OperationCanceledException)
+        {
+            _logger.LogInformation("Timed Hosted Service is stopping.");
+        }
+    }
+
+    private async Task RunRunnableJob()
+    {
+        IDistributedBackgroundJob? job = await _distributedJobService.TryTakeRunnableAsync();
+
+        if (job is null)
+        {
+            // No runnable jobs for now, return
+            return;
+        }
+
+        try
+        {
+            await job.ExecuteAsync();
+        }
+        catch (Exception ex)
+        {
+            _logger.LogError(ex, "An exception occurred while running distributed background job '{JobName}'.", job.Name);
+        }
+        finally
+        {
+            try
+            {
+                await _distributedJobService.FinishAsync(job.Name);
+            }
+            catch (Exception ex)
+            {
+                _logger.LogError(ex, "An exception occurred while finishing distributed background job '{JobName}'.", job.Name);
+            }
+        }
+    }
+}

src/Umbraco.Infrastructure/BackgroundJobs/IDistributedBackgroundJob.cs

@@ -0,0 +1,23 @@
+namespace Umbraco.Cms.Infrastructure.BackgroundJobs;
+
+/// <summary>
+/// A background job that will be executed by an available server. With a single server setup this will always be the same.
+/// With a load balanced setup, the executing server might change every time this needs to be executed.
+/// </summary>
+public interface IDistributedBackgroundJob
+{
+    /// <summary>
+    /// Name of the job.
+    /// </summary>
+    string Name { get; }
+
+    /// <summary>
+    /// Timespan representing how often the task should recur.
+    /// </summary>
+    TimeSpan Period { get; }
+
+    /// <summary>
+    /// Run the job.
+    /// </summary>
+    Task ExecuteAsync();
+}

…Jobs/Jobs/CacheInstructionsPruningJob.cs …butedJobs/CacheInstructionsPruningJob.cssrc/Umbraco.Infrastructure/BackgroundJobs/Jobs/CacheInstructionsPruningJob.cs renamed to src/Umbraco.Infrastructure/BackgroundJobs/Jobs/DistributedJobs/CacheInstructionsPruningJob.cs src/Umbraco.Infrastructure/BackgroundJobs/Jobs/CacheInstructionsPruningJob.cs renamed to src/Umbraco.Infrastructure/BackgroundJobs/Jobs/DistributedJobs/CacheInstructionsPruningJob.cs

@@ -2,14 +2,13 @@
 using Umbraco.Cms.Core.Configuration.Models;
 using Umbraco.Cms.Core.Persistence.Repositories;
 using Umbraco.Cms.Core.Scoping;
-using Umbraco.Cms.Infrastructure.Scoping;
 
-namespace Umbraco.Cms.Infrastructure.BackgroundJobs.Jobs;
+namespace Umbraco.Cms.Infrastructure.BackgroundJobs.Jobs.DistributedJobs;
 
 /// <summary>
 /// A background job that prunes cache instructions from the database.
 /// </summary>
-public class CacheInstructionsPruningJob : IRecurringBackgroundJob
+internal class CacheInstructionsPruningJob : IDistributedBackgroundJob
 {
     private readonly IOptions<GlobalSettings> _globalSettings;
     private readonly ICacheInstructionRepository _cacheInstructionRepository;
@@ -36,18 +35,13 @@ public CacheInstructionsPruningJob(
         Period = globalSettings.Value.DatabaseServerMessenger.TimeBetweenPruneOperations;
     }
 
-    /// <inheritdoc />
-    public event EventHandler PeriodChanged
-    {
-        add { }
-        remove { }
-    }
+    public string Name => "CacheInstructionsPruningJob";
 
     /// <inheritdoc />
     public TimeSpan Period { get; }
 
     /// <inheritdoc />
-    public Task RunJobAsync()
+    public Task ExecuteAsync()
     {
         DateTimeOffset pruneDate = _timeProvider.GetUtcNow() - _globalSettings.Value.DatabaseServerMessenger.TimeToRetainInstructions;
         using (ICoreScope scope = _scopeProvider.CreateCoreScope())