Removed worker process and have made KeeperData.Bridge.Worker a class library

HussRasit · HussRasit · commit b649800427a9 · 2025-09-24T11:52:06.000+01:00
diff --git a/KeeperData.Bridge.sln b/KeeperData.Bridge.sln
@@ -32,10 +32,6 @@ Project("{E53339B2-1760-4266-BCC7-CA923CBCF16C}") = "docker-compose", "docker-co
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "KeeperData.Bridge.Worker", "src\KeeperData.Bridge.Worker\KeeperData.Bridge.Worker.csproj", "{C20E1970-BB6F-46BC-886E-7E856DAA763E}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "KeeperData.Bridge.Worker.Tests.Unit", "tests\KeeperData.Bridge.Worker.Tests.Unit\KeeperData.Bridge.Worker.Tests.Unit.csproj", "{8830A129-0A8B-49AD-AB5A-BBCE8197C2A5}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "KeeperData.Bridge.Worker.Tests.Component", "tests\KeeperData.Bridge.Worker.Tests.Component\KeeperData.Bridge.Worker.Tests.Component.csproj", "{DBCDA320-A044-43BD-BE05-0B0FAF1ED80E}"
-EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -162,30 +158,6 @@ Global
 		{C20E1970-BB6F-46BC-886E-7E856DAA763E}.Release|x64.Build.0 = Release|Any CPU
 		{C20E1970-BB6F-46BC-886E-7E856DAA763E}.Release|x86.ActiveCfg = Release|Any CPU
 		{C20E1970-BB6F-46BC-886E-7E856DAA763E}.Release|x86.Build.0 = Release|Any CPU
-		{8830A129-0A8B-49AD-AB5A-BBCE8197C2A5}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{8830A129-0A8B-49AD-AB5A-BBCE8197C2A5}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{8830A129-0A8B-49AD-AB5A-BBCE8197C2A5}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{8830A129-0A8B-49AD-AB5A-BBCE8197C2A5}.Debug|x64.Build.0 = Debug|Any CPU
-		{8830A129-0A8B-49AD-AB5A-BBCE8197C2A5}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{8830A129-0A8B-49AD-AB5A-BBCE8197C2A5}.Debug|x86.Build.0 = Debug|Any CPU
-		{8830A129-0A8B-49AD-AB5A-BBCE8197C2A5}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{8830A129-0A8B-49AD-AB5A-BBCE8197C2A5}.Release|Any CPU.Build.0 = Release|Any CPU
-		{8830A129-0A8B-49AD-AB5A-BBCE8197C2A5}.Release|x64.ActiveCfg = Release|Any CPU
-		{8830A129-0A8B-49AD-AB5A-BBCE8197C2A5}.Release|x64.Build.0 = Release|Any CPU
-		{8830A129-0A8B-49AD-AB5A-BBCE8197C2A5}.Release|x86.ActiveCfg = Release|Any CPU
-		{8830A129-0A8B-49AD-AB5A-BBCE8197C2A5}.Release|x86.Build.0 = Release|Any CPU
-		{DBCDA320-A044-43BD-BE05-0B0FAF1ED80E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{DBCDA320-A044-43BD-BE05-0B0FAF1ED80E}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{DBCDA320-A044-43BD-BE05-0B0FAF1ED80E}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{DBCDA320-A044-43BD-BE05-0B0FAF1ED80E}.Debug|x64.Build.0 = Debug|Any CPU
-		{DBCDA320-A044-43BD-BE05-0B0FAF1ED80E}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{DBCDA320-A044-43BD-BE05-0B0FAF1ED80E}.Debug|x86.Build.0 = Debug|Any CPU
-		{DBCDA320-A044-43BD-BE05-0B0FAF1ED80E}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{DBCDA320-A044-43BD-BE05-0B0FAF1ED80E}.Release|Any CPU.Build.0 = Release|Any CPU
-		{DBCDA320-A044-43BD-BE05-0B0FAF1ED80E}.Release|x64.ActiveCfg = Release|Any CPU
-		{DBCDA320-A044-43BD-BE05-0B0FAF1ED80E}.Release|x64.Build.0 = Release|Any CPU
-		{DBCDA320-A044-43BD-BE05-0B0FAF1ED80E}.Release|x86.ActiveCfg = Release|Any CPU
-		{DBCDA320-A044-43BD-BE05-0B0FAF1ED80E}.Release|x86.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -195,8 +167,6 @@ Global
 		{324F5874-B7B9-4B87-9FAB-BDB970971919} = {02EA681E-C7D8-13C7-8484-4AC65E1B71E8}
 		{E2B6D37D-2D17-4B43-83DC-36F1C4961526} = {02EA681E-C7D8-13C7-8484-4AC65E1B71E8}
 		{16AA106D-B4F1-46C9-92F3-548AE89FDA3A} = {02EA681E-C7D8-13C7-8484-4AC65E1B71E8}
-		{8830A129-0A8B-49AD-AB5A-BBCE8197C2A5} = {02EA681E-C7D8-13C7-8484-4AC65E1B71E8}
-		{DBCDA320-A044-43BD-BE05-0B0FAF1ED80E} = {02EA681E-C7D8-13C7-8484-4AC65E1B71E8}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {235917AE-9AF5-49A5-AB06-472A5E5DB30D}
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -66,17 +66,6 @@ services:
     networks:
       - cdp-tenant
 
-  keeperdata_bridge_worker:
-    container_name: "keeperdata_bridge_worker"
-    build:
-      context: .
-      dockerfile: src/KeeperData.Bridge.Worker/Dockerfile
-    depends_on:
-      - mongodb
-      - localstack
-    networks:
-      - cdp-tenant
-
 volumes:
   mongodb-data:
 
diff --git a/src/KeeperData.Bridge.Worker/Dockerfile b/src/KeeperData.Bridge.Worker/Dockerfile
diff --git a/src/KeeperData.Bridge.Worker/Jobs/DataProcessingOrchestratorJob.cs b/src/KeeperData.Bridge.Worker/Jobs/DataProcessingOrchestratorJob.cs
@@ -1,4 +1,5 @@
 using KeeperData.Bridge.Worker.Tasks;
+using Microsoft.Extensions.Logging;
 using Quartz;
 using System;
 using System.Collections.Generic;
@@ -29,7 +30,6 @@ public async Task Execute(IJobExecutionContext context)
             catch (Exception ex)
             {
                 logger.LogError(ex, "Orchestration Job failed.");
-                // Optionally, rethrow the exception to have Quartz mark the job as failed
                 throw;
             }
         }
diff --git a/src/KeeperData.Bridge.Worker/KeeperData.Bridge.Worker.csproj b/src/KeeperData.Bridge.Worker/KeeperData.Bridge.Worker.csproj
@@ -1,4 +1,4 @@
-﻿<Project Sdk="Microsoft.NET.Sdk.Worker">
+﻿<Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
     <TargetFramework>net8.0</TargetFramework>
diff --git a/src/KeeperData.Bridge.Worker/Program.cs b/src/KeeperData.Bridge.Worker/Program.cs
diff --git a/src/KeeperData.Bridge.Worker/Tasks/TaskDownload.cs b/src/KeeperData.Bridge.Worker/Tasks/TaskDownload.cs
@@ -1,3 +1,4 @@
+using Microsoft.Extensions.Logging;
 using System;
 using System.Collections.Generic;
 using System.Linq;
diff --git a/src/KeeperData.Bridge.Worker/Tasks/TaskProcess.cs b/src/KeeperData.Bridge.Worker/Tasks/TaskProcess.cs
@@ -1,3 +1,4 @@
+using Microsoft.Extensions.Logging;
 using System;
 using System.Collections.Generic;
 using System.Linq;
diff --git a/src/KeeperData.Bridge.Worker/appsettings.Development.json b/src/KeeperData.Bridge.Worker/appsettings.Development.json
diff --git a/src/KeeperData.Bridge.Worker/appsettings.json b/src/KeeperData.Bridge.Worker/appsettings.json
diff --git a/src/KeeperData.Bridge/KeeperData.Bridge.csproj b/src/KeeperData.Bridge/KeeperData.Bridge.csproj
@@ -20,6 +20,7 @@
   </ItemGroup>
 
   <ItemGroup>
+    <ProjectReference Include="..\KeeperData.Bridge.Worker\KeeperData.Bridge.Worker.csproj" />
     <ProjectReference Include="..\KeeperData.Infrastructure\KeeperData.Infrastructure.csproj" />
   </ItemGroup>
 
diff --git a/src/KeeperData.Bridge/Setup/ServiceCollectionExtensions.cs b/src/KeeperData.Bridge/Setup/ServiceCollectionExtensions.cs
@@ -1,7 +1,10 @@
 using KeeperData.Application.Setup;
+using KeeperData.Bridge.Worker.Jobs;
+using KeeperData.Bridge.Worker.Tasks;
 using KeeperData.Infrastructure.Database.Setup;
 using KeeperData.Infrastructure.Messaging.Setup;
 using KeeperData.Infrastructure.Storage.Setup;
+using Quartz;
 
 namespace KeeperData.Bridge.Setup
 {
@@ -20,11 +23,48 @@ public static void ConfigureApi(this IServiceCollection services, IConfiguration
             services.AddMessagingDependencies(configuration);
 
             services.AddStorageDependencies(configuration);
+
+            services.AddJobServices();
+            services.AddConfiguredQuartz(configuration);
         }
 
         private static void ConfigureHealthChecks(this IServiceCollection services)
         {
             services.AddHealthChecks();
         }
+
+        private static void AddJobServices(this IServiceCollection services)
+        {
+            services.AddScoped<ITaskDownload, TaskDownload>();
+            services.AddScoped<ITaskProcess, TaskProcess>();
+            services.AddScoped<DataProcessingOrchestratorJob>();
+        }
+
+        private static void AddConfiguredQuartz(this IServiceCollection services, IConfiguration configuration)
+        {
+            services.AddQuartz(q =>
+            {
+                var jobKey = new JobKey("DataProcessingOrchestratorJob");
+                var cronSchedule = configuration["Quartz:Jobs:DataProcessingOrchestrator:CronSchedule"];
+
+                if (string.IsNullOrWhiteSpace(cronSchedule))
+                {
+                    Console.WriteLine("WARNING: Cron schedule for the orchestrator job is not configured. The job will not run.");
+                    return;
+                }
+
+                q.AddJob<DataProcessingOrchestratorJob>(opts => opts.WithIdentity(jobKey));
+
+                q.AddTrigger(opts => opts
+                    .ForJob(jobKey)
+                    .WithIdentity($"{jobKey.Name}-trigger")
+                    .WithCronSchedule(cronSchedule));
+            });
+
+            services.AddQuartzHostedService(q =>
+            {
+                q.WaitForJobsToComplete = false;
+            });
+        }
     }
 }
diff --git a/src/KeeperData.Bridge/appsettings.json b/src/KeeperData.Bridge/appsettings.json
@@ -45,5 +45,15 @@
       "TopicName": "ls-keeper-data-bridge-events",
       "TopicArn": ""
     }
+  },
+  "Quartz": {
+    "Scheduler": {
+      "InstanceName": "KeeperDataBridgeScheduler"
+    },
+    "Jobs": {
+      "DataProcessingOrchestrator": {
+        "CronSchedule": "0/30 * * * * ?"
+      }
+    }
   }
 }
diff --git a/tests/KeeperData.Application.Tests.Unit/KeeperData.Application.Tests.Unit.csproj b/tests/KeeperData.Application.Tests.Unit/KeeperData.Application.Tests.Unit.csproj
@@ -18,6 +18,7 @@
     </PackageReference>
     <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.8.0" />
     <PackageReference Include="Moq" Version="4.20.72" />
+    <PackageReference Include="Quartz" Version="3.15.0" />
     <PackageReference Include="xunit" Version="2.5.3" />
     <PackageReference Include="xunit.runner.visualstudio" Version="2.5.3" />
   </ItemGroup>
diff --git a/tests/KeeperData.Bridge.Tests.Component/Scheduling/SchedulerTests.cs b/tests/KeeperData.Bridge.Tests.Component/Scheduling/SchedulerTests.cs
@@ -0,0 +1,63 @@
+using KeeperData.Bridge.Worker.Jobs;
+using KeeperData.Bridge.Worker.Tasks;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Hosting;
+using Moq;
+using Quartz;
+
+namespace KeeperData.Bridge.Tests.Component;
+
+public class SchedulerTests
+{
+    [Fact]
+    public async Task Scheduler_WhenJobIsTriggered_ExecutesOrchestratorJobSuccessfully()
+    {
+        var jobDidRun = new ManualResetEventSlim(false);
+
+        var mockTaskDownload = new Mock<ITaskDownload>();
+        var mockTaskProcess = new Mock<ITaskProcess>();
+
+        mockTaskProcess.Setup(x => x.RunAsync(It.IsAny<CancellationToken>()))
+            .Returns(() =>
+            {
+                jobDidRun.Set();
+                return Task.CompletedTask;
+            });
+
+        var host = Host.CreateDefaultBuilder()
+            .ConfigureServices((hostContext, services) =>
+            {
+                services.AddScoped<ITaskDownload>(_ => mockTaskDownload.Object);
+                services.AddScoped<ITaskProcess>(_ => mockTaskProcess.Object);
+
+                services.AddScoped<DataProcessingOrchestratorJob>();
+
+                services.AddQuartz(q =>
+                {
+
+                    q.UseInMemoryStore();
+
+                    var jobKey = new JobKey("TestOrchestratorJob");
+
+                    // Durable as don't want a timed trigger in tests
+                    q.AddJob<DataProcessingOrchestratorJob>(opts => opts.WithIdentity(jobKey).StoreDurably());
+                });
+                services.AddQuartzHostedService(q => q.WaitForJobsToComplete = true);
+            }).Build();
+
+
+        await host.StartAsync();
+
+        var scheduler = host.Services.GetRequiredService<ISchedulerFactory>().GetScheduler().Result;
+
+        await scheduler.TriggerJob(new JobKey("TestOrchestratorJob"));
+
+        var completedInTime = jobDidRun.Wait(TimeSpan.FromSeconds(10));
+
+        await host.StopAsync();
+
+        Assert.True(completedInTime, "The job did not complete in the expected time.");
+        mockTaskDownload.Verify(x => x.RunAsync(It.IsAny<CancellationToken>()), Times.Once);
+        mockTaskProcess.Verify(x => x.RunAsync(It.IsAny<CancellationToken>()), Times.Once);
+    }
+}
diff --git a/tests/KeeperData.Infrastructure.Tests.Unit/KeeperData.Infrastructure.Tests.Unit.csproj b/tests/KeeperData.Infrastructure.Tests.Unit/KeeperData.Infrastructure.Tests.Unit.csproj
@@ -1,4 +1,4 @@
-<Project Sdk="Microsoft.NET.Sdk">
+﻿<Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
     <TargetFramework>net8.0</TargetFramework>
@@ -25,6 +25,7 @@
 
   <ItemGroup>
     <ProjectReference Include="..\..\src\KeeperData.Infrastructure\KeeperData.Infrastructure.csproj" />
+    <ProjectReference Include="..\..\src\KeeperData.Bridge.Worker\KeeperData.Bridge.Worker.csproj" />
   </ItemGroup>
 
   <ItemGroup>
diff --git a/tests/KeeperData.Infrastructure.Tests.Unit/Scheduling/DataProcessingOrchestratorJobTests.cs b/tests/KeeperData.Infrastructure.Tests.Unit/Scheduling/DataProcessingOrchestratorJobTests.cs

Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,5 @@`
`1`	`1`	`using KeeperData.Bridge.Worker.Tasks;`
	`2`	`+using Microsoft.Extensions.Logging;`
`2`	`3`	`using Quartz;`
`3`	`4`	`using System;`
`4`	`5`	`using System.Collections.Generic;`
`@@ -29,7 +30,6 @@ public async Task Execute(IJobExecutionContext context)`
`29`	`30`	`catch (Exception ex)`
`30`	`31`	`{`
`31`	`32`	`logger.LogError(ex, "Orchestration Job failed.");`
`32`		`- // Optionally, rethrow the exception to have Quartz mark the job as failed`
`33`	`33`	`throw;`
`34`	`34`	`}`
`35`	`35`	`}`
Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		`-<Project Sdk="Microsoft.NET.Sdk.Worker">`
	`1`	`+<Project Sdk="Microsoft.NET.Sdk">`
`2`	`2`
`3`	`3`	`<PropertyGroup>`
`4`	`4`	`<TargetFramework>net8.0</TargetFramework>`
Original file line number	Diff line number	Diff line change
`@@ -1,3 +1,4 @@`
	`1`	`+using Microsoft.Extensions.Logging;`
`1`	`2`	`using System;`
`2`	`3`	`using System.Collections.Generic;`
`3`	`4`	`using System.Linq;`