diff --git a/Directory.Build.props b/Directory.Build.props index 802a1f7..78603a7 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -1,12 +1,4 @@ - - $(MSBuildThisFileDirectory)development.local.env - - - - - - net10.0 diff --git a/Directory.Packages.props b/Directory.Packages.props index cbe3af7..5cf07b3 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -22,7 +22,6 @@ - diff --git a/README.md b/README.md index 662c0db..99b9376 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ HMPPS Creating Future Opportunities (CFO) - Data Management System (DMS). It is ## Architecture CFO DMS is built as a microservices architecture using .NET Aspire for orchestration. Data flows through the following pipeline: -**File Ingestion → Parsing/Cleaning → Staging → Import → Running Picture → Matching → Clustering** +**File Ingestion → Parsing/Cleaning → Staging → Import → Running Picture → Blocking/Matching → Clustering** 1. **FileSync** monitors MinIO/S3/FileSystem storage and syncs incoming files 2. **Parsers/Cleaners** (Offloc, Delius) transform raw PNOMIS and NDelius files into structured records in staging databases diff --git a/development.local.env b/development.local.env deleted file mode 100644 index a639dd8..0000000 --- a/development.local.env +++ /dev/null @@ -1,54 +0,0 @@ -DOTNET_ENVIRONMENT=Development - -RedundantOfflocFields= Age, Occupation Description, Check Hold Governor, Check Hold General (to be left blank), Check Hold Discipline (to be left blank), Check Hold Allocation, Check Hold Security, Check Hold Medical, Check Hold Parole, ACCT Status (F2052), Status Rank (to be left blank), Pending Transfers (Full Establishment Name), Received From, Vulnerable Prisoner Alert, Height (metres), Complexion, Hair Colour, Left Eye, Right Eye, Build, Facial Shape, Facial Hair, Physical Mark Head, Physical Mark Body, Rule 45/YOI Rule 49, ACCT (Self Harm) Status, ACCT (Self Harm) Start Date, Remark Type Allocation, Remarks Allocation, Remark Type Security, Remarks Security, Remark Type Medical, Remarks Medical, Remark Type Parole, Remarks Parole, Remark Type Discipline, Remarks Discipline, Remark Type General, Remarks General, Remark Type Reception, Remarks Reception, Remark Type Labour, Remarks Labour, Date Of First Movement, Diary Details - -# Rabbit -RABBIT_HOSTING=localhost -RABBIT_USERNAME=dms -RABBIT_PASSWORD=R@bbit - -# Connection Strings - uncomment in production -#USER_ID=sa -#SQL_HOST=localhost,1433 -#SA_PASSWORD=Pass@word -#ConnectionStrings__ClusterDb=Data Source=${SQL_HOST};Initial Catalog=ClusterDb;User Id=${USER_ID};Password=${SA_PASSWORD};TrustServerCertificate=true -#ConnectionStrings__DeliusRunningPictureDb=Data Source=${SQL_HOST};Initial Catalog=DeliusRunningPictureDb;User Id=${USER_ID};Password=${SA_PASSWORD};TrustServerCertificate=true -#ConnectionStrings__DeliusStagingDb=Data Source=${SQL_HOST};Initial Catalog=DeliusStagingDb;User Id=${USER_ID};Password=${SA_PASSWORD};TrustServerCertificate=true -#ConnectionStrings__MatchingDb=Data Source=${SQL_HOST};Initial Catalog=MatchingDb;User Id=${USER_ID};Password=${SA_PASSWORD};TrustServerCertificate=true -#ConnectionStrings__OfflocRunningPictureDb=Data Source=${SQL_HOST};Initial Catalog=OfflocRunningPictureDb;User Id=${USER_ID};Password=${SA_PASSWORD};TrustServerCertificate=true -#ConnectionStrings__OfflocStagingDb=Data Source=${SQL_HOST};Initial Catalog=OfflocStagingDb;User Id=${USER_ID};Password=${SA_PASSWORD};TrustServerCertificate=true - -# Sentry SDK -Sentry_Dsn=https:// -Sentry_Environment=${DOTNET_ENVIRONMENT} - -# Serilog -Serilog__MinimumLevel__Default=Information -Serilog__MinimumLevel__Override__Microsoft=Information -Serilog__MinimumLevel__Override__System=Warning - -# Serilog: Console Sink -Serilog__WriteTo__0__Name=Console - -# Serilog: File Sink -Serilog__WriteTo__1__Name=File -Serilog__WriteTo__1__Args__path=.\logs\log-.txt #Overrides the default appsettings.json value -Serilog__WriteTo__1__Args__rollingInterval=Day - -# Serilog: Sentry Sink -#Serilog__WriteTo__2__Name=Sentry -#Serilog__WriteTo__2__Args__Dsn=${Sentry_Dsn} -#Serilog__WriteTo__2__Args__Environment=${Sentry_Environment} -#Serilog__WriteTo__2__Args__MinimumEventLevel=Information -#Serilog__WriteTo__2__Args__MinimumBreadcrumbLevel=Information -#Serilog__WriteTo__2__Args__AttachStacktrace=true -#Serilog__WriteTo__2__Args__SendDefaultPii=true - -# Serilog: Enrichers -Serilog__Enrich__0=FromLogContext -Serilog__Enrich__1=WithMachineName -Serilog__Enrich__2=WithProcessId -Serilog__Enrich__3=WithThreadId - -# e.g. /app/ -#DMSFilesBasePath=~/DMS/ \ No newline at end of file diff --git a/dms.sln b/dms.sln index 3eeeeb9..053f510 100644 --- a/dms.sln +++ b/dms.sln @@ -1,4 +1,4 @@ - + Microsoft Visual Studio Solution File, Format Version 12.00 # Visual Studio Version 18 VisualStudioVersion = 18.0.11116.177 @@ -38,9 +38,6 @@ EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{D3DAF78A-03D7-42EC-8B73-4B4622E61EF8}" ProjectSection(SolutionItems) = preProject .gitignore = .gitignore - base.development.env = base.development.env - development.docker.env = development.docker.env - development.local.env = development.local.env Directory.Build.props = Directory.Build.props Directory.Packages.props = Directory.Packages.props NuGet.config = NuGet.config diff --git a/src/API/API.csproj b/src/API/API.csproj index 0f6eb6c..84aa7f8 100644 --- a/src/API/API.csproj +++ b/src/API/API.csproj @@ -16,6 +16,10 @@ + + + + diff --git a/src/API/Program.cs b/src/API/Program.cs index 90163e0..6b293f9 100644 --- a/src/API/Program.cs +++ b/src/API/Program.cs @@ -4,15 +4,13 @@ using Infrastructure.Middlewares; using Microsoft.AspNetCore.Authentication.JwtBearer; using Microsoft.Identity.Web; -using Namotion.Reflection; using OpenApiContact = NSwag.OpenApiContact; using OpenApiInfo = NSwag.OpenApiInfo; using OpenApiSecurityScheme = NSwag.OpenApiSecurityScheme; - var builder = WebApplication.CreateBuilder(args); -IConfiguration configuration = builder.Configuration; +builder.UseDmsSerilog(); // Add service defaults & Aspire components. builder.AddServiceDefaults(); @@ -96,7 +94,7 @@ // Configure the HTTP request pipeline. app.UseExceptionHandler(); -if (configuration["IsDevelopment"] is not null && configuration.GetValue("IsDevelopment")) +if (builder.Configuration["IsDevelopment"] is not null && builder.Configuration.GetValue("IsDevelopment")) { app.RegisterDevelopmentEndpoints(); diff --git a/src/API/appsettings.json b/src/API/appsettings.json index 2c2d4ce..4d89857 100644 --- a/src/API/appsettings.json +++ b/src/API/appsettings.json @@ -1,60 +1,42 @@ { - "Logging": { - "LogLevel": { - "Default": "Information", - "Microsoft.AspNetCore": "Warning" - } - }, - "AllowedHosts": "*", - "Authentication": { - "ApiKey": "3A27A442D05942689AF3800E0FE8B242" - }, - "AzureAd": { - "TenantId": "c6874728-71e6-41fe-a9e1-2e8c36776ad8", - "Instance": "https://login.microsoftonline.com/", - "ClientId": "", - "Scopes": { - "Read": "dms.read", - "Write": "dms.write" - } - }, "ConnectionStrings": { - "AuditDb": "Data Source=localhost;Initial Catalog=AuditDb;User Id=sa;Password=YourStrong@Passw0rd;TrustServerCertificate=true", - "ClusterDb": "Data Source=localhost;Initial Catalog=ClusterDb;User Id=sa;Password=YourStrong@Passw0rd;TrustServerCertificate=true", - "DeliusRunningPictureDb": "Data Source=localhost;Initial Catalog=DeliusRunningPictureDb;User Id=sa;Password=YourStrong@Passw0rd;TrustServerCertificate=true", - "OfflocRunningPictureDb": "Data Source=localhost;Initial Catalog=OfflocRunningPictureDb;User Id=sa;Password=YourStrong@Passw0rd;TrustServerCertificate=true" + "AuditDb": "", + "DeliusRunningPictureDb": "", + "OfflocRunningPictureDb": "", + "ClusterDb": "" }, "Serilog": { - "Using": [ - "Serilog.Sinks.Console" - ], "MinimumLevel": { - "Default": "Debug", + "Default": "Information", "Override": { + "Microsoft.AspNetCore": "Warning", "Microsoft": "Information", - "System": "Information" + "System": "Warning" } }, "WriteTo": [ - { - "Name": "Console" - }, - { + { "Name": "Console" }, + { "Name": "File", "Args": { - "path": "../../logs/log-.txt", - "rollingInterval": "Day", - "restrictedToMinimumLevel": "Debug" + "path": "./logs/API-.txt", + "rollingInterval": "Day" } } ], - "Enrich": [ - "FromLogContext", - "WithMachineName", - "WithThreadId" - ], - "Properties": { - "Application": "DMS" + "Enrich": ["FromLogContext", "WithMachineName", "WithProcessId", "WithThreadId"] + }, + "AllowedHosts": "*", + "Authentication": { + "ApiKey": "3A27A442D05942689AF3800E0FE8B242" + }, + "AzureAd": { + "TenantId": "c6874728-71e6-41fe-a9e1-2e8c36776ad8", + "Instance": "https://login.microsoftonline.com/", + "ClientId": "", + "Scopes": { + "Read": "dms.read", + "Write": "dms.write" } }, "IsDevelopment": "False" diff --git a/src/Aspire/Aspire.AppHost/Extensions/AppExtensions.cs b/src/Aspire/Aspire.AppHost/Extensions/AppExtensions.cs index c05217e..d9f9cd5 100644 --- a/src/Aspire/Aspire.AppHost/Extensions/AppExtensions.cs +++ b/src/Aspire/Aspire.AppHost/Extensions/AppExtensions.cs @@ -10,7 +10,7 @@ public static IResourceBuilder AddDmsApi( IResourceBuilder apiKey, IResourceBuilder isDevelopment) { - return builder.AddProject("api") + return builder.AddProject("Api") .WithDmsDatabaseReference(databases.OfflocRunningPicture) .WithDmsDatabaseReference(databases.DeliusRunningPicture) .WithDmsDatabaseReference(databases.Cluster) @@ -23,7 +23,7 @@ public static IResourceBuilder AddDmsVisualiser( this IDistributedApplicationBuilder builder, IResourceBuilder apiService) { - return builder.AddProject("visualiser") + return builder.AddProject("Visualiser") .WithReference(apiService).WaitFor(apiService); } diff --git a/src/Blocking/Blocking.csproj b/src/Blocking/Blocking.csproj index 3ed3060..8c7a711 100644 --- a/src/Blocking/Blocking.csproj +++ b/src/Blocking/Blocking.csproj @@ -4,23 +4,10 @@ Exe enable enable - Linux - ..\..\.. - - - - - - - - - - - - + diff --git a/src/Blocking/ConfigurationModels/BlockingConfigurationDefaults.cs b/src/Blocking/ConfigurationModels/BlockingConfigurationDefaults.cs deleted file mode 100644 index 7563656..0000000 --- a/src/Blocking/ConfigurationModels/BlockingConfigurationDefaults.cs +++ /dev/null @@ -1,17 +0,0 @@ - -namespace Blocking.ConfigurationModels; - -public class BlockingConfigurationDefaults -{ - public DatabaseTable offlocTable = new DatabaseTable - { - DatabaseName = "OfflocRunningPictureDb", - TableName = "[OfflocRunningPicture].[RecordMatchingView]" - }; - - public DatabaseTable deliusTable = new DatabaseTable - { - DatabaseName = "DeliusRunningPictureDb", - TableName = "[DeliusRunningPicture].[RecordMatchingView]" - }; -} diff --git a/src/Blocking/ConfigurationModels/BlockingField.cs b/src/Blocking/ConfigurationModels/BlockingField.cs deleted file mode 100644 index c50119c..0000000 --- a/src/Blocking/ConfigurationModels/BlockingField.cs +++ /dev/null @@ -1,10 +0,0 @@ -using Blocking.ConfigurationModels.Enums; - -namespace Blocking.ConfigurationModels; - -public class BlockingField -{ - public TFieldType FieldType { get; init; } - public TableField OfflocField { get; init; } = new(); - public TableField DeliusField { get; init; } = new(); -} diff --git a/src/Blocking/ConfigurationModels/BlockingFieldGroup.cs b/src/Blocking/ConfigurationModels/BlockingFieldGroup.cs deleted file mode 100644 index af9b4b8..0000000 --- a/src/Blocking/ConfigurationModels/BlockingFieldGroup.cs +++ /dev/null @@ -1,19 +0,0 @@ - -namespace Blocking.ConfigurationModels; - -//Necessary to be able to match between different tables. -public class BlockingFieldsGroup -{ - public BlockingField[] BlockingFields { get; init; } = Array.Empty(); - //These could also be views - public DatabaseTable OfflocTable { get; init; } = new DatabaseTable //Default. - { - DatabaseName = "OfflocRunningPictureDb", - TableName = "[OfflocRunningPicture].[RecordMatchingView]" - }; - public DatabaseTable DeliusTable { get; init; } = new DatabaseTable //Default. - { - DatabaseName = "DeliusRunningPictureDb", - TableName = "[DeliusRunningPicture].[RecordMatchingView]" - }; -} diff --git a/src/Blocking/ConfigurationModels/BlockingFieldsConfig.cs b/src/Blocking/ConfigurationModels/BlockingFieldsConfig.cs deleted file mode 100644 index 740b4fe..0000000 --- a/src/Blocking/ConfigurationModels/BlockingFieldsConfig.cs +++ /dev/null @@ -1,8 +0,0 @@ -namespace Blocking.ConfigurationModels; - -//For making configuration type-safe. -public class BlockingFieldsConfig -{ - //Blocking fields in the same inner array will be blocked on together (they will all need to match to be considered a candidate). - public BlockingFieldsGroup[] BlockingFieldsGroups { get; init; } = []; -} diff --git a/src/Blocking/ConfigurationModels/CleaningConfig.cs b/src/Blocking/ConfigurationModels/CleaningConfig.cs deleted file mode 100644 index 65903fa..0000000 --- a/src/Blocking/ConfigurationModels/CleaningConfig.cs +++ /dev/null @@ -1,9 +0,0 @@ -using Blocking.ConfigurationModels.Enums; - -namespace Blocking.ConfigurationModels; - -public class CleaningConfiguration -{ - public TCleaningType CleaningType { get; init; } - public int? TrimAmount { get; init; } -} diff --git a/src/Blocking/ConfigurationModels/DatabaseTable.cs b/src/Blocking/ConfigurationModels/DatabaseTable.cs deleted file mode 100644 index 082003e..0000000 --- a/src/Blocking/ConfigurationModels/DatabaseTable.cs +++ /dev/null @@ -1,34 +0,0 @@ - -namespace Blocking.ConfigurationModels; - -public class DatabaseTable -{ - public string DatabaseName { get; init; } = string.Empty; - public string TableName { get ; init; } = string.Empty; - public string? Columns { get; init; } - - public string FormulateExpression => $"{DatabaseName}.{TableName}"; - - public override bool Equals(object? obj) - { - DatabaseTable castObj; - try - { - castObj = (DatabaseTable)obj; - } - catch (InvalidCastException ex) - { - return false; - } - - if(obj == null) - { - return false; - } - else if(this.FormulateExpression == castObj.FormulateExpression && this.Columns == castObj.Columns) - { - return true; - } - return false; - } -} diff --git a/src/Blocking/ConfigurationModels/Enums/TCleaningType.cs b/src/Blocking/ConfigurationModels/Enums/TCleaningType.cs deleted file mode 100644 index 7c8c56c..0000000 --- a/src/Blocking/ConfigurationModels/Enums/TCleaningType.cs +++ /dev/null @@ -1,9 +0,0 @@ -namespace Blocking.ConfigurationModels.Enums; - -public enum TCleaningType -{ - Normalization, - TrimStart, - TrimEnd, - PNC -} diff --git a/src/Blocking/ConfigurationModels/Enums/TFieldType.cs b/src/Blocking/ConfigurationModels/Enums/TFieldType.cs deleted file mode 100644 index 1ecf88e..0000000 --- a/src/Blocking/ConfigurationModels/Enums/TFieldType.cs +++ /dev/null @@ -1,9 +0,0 @@ -namespace Blocking.ConfigurationModels.Enums; - -public enum TFieldType -{ - String, - Integer, - SpecialIdentifier, //Idenifier with a /. Either CRO or PNC - Date -} diff --git a/src/Blocking/ConfigurationModels/TableField.cs b/src/Blocking/ConfigurationModels/TableField.cs deleted file mode 100644 index 6794553..0000000 --- a/src/Blocking/ConfigurationModels/TableField.cs +++ /dev/null @@ -1,7 +0,0 @@ -namespace Blocking.ConfigurationModels; - -public class TableField -{ - public string FieldName { get; set; } = string.Empty; - public CleaningConfiguration? CleaningConfig { get; init; } -} diff --git a/src/Blocking/DatabaseInsert.cs b/src/Blocking/DatabaseInsert.cs index 1d2c184..65d6281 100644 --- a/src/Blocking/DatabaseInsert.cs +++ b/src/Blocking/DatabaseInsert.cs @@ -1,42 +1,29 @@ - using Blocking.ConfigurationModels; using Microsoft.Data.SqlClient; -using System.ComponentModel.DataAnnotations; using System.Data; -using System.Data.SqlTypes; -using System.Runtime.InteropServices.JavaScript; using Newtonsoft.Json; -using System.Linq.Expressions; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Options; namespace Blocking; -public class DatabaseInsert +public class DatabaseInsert( + IConfiguration configuration, + IOptions storedProceduresOptions, + IOptions blockingQueriesOptions) { - private readonly string connString; - - private readonly string truncateProcedure; - private readonly string insertCandidatesProcedure; - private readonly BlockingQueriesConfig blockingQueriesConfig; - - public DatabaseInsert(ServerConfiguration config) - { - connString = config.connectionString; - - (truncateProcedure, insertCandidatesProcedure, blockingQueriesConfig) = - (config.storedProceduresConfig.TruncateStoredProcedure, - config.storedProceduresConfig.InsertCandidatesStoredProcedure, - config.blockingQueriesConfig); - } - public async Task ClearTables() { - SqlConnection conn = new SqlConnection(connString); + var connectionString = configuration.GetConnectionString("MatchingDb") + ?? throw new InvalidOperationException("Connection string 'MatchingDb' is not configured."); + + SqlConnection conn = new SqlConnection(connectionString); await conn.OpenAsync(); using (conn) { - SqlCommand command = new SqlCommand(truncateProcedure, conn); + SqlCommand command = new SqlCommand(storedProceduresOptions.Value.TruncateStoredProcedure, conn); command.CommandTimeout = 120; command.CommandType = CommandType.StoredProcedure; @@ -47,28 +34,32 @@ public async Task ClearTables() catch (SqlException ex) { Console.WriteLine(ex.ToString()); - throw ex; + throw; } } } public async Task InsertCandidates() { - var blockingQueriesGroups = blockingQueriesConfig.BlockingQueriesGroups; + var blockingQueriesGroups = blockingQueriesOptions.Value.BlockingQueriesGroups; if (blockingQueriesGroups == null || blockingQueriesGroups.Length == 0) { //throw exception? or publish a different message? or may be next phase return; } + + var connectionString = configuration.GetConnectionString("MatchingDb") + ?? throw new InvalidOperationException("Connection string 'MatchingDb' is not configured."); + var jsonQueries = JsonConvert.SerializeObject(blockingQueriesGroups); - SqlConnection conn = new SqlConnection(connString); + SqlConnection conn = new SqlConnection(connectionString); await conn.OpenAsync(); using (conn) { - SqlCommand command = new SqlCommand(insertCandidatesProcedure, conn); + SqlCommand command = new SqlCommand(storedProceduresOptions.Value.InsertCandidatesStoredProcedure, conn); command.CommandTimeout = 600; command.CommandType = CommandType.StoredProcedure; command.Parameters.AddWithValue("@jsonQueries", jsonQueries); @@ -80,7 +71,7 @@ public async Task InsertCandidates() catch (SqlException ex) { Console.WriteLine(ex.ToString()); - throw ex; + throw; } } } diff --git a/src/Blocking/Program.cs b/src/Blocking/Program.cs index 277a9c7..0b0bc24 100644 --- a/src/Blocking/Program.cs +++ b/src/Blocking/Program.cs @@ -1,55 +1,25 @@ -using Messaging.Interfaces; -using Messaging.Services; +using Messaging.Extensions; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; using EnvironmentSetup; using Blocking; using Blocking.ConfigurationModels; -using Newtonsoft.Json; -using Newtonsoft.Json.Linq; using Serilog; -Log.Logger = new LoggerConfiguration() - .Enrich.FromLogContext() - .WriteTo.Console() - .WriteTo.File(@".\logs\fatal.txt", Serilog.Events.LogEventLevel.Fatal) - .CreateBootstrapLogger(); - try { - HostApplicationBuilder builder = Host.CreateApplicationBuilder(args); - - builder.Configuration.AddJsonFile("appsettings.json").AddEnvironmentVariables(); - builder.Configuration.ConfigureByEnvironment(); - - builder.Services.ConfigureServices(builder.Configuration); + var builder = Host.CreateApplicationBuilder(args); - var storedProceduresConfig = builder.Configuration.GetSection("StoredProceduresConfig")!; - var blockingQueriesGroups = builder.Configuration.GetSection("BlockingQueriesConfig")!; + builder.AddDmsCoreWorkerService(); - builder.Services.AddSingleton( - new ServerConfiguration - { - connectionString = builder.Configuration.GetConnectionString("MatchingDb")!, - storedProceduresConfig = builder.Configuration.GetRequiredSection("StoredProceduresConfig").Get()!, - blockingQueriesConfig = builder.Configuration.GetSection("BlockingQueriesConfig").Get()! - } - ); + builder.Services.Configure( + builder.Configuration.GetRequiredSection("StoredProceduresConfig")); + builder.Services.Configure( + builder.Configuration.GetRequiredSection("BlockingQueriesConfig")); builder.Services.AddSingleton(); - - builder.Services.AddSingleton(sp => - { - var rabbitContext = sp.GetRequiredService(); - return RabbitService.CreateAsync(rabbitContext).GetAwaiter().GetResult(); - }); - builder.Services.AddSingleton(sp => sp.GetRequiredService()); - builder.Services.AddSingleton(sp => sp.GetRequiredService()); - builder.Services.AddSingleton(sp => sp.GetRequiredService()); - builder.Services.AddSingleton(sp => sp.GetRequiredService()); - builder.Services.AddSingleton(sp => sp.GetRequiredService()); - + builder.Services.AddDmsRabbitMQ(builder.Configuration); builder.Services.AddHostedService(); diff --git a/src/Blocking/Properties/launchSettings.json b/src/Blocking/Properties/launchSettings.json deleted file mode 100644 index 8646c87..0000000 --- a/src/Blocking/Properties/launchSettings.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "profiles": { - "Blocking": { - "commandName": "Project", - "environmentVariables": { - "DOTNET_ENVIRONMENT": "Development" - } - }, - "Docker": { - "commandName": "Docker" - } - } -} \ No newline at end of file diff --git a/src/Blocking/ServerConfiguration.cs b/src/Blocking/ServerConfiguration.cs deleted file mode 100644 index 5df237d..0000000 --- a/src/Blocking/ServerConfiguration.cs +++ /dev/null @@ -1,11 +0,0 @@ - -using Blocking.ConfigurationModels; - -namespace Blocking; - -public class ServerConfiguration -{ - public string connectionString = string.Empty; - public StoredProceduresConfig storedProceduresConfig = new(); - public BlockingQueriesConfig blockingQueriesConfig = new(); -} diff --git a/src/Blocking/StoredProceduresConfig.cs b/src/Blocking/StoredProceduresConfig.cs index 0d0b222..a390955 100644 --- a/src/Blocking/StoredProceduresConfig.cs +++ b/src/Blocking/StoredProceduresConfig.cs @@ -1,11 +1,7 @@ - -using Blocking.ConfigurationModels; - namespace Blocking; public class StoredProceduresConfig { public string TruncateStoredProcedure { get; set; } = string.Empty; public string InsertCandidatesStoredProcedure { get; set; } = string.Empty; - public BlockingQueriesConfig blockingQueriesConfig { get; set; } = new BlockingQueriesConfig(); } diff --git a/src/Blocking/appsettings.json b/src/Blocking/appsettings.json index 4696206..35cd8eb 100644 --- a/src/Blocking/appsettings.json +++ b/src/Blocking/appsettings.json @@ -1,5 +1,28 @@ { - "Serilog:WriteTo:1:Args:path": "%USERPROFILE%/DMS/logs/Blocking-.txt", + "ConnectionStrings": { + "MatchingDb": "", + "RabbitMQ": "" + }, + "Serilog": { + "MinimumLevel": { + "Default": "Information", + "Override": { + "Microsoft": "Information", + "System": "Warning" + } + }, + "WriteTo": [ + { "Name": "Console" }, + { + "Name": "File", + "Args": { + "path": "./logs/Blocking-.txt", + "rollingInterval": "Day" + } + } + ], + "Enrich": ["FromLogContext", "WithMachineName", "WithProcessId", "WithThreadId"] + }, "StoredProceduresConfig": { "TruncateStoredProcedure": "[Matching].[TruncateTables]", "InsertCandidatesStoredProcedure": "[Matching].[InsertCandidates]", diff --git a/src/Cleanup/Cleanup.csproj b/src/Cleanup/Cleanup.csproj index 7f81d8a..4e16161 100644 --- a/src/Cleanup/Cleanup.csproj +++ b/src/Cleanup/Cleanup.csproj @@ -5,20 +5,8 @@ enable enable 32b35ac4-a44b-4a1e-bdfd-d7ab13dbdc5f - Linux - ..\..\.. - - - - - - - - - - diff --git a/src/Cleanup/Program.cs b/src/Cleanup/Program.cs index c169cde..d0f2b15 100644 --- a/src/Cleanup/Program.cs +++ b/src/Cleanup/Program.cs @@ -1,41 +1,20 @@ - +using Messaging.Extensions; using Cleanup; using Cleanup.CleanupServices.LiveCleanup; using EnvironmentSetup; -using Messaging.Interfaces; -using Messaging.Services; -using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; using Serilog; -Log.Logger = new LoggerConfiguration() - .Enrich.FromLogContext() - .WriteTo.Console() - .WriteTo.File(@".\logs\fatal.txt", Serilog.Events.LogEventLevel.Fatal) - .CreateBootstrapLogger(); - try { - HostApplicationBuilder builder = Host.CreateApplicationBuilder(args); - - builder.Configuration.AddEnvironmentVariables(); - builder.Configuration.ConfigureByEnvironment(); + var builder = Host.CreateApplicationBuilder(args); - builder.Services.ConfigureServices(builder.Configuration); + builder.AddDmsCoreWorkerService(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); - - builder.Services.AddSingleton(sp => - { - var rabbitContext = sp.GetRequiredService(); - return RabbitService.CreateAsync(rabbitContext).GetAwaiter().GetResult(); - }); - builder.Services.AddSingleton(sp => sp.GetRequiredService()); - builder.Services.AddSingleton(sp => sp.GetRequiredService()); - builder.Services.AddSingleton(sp => sp.GetRequiredService()); - builder.Services.AddSingleton(sp => sp.GetRequiredService()); + builder.Services.AddDmsRabbitMQ(builder.Configuration); builder.Services.AddHostedService(); diff --git a/src/Cleanup/appsettings.json b/src/Cleanup/appsettings.json index 906e0ae..d2f4a8c 100644 --- a/src/Cleanup/appsettings.json +++ b/src/Cleanup/appsettings.json @@ -1,6 +1,27 @@ { - "Serilog:WriteTo:1:Args:path": "%USERPROFILE%/DMS/logs/Cleanup-.txt", - "EnvFilePath": "../", + "ConnectionStrings": { + "RabbitMQ": "" + }, + "Serilog": { + "MinimumLevel": { + "Default": "Information", + "Override": { + "Microsoft": "Information", + "System": "Warning" + } + }, + "WriteTo": [ + { "Name": "Console" }, + { + "Name": "File", + "Args": { + "path": "./logs/Cleanup-.txt", + "rollingInterval": "Day" + } + } + ], + "Enrich": ["FromLogContext", "WithMachineName", "WithProcessId", "WithThreadId"] + }, "exclude": [ "**/bin", "**/bower_components", diff --git a/src/DbInteractions/ConnectionStrings.cs b/src/DbInteractions/ConnectionStrings.cs deleted file mode 100644 index 93e42b1..0000000 --- a/src/DbInteractions/ConnectionStrings.cs +++ /dev/null @@ -1,10 +0,0 @@ - -namespace DbInteractions; - -public class ConnectionStrings -{ - public string deliusStagingConnectionString { get; set; } = string.Empty; - public string offlocStagingConnectionString { get; set; } = string.Empty; - public string deliusPictureConnectionString { get; set; } = string.Empty; - public string offlocPictureConnectionString { get; set; } = string.Empty; -} diff --git a/src/DbInteractions/DbInteractions.csproj b/src/DbInteractions/DbInteractions.csproj index f0648c3..4b38dda 100644 --- a/src/DbInteractions/DbInteractions.csproj +++ b/src/DbInteractions/DbInteractions.csproj @@ -4,8 +4,6 @@ enable enable 2a7a51b3-c06a-45a4-a32f-29f1d000aa24 - Linux - ..\..\.. diff --git a/src/DbInteractions/Program.cs b/src/DbInteractions/Program.cs index 17125a5..7137f56 100644 --- a/src/DbInteractions/Program.cs +++ b/src/DbInteractions/Program.cs @@ -1,58 +1,23 @@ -//This project is so all database interactions occur through a single service and so there's +//This project is so all database interactions occur through a single service and so there's //only 1 set of connection strings to maintan. It will also make the examining of logs if //something goes wrong with the database easier in debugging. - using DbInteractions; using DbInteractions.Services; -using Messaging.Interfaces; -using Messaging.Services; -using Microsoft.Extensions.Configuration; +using Messaging.Extensions; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; -using Microsoft.Extensions.Options; using EnvironmentSetup; using Serilog; -Log.Logger = new LoggerConfiguration() - .Enrich.FromLogContext() - .WriteTo.Console() - .WriteTo.File(@".\logs\fatal.txt", Serilog.Events.LogEventLevel.Fatal) - .CreateBootstrapLogger(); - try { - HostApplicationBuilder builder = Host.CreateApplicationBuilder(args); - - builder.Configuration.AddJsonFile("appsettings.json").AddEnvironmentVariables(); - builder.Configuration.ConfigureByEnvironment(); - - builder.Services.ConfigureServices(builder.Configuration); - - builder.Services.Configure( - builder.Configuration.GetSection("ServerConfiguration") - ); - - builder.Services.AddSingleton(s => s.GetRequiredService>().Value); + var builder = Host.CreateApplicationBuilder(args); - builder.Services.AddSingleton( - new ConnectionStrings { - deliusStagingConnectionString = builder.Configuration.GetConnectionString("DeliusStagingDb")!, - offlocStagingConnectionString = builder.Configuration.GetConnectionString("OfflocStagingDb")!, - deliusPictureConnectionString = builder.Configuration.GetConnectionString("DeliusRunningPictureDb")!, - offlocPictureConnectionString = builder.Configuration.GetConnectionString("OfflocRunningPictureDb")! - } - ); + builder.AddDmsCoreWorkerService(); - builder.Services.AddSingleton(builder.Configuration); + builder.Services.AddOptions().BindConfiguration("ServerConfiguration"); - builder.Services.AddSingleton(sp => - { - var rabbitContext = sp.GetRequiredService(); - return RabbitService.CreateAsync(rabbitContext).GetAwaiter().GetResult(); - }); - builder.Services.AddSingleton(sp => sp.GetRequiredService()); - builder.Services.AddSingleton(sp => sp.GetRequiredService()); - builder.Services.AddSingleton(sp => sp.GetRequiredService()); + builder.Services.AddDmsRabbitMQ(builder.Configuration); builder.Services.AddSingleton(); builder.Services.AddHostedService(); diff --git a/src/DbInteractions/Properties/launchSettings.json b/src/DbInteractions/Properties/launchSettings.json deleted file mode 100644 index 33d63a6..0000000 --- a/src/DbInteractions/Properties/launchSettings.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "profiles": { - "DbInteractions": { - "commandName": "Project", - "environmentVariables": { - "DOTNET_ENVIRONMENT": "Development", - "SYSTEM_ENVIRONMENT": "Development" - } - }, - "Docker": { - "commandName": "Project", - "environmentVariables": { - "DOTNET_ENVIRONMENT": "Development", - "SYSTEM_ENVIRONMENT": "Development" - } - } - } -} \ No newline at end of file diff --git a/src/DbInteractions/Services/DbInteractionService.cs b/src/DbInteractions/Services/DbInteractionService.cs index a0d26db..08cee2c 100644 --- a/src/DbInteractions/Services/DbInteractionService.cs +++ b/src/DbInteractions/Services/DbInteractionService.cs @@ -3,44 +3,35 @@ using Messaging.Messages.StatusMessages; using Microsoft.Data.SqlClient; using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Options; using System.Data; namespace DbInteractions.Services; public class DbInteractionService : IDbInteractionService { - private IStatusMessagingService statusService; - private IFileLocations fileLocations; - - private ServerConfiguration serverConfig; - - private string deliusStagingConnString; - private string offlocStagingConnString; - private string deliusPictureConnString; - private string offlocPictureConnString; - - private bool inContainer; - - public DbInteractionService(IStatusMessagingService messageService, - ConnectionStrings connStrings, ServerConfiguration serverConfig, - IConfiguration config, IFileLocations fileLocations) + private readonly IStatusMessagingService statusService; + private readonly IFileLocations fileLocations; + private readonly ServerConfiguration serverConfig; + private readonly IConfiguration configuration; + private readonly bool inContainer; + + public DbInteractionService( + IStatusMessagingService messageService, + IOptions serverConfig, + IConfiguration config, + IFileLocations fileLocations) { this.statusService = messageService; - this.serverConfig = serverConfig; + this.serverConfig = serverConfig.Value; + this.configuration = config; this.fileLocations = fileLocations; - - deliusStagingConnString = connStrings.deliusStagingConnectionString; - offlocStagingConnString = connStrings.offlocStagingConnectionString; - deliusPictureConnString = connStrings.deliusPictureConnectionString; - offlocPictureConnString = connStrings.offlocPictureConnectionString; - - //Make tidier. - inContainer = config.GetValue("RUNNING_IN_CONTAINER"); + this.inContainer = config.GetValue("RUNNING_IN_CONTAINER"); } public async Task GetProcessedDeliusFileNames() { - SqlConnection conn = new SqlConnection(deliusPictureConnString); + SqlConnection conn = new SqlConnection(configuration.GetConnectionString("DeliusRunningPictureDb")!); await conn.OpenAsync(); @@ -69,7 +60,7 @@ public async Task DeliusGetFileIdLastFull() { int fileId = 0; - SqlConnection conn = new SqlConnection(deliusPictureConnString); + SqlConnection conn = new SqlConnection(configuration.GetConnectionString("DeliusRunningPictureDb")!); using (conn) { @@ -101,7 +92,7 @@ public async Task DeliusGetFileIdLastFull() //Returning primitive types instead of strings should make DMS work better over time. public async Task GetProcessedOfflocIds() { - SqlConnection conn = new SqlConnection(offlocPictureConnString); + SqlConnection conn = new SqlConnection(configuration.GetConnectionString("OfflocRunningPictureDb")!); using (conn) { @@ -133,7 +124,7 @@ public async Task GetProcessedOfflocIds() public async Task GetProcessedOfflocFileNames() { - SqlConnection conn = new SqlConnection(offlocPictureConnString); + SqlConnection conn = new SqlConnection(configuration.GetConnectionString("OfflocRunningPictureDb")!); using (conn) { @@ -172,7 +163,7 @@ public async Task StageDelius(string fileName, string filePath) containerFlag = "N"; } - var deliusConn = new SqlConnection(deliusStagingConnString); + var deliusConn = new SqlConnection(configuration.GetConnectionString("DeliusStagingDb")!); using (deliusConn) { await deliusConn.OpenAsync(); @@ -203,7 +194,7 @@ public async Task StageOffloc(string fileName) string folderName = fileName.Split('.').First(); await statusService.StatusPublishAsync(new StatusUpdateMessage($"Offloc staging started for file {fileName}.")); - var offlocConn = new SqlConnection(offlocStagingConnString); + var offlocConn = new SqlConnection(configuration.GetConnectionString("OfflocStagingDb")!); using (offlocConn) { await offlocConn.OpenAsync(); @@ -229,7 +220,7 @@ public async Task StageOffloc(string fileName) //Calls merge and then on completion public async Task StandardiseDeliusStaging() { - SqlConnection conn = new SqlConnection(deliusStagingConnString); + SqlConnection conn = new SqlConnection(configuration.GetConnectionString("DeliusStagingDb")!); using (conn) { @@ -254,7 +245,7 @@ public async Task StandardiseDeliusStaging() //Calls merge and then on completion public async Task MergeDeliusPicture(string fileName) { - SqlConnection conn = new SqlConnection(deliusPictureConnString); + SqlConnection conn = new SqlConnection(configuration.GetConnectionString("DeliusRunningPictureDb")!); using (conn) { @@ -281,7 +272,7 @@ public async Task MergeDeliusPicture(string fileName) public async Task ClearDeliusStaging() { - SqlConnection conn = new SqlConnection(deliusStagingConnString); + SqlConnection conn = new SqlConnection(configuration.GetConnectionString("DeliusStagingDb")!); using (conn) { @@ -306,7 +297,7 @@ public async Task ClearDeliusStaging() public async Task MergeOfflocPicture(string fileName) { - SqlConnection offlocConn = new SqlConnection(offlocPictureConnString); + SqlConnection offlocConn = new SqlConnection(configuration.GetConnectionString("OfflocRunningPictureDb")!); using (offlocConn) { @@ -333,7 +324,7 @@ public async Task MergeOfflocPicture(string fileName) public async Task ClearOfflocStaging() { - SqlConnection offlocConn = new SqlConnection(offlocStagingConnString); + SqlConnection offlocConn = new SqlConnection(configuration.GetConnectionString("OfflocStagingDb")!); using (offlocConn) { await offlocConn.OpenAsync(); @@ -358,7 +349,7 @@ public async Task ClearOfflocStaging() public async Task CreateOfflocProcessedFileEntry(string fileName, int fileId, string? archiveName = null) { - SqlConnection offlocConn = new(offlocPictureConnString); + SqlConnection offlocConn = new(configuration.GetConnectionString("OfflocRunningPictureDb")!); using (offlocConn) { @@ -388,7 +379,7 @@ public async Task CreateOfflocProcessedFileEntry(string fileName, int fileId, st public async Task CreateDeliusProcessedFileEntry(string fileName, string fileId) { - SqlConnection deliusConn = new(deliusPictureConnString); + SqlConnection deliusConn = new(configuration.GetConnectionString("DeliusRunningPictureDb")!); using (deliusConn) { @@ -417,7 +408,7 @@ public async Task CreateDeliusProcessedFileEntry(string fileName, string fileId) public async Task AssociateOfflocFileWithArchive(string fileName, string archiveName) { - SqlConnection offlocConn = new(offlocPictureConnString); + SqlConnection offlocConn = new(configuration.GetConnectionString("OfflocRunningPictureDb")!); using (offlocConn) { @@ -449,7 +440,7 @@ UPDATE [OfflocRunningPicture].[ProcessedFiles] public async Task IsDeliusReadyForProcessing() { - SqlConnection deliusConn = new(deliusPictureConnString); + SqlConnection deliusConn = new(configuration.GetConnectionString("DeliusRunningPictureDb")!); using (deliusConn) { @@ -487,7 +478,7 @@ WHERE Status <> 'Merged' public async Task IsOfflocReadyForProcessing() { - SqlConnection offlocConn = new(offlocPictureConnString); + SqlConnection offlocConn = new(configuration.GetConnectionString("OfflocRunningPictureDb")!); using (offlocConn) { @@ -525,7 +516,7 @@ WHERE Status <> 'Merged' public async Task GetLastProcessedOfflocFileName() { - SqlConnection offlocConn = new(offlocPictureConnString); + SqlConnection offlocConn = new(configuration.GetConnectionString("OfflocRunningPictureDb")!); using (offlocConn) { @@ -557,7 +548,7 @@ FROM OfflocRunningPicture.ProcessedFiles public async Task GetLastProcessedDeliusFileName() { - SqlConnection deliusConn = new(deliusPictureConnString); + SqlConnection deliusConn = new(configuration.GetConnectionString("DeliusRunningPictureDb")!); using (deliusConn) { diff --git a/src/DbInteractions/appsettings.json b/src/DbInteractions/appsettings.json index 20ab3ff..78cd6af 100644 --- a/src/DbInteractions/appsettings.json +++ b/src/DbInteractions/appsettings.json @@ -1,11 +1,30 @@ { - "Serilog:WriteTo:1:Args:path": "%USERPROFILE%/DMS/logs/DbInterations-.txt", - "EnvFilePath": "../", "ConnectionStrings": { - "deliusStagingConnectionString": "", - "offlocStagingConnectionString": "", - "deliusPictureConnectionString": "", - "offlocPictureConnectionString": "" + "DeliusRunningPictureDb": "", + "OfflocRunningPictureDb": "", + "DeliusStagingDb": "", + "OfflocStagingDb": "", + "RabbitMQ": "" + }, + "Serilog": { + "MinimumLevel": { + "Default": "Information", + "Override": { + "Microsoft": "Information", + "System": "Warning" + } + }, + "WriteTo": [ + { "Name": "Console" }, + { + "Name": "File", + "Args": { + "path": "./logs/DbInteractions-.txt", + "rollingInterval": "Day" + } + } + ], + "Enrich": ["FromLogContext", "WithMachineName", "WithProcessId", "WithThreadId"] }, "ServerConfiguration": { "DeliusStagingProcedure": "DeliusStaging.StageDelius", diff --git a/src/Delius.Parser/AppConfig/ConfigurationParser.cs b/src/Delius.Parser/AppConfig/ConfigurationParser.cs deleted file mode 100644 index 816cb8d..0000000 --- a/src/Delius.Parser/AppConfig/ConfigurationParser.cs +++ /dev/null @@ -1,20 +0,0 @@ -using System.Xml.Serialization; -using Delius.Parser.Configuration.Models; - -namespace Delius.Parser.Config; - -//Parses DeliusConfiguration.xml in same directory to construct a list of lines that can then be used by the processor. -//Assumes structure and type of xml file is correct and simply copies lines in xml file to Line and Field instances. -public static class ConfigurationParser -{ - internal static Line[] GetLines() - { - var ser = new XmlSerializer(typeof(Line[])); - - var streamReader = new StreamReader(AppContext.BaseDirectory + "\\Config\\DeliusConfiguration.xml"); - - var lines = ser.Deserialize(streamReader) as Line[] ?? throw new ApplicationException("Cannot deserialize the file :("); - return lines; - - } -} diff --git a/src/Delius.Parser/AppConfig/Models/NextCloudCredentials.cs b/src/Delius.Parser/AppConfig/Models/NextCloudCredentials.cs deleted file mode 100644 index a3c8031..0000000 --- a/src/Delius.Parser/AppConfig/Models/NextCloudCredentials.cs +++ /dev/null @@ -1,9 +0,0 @@ -namespace Delius.Parser.AppConfig.Models; - -public class NextCloudCredentials -{ - public const string Key = nameof(NextCloudCredentials); - - public string Username { get; set; } = string.Empty; - public string Password { get; set; } = string.Empty; -} diff --git a/src/Delius.Parser/Delius.Parser.csproj b/src/Delius.Parser/Delius.Parser.csproj index 8731d4e..1a6cb4d 100644 --- a/src/Delius.Parser/Delius.Parser.csproj +++ b/src/Delius.Parser/Delius.Parser.csproj @@ -4,8 +4,6 @@ Exe enable enable - Linux - ..\..\.. bed042b3-1d10-47f8-9b97-b516f3ae5895 diff --git a/src/Delius.Parser/Program.cs b/src/Delius.Parser/Program.cs index 332945b..f85a4f0 100644 --- a/src/Delius.Parser/Program.cs +++ b/src/Delius.Parser/Program.cs @@ -1,41 +1,18 @@ - +using Messaging.Extensions; using Delius.Parser; using Delius.Parser.Core; using Delius.Parser.Services; using EnvironmentSetup; -using Messaging.Interfaces; -using Messaging.Services; -using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; using Serilog; -Log.Logger = new LoggerConfiguration() - .Enrich.FromLogContext() - .WriteTo.Console() - .WriteTo.File(@".\logs\fatal.txt", Serilog.Events.LogEventLevel.Fatal) - .CreateBootstrapLogger(); - try { - HostApplicationBuilder builder = Host.CreateApplicationBuilder(args); - - builder.Configuration.AddJsonFile("appsettings.json").AddEnvironmentVariables(); - builder.Configuration.ConfigureByEnvironment(); - - builder.Services.AddSingleton(builder.Configuration); - - builder.Services.ConfigureServices(builder.Configuration); - - builder.Services.AddSingleton(sp => - { - var rabbitContext = sp.GetRequiredService(); - return RabbitService.CreateAsync(rabbitContext).GetAwaiter().GetResult(); - }); - builder.Services.AddSingleton(sp => sp.GetRequiredService()); - builder.Services.AddSingleton(sp => sp.GetRequiredService()); - builder.Services.AddSingleton(sp => sp.GetRequiredService()); + var builder = Host.CreateApplicationBuilder(args); + builder.AddDmsCoreWorkerService(); + builder.Services.AddDmsRabbitMQ(builder.Configuration); builder.Services.AddSingleton(); diff --git a/src/Delius.Parser/Properties/launchSettings.json b/src/Delius.Parser/Properties/launchSettings.json deleted file mode 100644 index 26674c5..0000000 --- a/src/Delius.Parser/Properties/launchSettings.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "profiles": { - "Delius.Parser": { - "commandName": "Project", - "environmentVariables": { - "DOTNET_ENVIRONMENT": "Development" - } - }, - "Docker": { - "commandName": "Project" - } - } -} \ No newline at end of file diff --git a/src/Delius.Parser/appsettings.development.json b/src/Delius.Parser/appsettings.development.json deleted file mode 100644 index 9fd8aee..0000000 --- a/src/Delius.Parser/appsettings.development.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "exclude": [ - "**/bin", - "**/bower_components", - "**/jspm_packages", - "**/node_modules", - "**/obj", - "**/platforms" - ] - -} \ No newline at end of file diff --git a/src/Delius.Parser/appsettings.json b/src/Delius.Parser/appsettings.json index 888ff7d..9b74db2 100644 --- a/src/Delius.Parser/appsettings.json +++ b/src/Delius.Parser/appsettings.json @@ -1,4 +1,25 @@ { - "Serilog:WriteTo:1:Args:path": "%USERPROFILE%/DMS/logs/Delius.Parser-.txt", - "EnvFilePath": "..\\" + "ConnectionStrings": { + "RabbitMQ": "" + }, + "Serilog": { + "MinimumLevel": { + "Default": "Information", + "Override": { + "Microsoft": "Information", + "System": "Warning" + } + }, + "WriteTo": [ + { "Name": "Console" }, + { + "Name": "File", + "Args": { + "path": "./logs/Delius.Parser-.txt", + "rollingInterval": "Day" + } + } + ], + "Enrich": ["FromLogContext", "WithMachineName", "WithProcessId", "WithThreadId"] + } } diff --git a/src/FakeDataSeeder/FakeDataSeeder.csproj b/src/FakeDataSeeder/FakeDataSeeder.csproj index e1382c0..f254e9f 100644 --- a/src/FakeDataSeeder/FakeDataSeeder.csproj +++ b/src/FakeDataSeeder/FakeDataSeeder.csproj @@ -17,5 +17,11 @@ + + + Always + + + \ No newline at end of file diff --git a/src/FakeDataSeeder/appsettings.json b/src/FakeDataSeeder/appsettings.json index e0c297d..9141e13 100644 --- a/src/FakeDataSeeder/appsettings.json +++ b/src/FakeDataSeeder/appsettings.json @@ -1,3 +1,25 @@ { - "ConnectionStrings:ClusterDb": "" + "ConnectionStrings": { + "ClusterDb": "" + }, + "Serilog": { + "MinimumLevel": { + "Default": "Information", + "Override": { + "Microsoft": "Information", + "System": "Warning" + } + }, + "WriteTo": [ + { "Name": "Console" }, + { + "Name": "File", + "Args": { + "path": "./logs/FakeDataSeeder-.txt", + "rollingInterval": "Day" + } + } + ], + "Enrich": ["FromLogContext", "WithMachineName", "WithProcessId", "WithThreadId"] + } } diff --git a/src/FileSync/FileSync.csproj b/src/FileSync/FileSync.csproj index 951977c..cc50bc4 100644 --- a/src/FileSync/FileSync.csproj +++ b/src/FileSync/FileSync.csproj @@ -7,12 +7,6 @@ 8317f847-53e8-4625-a75d-5aab845b6f17 - - - Always - - - @@ -25,4 +19,10 @@ + + + Always + + + diff --git a/src/FileSync/Program.cs b/src/FileSync/Program.cs index 8540cf8..47e2c3d 100644 --- a/src/FileSync/Program.cs +++ b/src/FileSync/Program.cs @@ -1,26 +1,11 @@ -Log.Logger = new LoggerConfiguration() - .Enrich.FromLogContext() - .WriteTo.Console() - .WriteTo.File(@"./logs/fatal.txt", Serilog.Events.LogEventLevel.Fatal) - .CreateBootstrapLogger(); +using Messaging.Extensions; try { var builder = Host.CreateApplicationBuilder(args); - builder.Configuration.ConfigureByEnvironment(); - - builder.Services.ConfigureServices(builder.Configuration); - - builder.Services.AddSingleton(sp => - { - var rabbitContext = sp.GetRequiredService(); - return RabbitService.CreateAsync(rabbitContext).GetAwaiter().GetResult(); - }); - builder.Services.AddSingleton(sp => sp.GetRequiredService()); - builder.Services.AddSingleton(sp => sp.GetRequiredService()); - builder.Services.AddSingleton(sp => sp.GetRequiredService()); - builder.Services.AddSingleton(sp => sp.GetRequiredService()); + builder.AddDmsCoreWorkerService(); + builder.Services.AddDmsRabbitMQ(builder.Configuration); builder.Services.AddOptions().BindConfiguration("SyncOptions"); diff --git a/src/FileSync/appsettings.json b/src/FileSync/appsettings.json index 3ee8180..60c3079 100644 --- a/src/FileSync/appsettings.json +++ b/src/FileSync/appsettings.json @@ -1,4 +1,27 @@ { + "ConnectionStrings": { + "RabbitMQ": "" + }, + "Serilog": { + "MinimumLevel": { + "Default": "Information", + "Override": { + "Microsoft": "Information", + "System": "Warning" + } + }, + "WriteTo": [ + { "Name": "Console" }, + { + "Name": "File", + "Args": { + "path": "./logs/FileSync-.txt", + "rollingInterval": "Day" + } + } + ], + "Enrich": ["FromLogContext", "WithMachineName", "WithProcessId", "WithThreadId"] + }, "SyncOptions": { "ProcessOnStartup": true, "ProcessOnCompletion": true, diff --git a/src/Import/Import.csproj b/src/Import/Import.csproj index dfbab5e..71c2290 100644 --- a/src/Import/Import.csproj +++ b/src/Import/Import.csproj @@ -4,8 +4,6 @@ Exe enable enable - Linux - ..\..\.. diff --git a/src/Import/Program.cs b/src/Import/Program.cs index 9c98f74..9f0918e 100644 --- a/src/Import/Program.cs +++ b/src/Import/Program.cs @@ -1,35 +1,19 @@ +using Messaging.Extensions; using Import; -using Messaging.Interfaces; -using Messaging.Services; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; using EnvironmentSetup; using Serilog; -Log.Logger = new LoggerConfiguration() - .Enrich.FromLogContext() - .WriteTo.Console() - .WriteTo.File(@".\logs\fatal.txt", Serilog.Events.LogEventLevel.Fatal) - .CreateBootstrapLogger(); - try { - HostApplicationBuilder builder = Host.CreateApplicationBuilder(args); + var builder = Host.CreateApplicationBuilder(args); builder.Configuration.AddEnvironmentVariables(); - builder.Configuration.ConfigureByEnvironment(); + builder.AddDmsCoreWorkerService(); + builder.Services.AddDmsRabbitMQ(builder.Configuration); - builder.Services.ConfigureServices(builder.Configuration); - builder.Services.AddSingleton(sp => - { - var rabbitContext = sp.GetRequiredService(); - return RabbitService.CreateAsync(rabbitContext).GetAwaiter().GetResult(); - }); - builder.Services.AddSingleton(sp => sp.GetRequiredService()); - builder.Services.AddSingleton(sp => sp.GetRequiredService()); - builder.Services.AddSingleton(sp => sp.GetRequiredService()); - builder.Services.AddSingleton(sp => sp.GetRequiredService()); builder.Services.AddHostedService(); var app = builder.Build(); diff --git a/src/Import/Properties/launchSettings.json b/src/Import/Properties/launchSettings.json deleted file mode 100644 index 3369a71..0000000 --- a/src/Import/Properties/launchSettings.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "profiles": { - "Import": { - "commandName": "Project", - "environmentVariables": { - "DOTNET_ENVIRONMENT": "Development", - "SYSTEM_ENVIRONMENT": "Development" - } - }, - "Docker": { - "commandName": "Project", - "environmentVariables": { - "SYSTEM_ENVIRONMENT": "Development", - "DOTNET_ENVIRONMENT": "Development" - } - } - } -} \ No newline at end of file diff --git a/src/Import/appsettings.json b/src/Import/appsettings.json index cd0c9a1..ca4ab15 100644 --- a/src/Import/appsettings.json +++ b/src/Import/appsettings.json @@ -1,6 +1,27 @@ { - "Serilog:WriteTo:1:Args:path": "%USERPROFILE%/DMS/logs/Import-.txt", - "EnvFilePath": "../", + "ConnectionStrings": { + "RabbitMQ": "" + }, + "Serilog": { + "MinimumLevel": { + "Default": "Information", + "Override": { + "Microsoft": "Information", + "System": "Warning" + } + }, + "WriteTo": [ + { "Name": "Console" }, + { + "Name": "File", + "Args": { + "path": "./logs/Import-.txt", + "rollingInterval": "Day" + } + } + ], + "Enrich": ["FromLogContext", "WithMachineName", "WithProcessId", "WithThreadId"] + }, "ServerConfiguration": { "deliusStagingProcedure": "dbo.StageDelius", "offlocStagingProcedure": "dbo.Import" diff --git a/src/Libraries/EnvironmentSetup/ConfigureEnvironment.cs b/src/Libraries/EnvironmentSetup/ConfigureEnvironment.cs deleted file mode 100644 index 865f6ca..0000000 --- a/src/Libraries/EnvironmentSetup/ConfigureEnvironment.cs +++ /dev/null @@ -1,108 +0,0 @@ -using DotNetEnv.Configuration; -using FileStorage; -using Microsoft.Extensions.Configuration; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Hosting; -using Microsoft.Extensions.Logging; -using Serilog; - -namespace EnvironmentSetup; - -public static class ConfigureEnvironment -{ - private static string envFileBasePath = $"{AppContext.BaseDirectory}.."; - - public static ConfigurationManager ConfigureByEnvironment(this ConfigurationManager configManager) - { - Directory.SetCurrentDirectory(AppContext.BaseDirectory); - - configManager.SetDotnetEnvironment(); - - if (!configManager.GetValue("RUNNING_IN_CONTAINER")) - { - ConfigureEnv(configManager); - } - - return configManager; - } - - public static ConfigurationManager ConfigureEnv(this ConfigurationManager configManager) - { - string envFilePath = Path.Combine(envFileBasePath, "development.local.env"); - - if (File.Exists(envFilePath)) - { - configManager.AddDotNetEnv(envFilePath); - } - - return configManager; - } - - public static void SetDotnetEnvironment(this ConfigurationManager config) - { - if (config.GetValue("DOTNET_ENVIRONMENT") == null) - { - config["DOTNET_ENVIRONMENT"] = "Production"; - } - } -} - -public static class ServiceConfiguration -{ - public static IServiceCollection ConfigureServices(this IServiceCollection services, ConfigurationManager configManager) - { - services.AddSingleton( - f => new FileLocations(configManager.GetValue("DMSFilesBasePath")!) - ); - - services.AddWindowsService(); - - services.ConfigureRabbit(configManager); - services.ConfigureLogging(configManager); - - return services; - } - - private static IServiceCollection ConfigureRabbit(this IServiceCollection services, IConfiguration config) - { - RabbitHostingContextWrapper rabbit; - - // Prefer connection string if available - if (config.GetConnectionString("RabbitMQ") is { Length: > 0 } connectionString) - { - rabbit = new RabbitHostingContextWrapper(new Uri(connectionString)); - } - else - { - string? hosting = config.GetValue("RABBIT_HOSTING"); - string? username = config.GetValue("RABBIT_USERNAME"); - string? password = config.GetValue("RABBIT_PASSWORD"); - - rabbit = new RabbitHostingContextWrapper(hosting, username, password); - } - - services.AddSingleton(rabbit); - - return services; - } - - - private static IServiceCollection ConfigureLogging(this IServiceCollection services, IConfiguration config) - { - Log.Logger = new LoggerConfiguration() - .ReadFrom.Configuration(config) - .WriteTo.File(Path.Combine("logs", "fatal.txt"), Serilog.Events.LogEventLevel.Fatal) - .CreateLogger(); - - var loggerFactory = LoggerFactory.Create(builder => - { - builder.AddSerilog(dispose: true); - }); - - services.AddSingleton(loggerFactory); - services.AddLogging(); - - return services; - } - -} diff --git a/src/Libraries/EnvironmentSetup/DmsServiceExtensions.cs b/src/Libraries/EnvironmentSetup/DmsServiceExtensions.cs new file mode 100644 index 0000000..0d68d35 --- /dev/null +++ b/src/Libraries/EnvironmentSetup/DmsServiceExtensions.cs @@ -0,0 +1,69 @@ +using FileStorage; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Serilog; + +namespace EnvironmentSetup; + +public static class DmsServiceExtensions +{ + /// + /// Configures Serilog logging for all DMS applications. + /// Works for both ASP.NET Core apps and Worker Services. + /// Reads configuration from appsettings.json Serilog section. + /// + public static IHostApplicationBuilder UseDmsSerilog(this IHostApplicationBuilder builder) + { + builder.Services.AddSerilog(config => config + .ReadFrom.Configuration(builder.Configuration) + .WriteTo.File(Path.Combine("logs", "fatal.txt"), Serilog.Events.LogEventLevel.Fatal)); + + return builder; + } + + /// + /// Configures the application to run as a Windows Service. + /// Required for all DMS worker services running on Windows Server EC2. + /// + public static IServiceCollection AddDmsWindowsService(this IServiceCollection services) + { + services.AddWindowsService(); + return services; + } + + + + /// + /// Configures file location paths for services that process files. + /// Required by: FileSync, Offloc.Parser, Offloc.Cleaner, DbInteractions, Delius.Parser, Cleanup. + /// + public static IServiceCollection AddDmsFileLocations(this IServiceCollection services, IConfiguration config) + { + services.AddSingleton( + _ => new FileLocations(config.GetValue("DMSFilesBasePath")!) + ); + + return services; + } + + /// + /// Configures common DMS worker service infrastructure: Serilog logging, Windows Service hosting, + /// and file locations. + /// NOTE: RabbitMQ must be configured separately using AddDmsRabbitMQ() from Messaging library. + /// This is a convenience method for the core DMS worker services. + /// + /// The host application builder + /// + /// Suitable for: FileSync, Offloc.Parser, Offloc.Cleaner, DbInteractions, Delius.Parser, Cleanup, Import, Blocking, Matching.Engine, Logging. + /// NOT suitable for: Meow (uses CATS RabbitMQ), API (ASP.NET Core), Visualiser (container). + /// + public static IHostApplicationBuilder AddDmsCoreWorkerService(this IHostApplicationBuilder builder) + { + builder.UseDmsSerilog(); + builder.Services.AddDmsWindowsService(); + builder.Services.AddDmsFileLocations(builder.Configuration); + + return builder; + } +} diff --git a/src/Libraries/EnvironmentSetup/EnvironmentSetup.csproj b/src/Libraries/EnvironmentSetup/EnvironmentSetup.csproj index 12a1f3e..2770135 100644 --- a/src/Libraries/EnvironmentSetup/EnvironmentSetup.csproj +++ b/src/Libraries/EnvironmentSetup/EnvironmentSetup.csproj @@ -6,7 +6,6 @@ - diff --git a/src/Libraries/EnvironmentSetup/RabbitHostingContextWrapper.cs b/src/Libraries/EnvironmentSetup/RabbitHostingContextWrapper.cs deleted file mode 100644 index 3c81bbe..0000000 --- a/src/Libraries/EnvironmentSetup/RabbitHostingContextWrapper.cs +++ /dev/null @@ -1,25 +0,0 @@ - -namespace EnvironmentSetup; - -public class RabbitHostingContextWrapper -{ - private const string defaultContext = "host.docker.internal"; - //Default value assumes docker. - public string Context { get; } = defaultContext; - public string Username { get; } - public string Password { get; } - - public Uri? Uri { get; } - - public RabbitHostingContextWrapper(Uri uri) : this(defaultContext) - { - Uri = uri; - } - - public RabbitHostingContextWrapper(string? hostingContext, string username = "guest", string password = "guest") - { - Context = hostingContext ?? defaultContext; - Username = username; - Password = password; - } -} diff --git a/src/Libraries/Infrastructure/DatabaseExtensions.cs b/src/Libraries/Infrastructure/DatabaseExtensions.cs index b842445..af818c8 100644 --- a/src/Libraries/Infrastructure/DatabaseExtensions.cs +++ b/src/Libraries/Infrastructure/DatabaseExtensions.cs @@ -25,22 +25,22 @@ public static IHostApplicationBuilder AddDatabaseServices(this IHostApplicationB builder.Services.AddDbContext((sp, options) => { - options.UseSqlServer(builder.Configuration.GetConnectionString("AuditDb")); + options.UseSqlServer(GetRequiredConnectionString(builder.Configuration, "AuditDb")); }); builder.Services.AddDbContext((sp, options) => { - options.UseSqlServer(builder.Configuration.GetConnectionString("DeliusRunningPictureDb")); + options.UseSqlServer(GetRequiredConnectionString(builder.Configuration, "DeliusRunningPictureDb")); }); builder.Services.AddDbContext((sp, options) => { - options.UseSqlServer(builder.Configuration.GetConnectionString("OfflocRunningPictureDb")); + options.UseSqlServer(GetRequiredConnectionString(builder.Configuration, "OfflocRunningPictureDb")); }); builder.Services.AddDbContext((sp, options) => { - options.UseSqlServer(builder.Configuration.GetConnectionString("ClusterDb")); + options.UseSqlServer(GetRequiredConnectionString(builder.Configuration, "ClusterDb")); options.AddInterceptors(sp.GetRequiredService()); }); @@ -51,4 +51,7 @@ public static IHostApplicationBuilder AddDatabaseServices(this IHostApplicationB return builder; } + + private static string GetRequiredConnectionString(IConfiguration configuration, string name) => + configuration.GetConnectionString(name) ?? throw new InvalidOperationException($"Connection string '{name}' is not configured."); } diff --git a/src/Libraries/Messaging/Extensions/DmsRabbitMQExtensions.cs b/src/Libraries/Messaging/Extensions/DmsRabbitMQExtensions.cs new file mode 100644 index 0000000..2937f5d --- /dev/null +++ b/src/Libraries/Messaging/Extensions/DmsRabbitMQExtensions.cs @@ -0,0 +1,34 @@ +using Messaging.Interfaces; +using Messaging.Services; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; + +namespace Messaging.Extensions; + +public static class DmsRabbitMQExtensions +{ + /// + /// Configures DMS RabbitMQ messaging for worker services. + /// Registers RabbitService and all messaging interfaces it implements. + /// + public static IServiceCollection AddDmsRabbitMQ(this IServiceCollection services, IConfiguration config) + { + services.AddSingleton(sp => + { + var connectionString = config.GetConnectionString("RabbitMQ")!; + return RabbitService.CreateAsync(new Uri(connectionString)).GetAwaiter().GetResult(); + }); + + // Register all messaging interfaces + services.AddSingleton(sp => sp.GetRequiredService()); + services.AddSingleton(sp => sp.GetRequiredService()); + services.AddSingleton(sp => sp.GetRequiredService()); + services.AddSingleton(sp => sp.GetRequiredService()); + services.AddSingleton(sp => sp.GetRequiredService()); + services.AddSingleton(sp => sp.GetRequiredService()); + services.AddSingleton(sp => sp.GetRequiredService()); + services.AddSingleton(sp => sp.GetRequiredService()); + + return services; + } +} diff --git a/src/Libraries/Messaging/Services/RabbitService.cs b/src/Libraries/Messaging/Services/RabbitService.cs index 2fa08c0..d0bbcab 100644 --- a/src/Libraries/Messaging/Services/RabbitService.cs +++ b/src/Libraries/Messaging/Services/RabbitService.cs @@ -9,7 +9,6 @@ using Messaging.Interfaces; using Messaging.Messages.DbMessages.Sending; using Messaging.Messages.DbMessages.Receiving; -using EnvironmentSetup; using Messaging.Messages; using Messaging.Messages.ImportMessages; using Messaging.Messages.BlockingMessages; @@ -29,31 +28,16 @@ private RabbitService() { } - public static async Task CreateAsync(RabbitHostingContextWrapper hostingContext) + public static async Task CreateAsync(Uri connectionUri) { var service = new RabbitService(); - await service.InitializeAsync(hostingContext); + await service.InitializeAsync(connectionUri); return service; } - private async Task InitializeAsync(RabbitHostingContextWrapper hostingContext) + private async Task InitializeAsync(Uri connectionUri) { - ConnectionFactory factory; - - // Configures connection factory based on whether a full URI is provided or individual components. - if (hostingContext.Uri is not null) - { - factory = new ConnectionFactory() { Uri = hostingContext.Uri }; - } - else - { - factory = new ConnectionFactory() - { - HostName = hostingContext.Context, - UserName = hostingContext.Username, - Password = hostingContext.Password - }; - } + var factory = new ConnectionFactory() { Uri = connectionUri }; connection = await factory.CreateConnectionAsync(); channel = await connection.CreateChannelAsync(); diff --git a/src/Logging/Program.cs b/src/Logging/Program.cs index dafdd73..691ad19 100644 --- a/src/Logging/Program.cs +++ b/src/Logging/Program.cs @@ -1,25 +1,11 @@ -Log.Logger = new LoggerConfiguration() - .Enrich.FromLogContext() - .WriteTo.Console() - .WriteTo.File(@".\logs\fatal.txt", Serilog.Events.LogEventLevel.Fatal) - .CreateBootstrapLogger(); - -try +using Messaging.Extensions; +try { var builder = Host.CreateApplicationBuilder(args); - builder.Configuration.AddJsonFile("appsettings.json").AddEnvironmentVariables(); - - builder.Configuration.ConfigureByEnvironment(); - - builder.Services.ConfigureServices(builder.Configuration); + builder.AddDmsCoreWorkerService(); + builder.Services.AddDmsRabbitMQ(builder.Configuration); - builder.Services.AddSingleton(sp => - { - var rabbitContext = sp.GetRequiredService(); - return RabbitService.CreateAsync(rabbitContext).GetAwaiter().GetResult(); - }); - builder.Services.AddSingleton(sp => sp.GetRequiredService()); builder.Services.AddHostedService(); var app = builder.Build(); diff --git a/src/Logging/Properties/launchSettings.json b/src/Logging/Properties/launchSettings.json deleted file mode 100644 index 9851d14..0000000 --- a/src/Logging/Properties/launchSettings.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "profiles": { - "Logging": { - "commandName": "Project", - "environmentVariables": { - "DOTNET_ENVIRONMENT": "Development" - } - } - } -} \ No newline at end of file diff --git a/src/Logging/appsettings.json b/src/Logging/appsettings.json index c2ca43d..d3e5286 100644 --- a/src/Logging/appsettings.json +++ b/src/Logging/appsettings.json @@ -1,3 +1,25 @@ { - "Serilog:WriteTo:1:Args:path": "%USERPROFILE%/DMS/logs/Logging-.txt" + "ConnectionStrings": { + "RabbitMQ": "" + }, + "Serilog": { + "MinimumLevel": { + "Default": "Information", + "Override": { + "Microsoft": "Information", + "System": "Warning" + } + }, + "WriteTo": [ + { "Name": "Console" }, + { + "Name": "File", + "Args": { + "path": "./logs/Logging-.txt", + "rollingInterval": "Day" + } + } + ], + "Enrich": ["FromLogContext", "WithMachineName", "WithProcessId", "WithThreadId"] + } } \ No newline at end of file diff --git a/src/Matching.Engine/Extensions/DmsExtensions.cs b/src/Matching.Engine/Extensions/DmsExtensions.cs index 232dd26..42b9a84 100644 --- a/src/Matching.Engine/Extensions/DmsExtensions.cs +++ b/src/Matching.Engine/Extensions/DmsExtensions.cs @@ -1,4 +1,5 @@ -using Messaging.Services; +using Messaging.Extensions; +using Messaging.Services; using EnvironmentSetup; namespace Matching.Engine.Extensions; @@ -7,15 +8,7 @@ public static class DmsExtensions { public static void AddMessagingServices(this IHostApplicationBuilder builder) { - builder.Services.AddSingleton(sp => - { - var rabbitContext = sp.GetRequiredService(); - return RabbitService.CreateAsync(rabbitContext).GetAwaiter().GetResult(); - }); - builder.Services.AddSingleton(sp => sp.GetRequiredService()); - builder.Services.AddSingleton(sp => sp.GetRequiredService()); - builder.Services.AddSingleton(sp => sp.GetRequiredService()); - builder.Services.AddSingleton(sp => sp.GetRequiredService()); + builder.Services.AddDmsRabbitMQ(builder.Configuration); } } diff --git a/src/Matching.Engine/Matching.Engine.csproj b/src/Matching.Engine/Matching.Engine.csproj index 7580da1..0e48e1c 100644 --- a/src/Matching.Engine/Matching.Engine.csproj +++ b/src/Matching.Engine/Matching.Engine.csproj @@ -5,21 +5,7 @@ enable enable - - - - - - - - - Always - - - Always - - - + @@ -31,8 +17,14 @@ - - + + + + + + + Always + diff --git a/src/Matching.Engine/Program.cs b/src/Matching.Engine/Program.cs index 14f52a5..b9d17fc 100644 --- a/src/Matching.Engine/Program.cs +++ b/src/Matching.Engine/Program.cs @@ -1,10 +1,4 @@ -Log.Logger = new LoggerConfiguration() - .Enrich.FromLogContext() - .WriteTo.Console() - .WriteTo.File(@".\logs\fatal.txt", Serilog.Events.LogEventLevel.Fatal) - .CreateBootstrapLogger(); - -try +try { var builder = Host.CreateApplicationBuilder(args); @@ -14,8 +8,7 @@ container.RegisterScorers(builder.Configuration); }); - builder.Configuration.ConfigureByEnvironment(); - builder.Services.ConfigureServices(builder.Configuration); + builder.AddDmsCoreWorkerService(); builder.AddMessagingServices(); builder.AddApplicationServices(); diff --git a/src/Matching.Engine/Properties/launchSettings.json b/src/Matching.Engine/Properties/launchSettings.json deleted file mode 100644 index 04cea13..0000000 --- a/src/Matching.Engine/Properties/launchSettings.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "profiles": { - "Matching.Comparer": { - "commandName": "Project", - "environmentVariables": { - "DOTNET_ENVIRONMENT": "Development" - } - } - } -} diff --git a/src/Matching.Engine/appsettings.Development.json b/src/Matching.Engine/appsettings.Development.json deleted file mode 100644 index 0db3279..0000000 --- a/src/Matching.Engine/appsettings.Development.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - -} diff --git a/src/Matching.Engine/appsettings.json b/src/Matching.Engine/appsettings.json index c6243ed..41edfac 100644 --- a/src/Matching.Engine/appsettings.json +++ b/src/Matching.Engine/appsettings.json @@ -1,8 +1,28 @@ { - "Serilog:WriteTo:1:Args:path": "%USERPROFILE%/DMS/logs/Matching.Engine-.txt", "ConnectionStrings": { - "MatchingDb": "Server=localhost,32771;User ID=sa;Password=Pass@word;TrustServerCertificate=true;Database=MatchingDb", - "ClusterDb": "Server=localhost,32771;User ID=sa;Password=Pass@word;TrustServerCertificate=true;Database=ClusterDb" + "ClusterDb": "", + "MatchingDb": "", + "RabbitMQ": "" + }, + "Serilog": { + "MinimumLevel": { + "Default": "Information", + "Override": { + "Microsoft": "Information", + "System": "Warning" + } + }, + "WriteTo": [ + { "Name": "Console" }, + { + "Name": "File", + "Args": { + "path": "./logs/Matching.Engine-.txt", + "rollingInterval": "Day" + } + } + ], + "Enrich": ["FromLogContext", "WithMachineName", "WithProcessId", "WithThreadId"] }, "ProcessorConfig": { /* diff --git a/src/Meow/Program.cs b/src/Meow/Program.cs index b00f06a..b06612d 100644 --- a/src/Meow/Program.cs +++ b/src/Meow/Program.cs @@ -3,29 +3,22 @@ using Meow.Features.Participants.Handlers; using Rebus.Serialization; -Log.Logger = new LoggerConfiguration() - .Enrich.FromLogContext() - .WriteTo.Console() - .WriteTo.File(@".\logs\fatal.txt", Serilog.Events.LogEventLevel.Fatal) - .CreateBootstrapLogger(); - try { var builder = Host.CreateApplicationBuilder(args); - var services = builder.Services; - var configuration = builder.Configuration; + builder.UseDmsSerilog(); builder.Services.AddScoped(); builder.AddDatabaseServices(); - services.ConfigureServices(configuration); + builder.Services.AddDmsWindowsService(); builder.Services.AddRebus(configure => { - var connectionString = configuration.GetConnectionString("RabbitMQ"); - var rabbitSettings = configuration.GetRequiredSection("RabbitSettings"); + var connectionString = builder.Configuration.GetConnectionString("RabbitMQ"); + var rabbitSettings = builder.Configuration.GetRequiredSection("RabbitSettings"); string queueName = rabbitSettings["DmsService"]!, exchangeName = rabbitSettings["DirectExchange"]!, @@ -44,9 +37,10 @@ builder.Services.AddHostedService(); - var host = builder.Build(); - - await host.RunAsync(); + var app = builder.Build(); + + Log.Information("Starting application"); + app.Run(); return 0; } diff --git a/src/Meow/appsettings.json b/src/Meow/appsettings.json index f80dfae..5239b81 100644 --- a/src/Meow/appsettings.json +++ b/src/Meow/appsettings.json @@ -1,9 +1,30 @@ { "ConnectionStrings": { - "ClusterDb": "Data Source=localhost;Initial Catalog=ClusterDb;User Id=sa;Password=YourStrong@Passw0rd;TrustServerCertificate=true", - "DeliusRunningPictureDb": "Data Source=localhost;Initial Catalog=DeliusRunningPictureDb;User Id=sa;Password=YourStrong@Passw0rd;TrustServerCertificate=true", - "OfflocRunningPictureDb": "Data Source=localhost;Initial Catalog=OfflocRunningPictureDb;User Id=sa;Password=YourStrong@Passw0rd;TrustServerCertificate=true", - "RabbitMQ": "amqp://guest:guest@localhost:5672" + "AuditDb": "", + "DeliusRunningPictureDb": "", + "OfflocRunningPictureDb": "", + "ClusterDb": "", + "RabbitMQ": "" // Meow uses the CATS RabbitMQ instance + }, + "Serilog": { + "MinimumLevel": { + "Default": "Information", + "Override": { + "Microsoft": "Information", + "System": "Warning" + } + }, + "WriteTo": [ + { "Name": "Console" }, + { + "Name": "File", + "Args": { + "path": "./logs/Meow-.txt", + "rollingInterval": "Day" + } + } + ], + "Enrich": ["FromLogContext", "WithMachineName", "WithProcessId", "WithThreadId"] }, "RabbitSettings": { "TopicExchange": "CatsTopics", diff --git a/src/Offloc.Cleaner/Offloc.Cleaner.csproj b/src/Offloc.Cleaner/Offloc.Cleaner.csproj index faecf36..21d2dfa 100644 --- a/src/Offloc.Cleaner/Offloc.Cleaner.csproj +++ b/src/Offloc.Cleaner/Offloc.Cleaner.csproj @@ -4,8 +4,6 @@ Exe enable enable - Linux - ..\..\.. bd191539-371a-49fd-a9f4-f6c77878fc6a @@ -32,7 +30,5 @@ Always - - - + diff --git a/src/Offloc.Cleaner/Program.cs b/src/Offloc.Cleaner/Program.cs index ede2a27..f843351 100644 --- a/src/Offloc.Cleaner/Program.cs +++ b/src/Offloc.Cleaner/Program.cs @@ -1,44 +1,25 @@ - +using Messaging.Extensions; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; using Offloc.Cleaner; -using Messaging.Services; -using Messaging.Interfaces; using Microsoft.Extensions.Configuration; using Offloc.Cleaner.Services; using EnvironmentSetup; using Serilog; -Log.Logger = new LoggerConfiguration() - .Enrich.FromLogContext() - .WriteTo.Console() - .WriteTo.File(@".\logs\fatal.txt", Serilog.Events.LogEventLevel.Fatal) - .CreateBootstrapLogger(); - try { + var builder = Host.CreateApplicationBuilder(args); - HostApplicationBuilder builder = Host.CreateApplicationBuilder(args); - - builder.Configuration.AddJsonFile("appsettings.json").AddEnvironmentVariables(); - builder.Configuration.ConfigureByEnvironment(); - + builder.AddDmsCoreWorkerService(); + builder.Services.AddSingleton( new RedundantFieldsWrapper( builder.Configuration.GetValue("RedundantOfflocFields")! )); builder.Services.AddSingleton(); - - builder.Services.ConfigureServices(builder.Configuration); - builder.Services.AddSingleton(sp => - { - var rabbitContext = sp.GetRequiredService(); - return RabbitService.CreateAsync(rabbitContext).GetAwaiter().GetResult(); - }); - builder.Services.AddSingleton(sp => sp.GetRequiredService()); - builder.Services.AddSingleton(sp => sp.GetRequiredService()); - builder.Services.AddSingleton(sp => sp.GetRequiredService()); + builder.Services.AddDmsRabbitMQ(builder.Configuration); builder.Services.AddHostedService(); diff --git a/src/Offloc.Cleaner/Properties/launchSettings.json b/src/Offloc.Cleaner/Properties/launchSettings.json deleted file mode 100644 index 374e514..0000000 --- a/src/Offloc.Cleaner/Properties/launchSettings.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "profiles": { - "Offloc.Cleaner": { - "commandName": "Project", - "environmentVariables": { - "DOTNET_ENVIRONMENT": "Development" - } - }, - "Docker": { - "commandName": "Project" - } - } -} \ No newline at end of file diff --git a/src/Offloc.Cleaner/appsettings.json b/src/Offloc.Cleaner/appsettings.json index 9686cab..2673087 100644 --- a/src/Offloc.Cleaner/appsettings.json +++ b/src/Offloc.Cleaner/appsettings.json @@ -1,6 +1,28 @@ { - "Serilog:WriteTo:1:Args:path": "%USERPROFILE%/DMS/logs/Offloc.Cleaner-.txt", - "EnvFilePath": "../", + "ConnectionStrings": { + "RabbitMQ": "" + }, + "RedundantOfflocFields": "Age, Occupation Description, Check Hold Governor, Check Hold General (to be left blank), Check Hold Discipline (to be left blank), Check Hold Allocation, Check Hold Security, Check Hold Medical, Check Hold Parole, ACCT Status (F2052), Status Rank (to be left blank), Pending Transfers (Full Establishment Name), Received From, Vulnerable Prisoner Alert, Height (metres), Complexion, Hair Colour, Left Eye, Right Eye, Build, Facial Shape, Facial Hair, Physical Mark Head, Physical Mark Body, Rule 45/YOI Rule 49, ACCT (Self Harm) Status, ACCT (Self Harm) Start Date, Remark Type Allocation, Remarks Allocation, Remark Type Security, Remarks Security, Remark Type Medical, Remarks Medical, Remark Type Parole, Remarks Parole, Remark Type Discipline, Remarks Discipline, Remark Type General, Remarks General, Remark Type Reception, Remarks Reception, Remark Type Labour, Remarks Labour, Date Of First Movement, Diary Details", + "Serilog": { + "MinimumLevel": { + "Default": "Information", + "Override": { + "Microsoft": "Information", + "System": "Warning" + } + }, + "WriteTo": [ + { "Name": "Console" }, + { + "Name": "File", + "Args": { + "path": "./logs/Offloc.Cleaner-.txt", + "rollingInterval": "Day" + } + } + ], + "Enrich": ["FromLogContext", "WithMachineName", "WithProcessId", "WithThreadId"] + }, //Indexing here assumes 0-indexing. Corresponding Id in Offloc definition is 1 greater. "fieldsToIgnore": [ 11, diff --git a/src/Offloc.Parser/Offloc.Parser.csproj b/src/Offloc.Parser/Offloc.Parser.csproj index 4c0d382..0b8386a 100644 --- a/src/Offloc.Parser/Offloc.Parser.csproj +++ b/src/Offloc.Parser/Offloc.Parser.csproj @@ -4,8 +4,6 @@ Exe enable enable - Linux - ..\..\.. diff --git a/src/Offloc.Parser/Program.cs b/src/Offloc.Parser/Program.cs index ee40367..5eff30d 100644 --- a/src/Offloc.Parser/Program.cs +++ b/src/Offloc.Parser/Program.cs @@ -1,49 +1,22 @@ - +using Messaging.Extensions; using EnvironmentSetup; -using Messaging.Interfaces; -using Messaging.Services; -using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; using Offloc.Parser; -using Offloc.Parser.Configuration; -using Offloc.Parser.Processor; using Offloc.Parser.Services; using Offloc.Parser.Services.TrimmerContext; -using Offloc.Parser.Writers.Factory; using Serilog; -Log.Logger = new LoggerConfiguration() - .Enrich.FromLogContext() - .WriteTo.Console() - .WriteTo.File(@".\logs\fatal.txt", Serilog.Events.LogEventLevel.Fatal) - .CreateBootstrapLogger(); - try { - HostApplicationBuilder builder = Host.CreateApplicationBuilder(args); - - builder.Configuration.AddJsonFile("appsettings.json").AddEnvironmentVariables(); - builder.Configuration.ConfigureByEnvironment(); + var builder = Host.CreateApplicationBuilder(args); - builder.Services.AddSingleton(); + builder.AddDmsCoreWorkerService(); + builder.Services.AddDmsRabbitMQ(builder.Configuration); - //Extremely bad practice but just - builder.Services.AddSingleton(builder.Configuration); - - builder.Services.ConfigureServices(builder.Configuration); - builder.Services.AddSingleton(sp => - { - var rabbitContext = sp.GetRequiredService(); - return RabbitService.CreateAsync(rabbitContext).GetAwaiter().GetResult(); - }); - builder.Services.AddSingleton(sp => sp.GetRequiredService()); - builder.Services.AddSingleton(sp => sp.GetRequiredService()); + builder.Services.AddSingleton(); builder.Services.AddSingleton(); - //builder.Services.AddSingleton(); - //builder.Services.AddSingleton(); - //builder.Services.AddSingleton(); builder.Services.AddHostedService(); diff --git a/src/Offloc.Parser/Properties/launchSettings.json b/src/Offloc.Parser/Properties/launchSettings.json deleted file mode 100644 index f0489ae..0000000 --- a/src/Offloc.Parser/Properties/launchSettings.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "profiles": { - "Offloc.Parser": { - "commandName": "Project", - "environmentVariables": { - "DOTNET_ENVIRONMENT": "Development", - "SYSTEM_ENVIRONMENT": "Development" - } - }, - "Docker": { - "commandName": "Project" - } - } -} \ No newline at end of file diff --git a/src/Offloc.Parser/appsettings.json b/src/Offloc.Parser/appsettings.json index 64a3d25..5b8ccb3 100644 --- a/src/Offloc.Parser/appsettings.json +++ b/src/Offloc.Parser/appsettings.json @@ -1,11 +1,27 @@ { - "Serilog:WriteTo:1:Args:path": "%USERPROFILE%/DMS/logs/Offloc.Parser-.txt", - "EnvFilePath": "../", - "Logging": { - "LogLevel": { - "Default": "Warning", - "Microsoft.Hosting.Lifetime": "Warning" - } + "ConnectionStrings": { + "RabbitMQ": "" + }, + "RedundantOfflocFields": "Age, Occupation Description, Check Hold Governor, Check Hold General (to be left blank), Check Hold Discipline (to be left blank), Check Hold Allocation, Check Hold Security, Check Hold Medical, Check Hold Parole, ACCT Status (F2052), Status Rank (to be left blank), Pending Transfers (Full Establishment Name), Received From, Vulnerable Prisoner Alert, Height (metres), Complexion, Hair Colour, Left Eye, Right Eye, Build, Facial Shape, Facial Hair, Physical Mark Head, Physical Mark Body, Rule 45/YOI Rule 49, ACCT (Self Harm) Status, ACCT (Self Harm) Start Date, Remark Type Allocation, Remarks Allocation, Remark Type Security, Remarks Security, Remark Type Medical, Remarks Medical, Remark Type Parole, Remarks Parole, Remark Type Discipline, Remarks Discipline, Remark Type General, Remarks General, Remark Type Reception, Remarks Reception, Remark Type Labour, Remarks Labour, Date Of First Movement, Diary Details", + "Serilog": { + "MinimumLevel": { + "Default": "Information", + "Override": { + "Microsoft": "Information", + "System": "Warning" + } + }, + "WriteTo": [ + { "Name": "Console" }, + { + "Name": "File", + "Args": { + "path": "./logs/Offloc.Parser-.txt", + "rollingInterval": "Day" + } + } + ], + "Enrich": ["FromLogContext", "WithMachineName", "WithProcessId", "WithThreadId"] }, //Indexing here assumes 0-indexing. Corresponding Id in Offloc definition is 1 greater. "fieldsToIgnore": [ diff --git a/src/Visualiser/.dockerignore b/src/Visualiser/.dockerignore deleted file mode 100644 index fe1152b..0000000 --- a/src/Visualiser/.dockerignore +++ /dev/null @@ -1,30 +0,0 @@ -**/.classpath -**/.dockerignore -**/.env -**/.git -**/.gitignore -**/.project -**/.settings -**/.toolstarget -**/.vs -**/.vscode -**/*.*proj.user -**/*.dbmdl -**/*.jfm -**/azds.yaml -**/bin -**/charts -**/docker-compose* -**/Dockerfile* -**/node_modules -**/npm-debug.log -**/obj -**/secrets.dev.yaml -**/values.dev.yaml -LICENSE -README.md -!**/.gitignore -!.git/HEAD -!.git/config -!.git/packed-refs -!.git/refs/heads/** \ No newline at end of file diff --git a/src/Visualiser/Dockerfile b/src/Visualiser/Dockerfile deleted file mode 100644 index ef68649..0000000 --- a/src/Visualiser/Dockerfile +++ /dev/null @@ -1,30 +0,0 @@ -# See https://aka.ms/customizecontainer to learn how to customize your debug container and how Visual Studio uses this Dockerfile to build your images for faster debugging. - -# This stage is used when running from VS in fast mode (Default for Debug configuration) -FROM mcr.microsoft.com/dotnet/aspnet:10.0 AS base -USER $APP_UID -WORKDIR /app -EXPOSE 8080 -EXPOSE 8081 - - -# This stage is used to build the service project -FROM mcr.microsoft.com/dotnet/sdk:10.0 AS build -ARG BUILD_CONFIGURATION=Release -WORKDIR /src -COPY ["Visualiser.csproj", "."] -RUN dotnet restore "./Visualiser.csproj" -COPY . . -WORKDIR "/src/." -RUN dotnet build "./Visualiser.csproj" -c $BUILD_CONFIGURATION -o /app/build - -# This stage is used to publish the service project to be copied to the final stage -FROM build AS publish -ARG BUILD_CONFIGURATION=Release -RUN dotnet publish "./Visualiser.csproj" -c $BUILD_CONFIGURATION -o /app/publish /p:UseAppHost=false - -# This stage is used in production or when running from VS in regular mode (Default when not using the Debug configuration) -FROM base AS final -WORKDIR /app -COPY --from=publish /app/publish . -ENTRYPOINT ["dotnet", "Visualiser.dll"] \ No newline at end of file diff --git a/src/Visualiser/Program.cs b/src/Visualiser/Program.cs index 1382cb9..b325355 100644 --- a/src/Visualiser/Program.cs +++ b/src/Visualiser/Program.cs @@ -1,3 +1,4 @@ +using EnvironmentSetup; using Microsoft.AspNetCore.Authentication.OpenIdConnect; using Microsoft.AspNetCore.HttpOverrides; using Microsoft.Identity.Web; @@ -5,6 +6,8 @@ var builder = WebApplication.CreateBuilder(args); +builder.UseDmsSerilog(); + builder.Services.AddHealthChecks(); builder.Services.AddAuthentication(OpenIdConnectDefaults.AuthenticationScheme) diff --git a/src/Visualiser/Properties/launchSettings.json b/src/Visualiser/Properties/launchSettings.json index 14eadb8..8735697 100644 --- a/src/Visualiser/Properties/launchSettings.json +++ b/src/Visualiser/Properties/launchSettings.json @@ -8,17 +8,6 @@ }, "dotnetRunMessages": true, "applicationUrl": "https://localhost:7123;http://localhost:5015" - }, - "Container (Dockerfile)": { - "commandName": "Docker", - "launchBrowser": true, - "launchUrl": "{Scheme}://{ServiceHost}:{ServicePort}", - "environmentVariables": { - "ASPNETCORE_HTTPS_PORTS": "8081", - "ASPNETCORE_HTTP_PORTS": "8080" - }, - "publishAllPorts": true, - "useSSL": true } }, "$schema": "https://json.schemastore.org/launchsettings.json" diff --git a/src/Visualiser/Visualiser.csproj b/src/Visualiser/Visualiser.csproj index 623fd3c..4671b49 100644 --- a/src/Visualiser/Visualiser.csproj +++ b/src/Visualiser/Visualiser.csproj @@ -5,8 +5,6 @@ enable Visualiser 6b76fc04-e0a8-4265-b2ac-d8600fd901dc - Linux - . @@ -15,6 +13,14 @@ + + + + + + + + diff --git a/src/Visualiser/appsettings.json b/src/Visualiser/appsettings.json index 2753e63..24c3e4f 100644 --- a/src/Visualiser/appsettings.json +++ b/src/Visualiser/appsettings.json @@ -20,11 +20,26 @@ For more info see https://aka.ms/dotnet-template-ms-identity-platform // E.g. "api://{api_client_id}/dms.read" and "api://{api_client_id}/dms.write" ] }, - "Logging": { - "LogLevel": { + "Serilog": { + "MinimumLevel": { "Default": "Information", - "Microsoft.AspNetCore": "Warning" - } + "Override": { + "Microsoft.AspNetCore": "Warning", + "Microsoft": "Information", + "System": "Warning" + } + }, + "WriteTo": [ + { "Name": "Console" }, + { + "Name": "File", + "Args": { + "path": "./logs/Visualiser-.txt", + "rollingInterval": "Day" + } + } + ], + "Enrich": ["FromLogContext", "WithMachineName", "WithProcessId", "WithThreadId"] }, "AllowedHosts": "*" }