Remove environment file

Directory.Build.props

@@ -1,12 +1,4 @@
 <Project>
-	<PropertyGroup Condition="'$(Configuration)' != 'Release'">
-		<EnvFile>$(MSBuildThisFileDirectory)development.local.env</EnvFile>
-	</PropertyGroup>
-
-	<Target Name="CopyEnvFile" AfterTargets="Build" Condition="'$(Configuration)' != 'Release'">
-		<Copy SourceFiles="$(EnvFile)" DestinationFolder="$(OutputPath).." />
-	</Target>
-
 	<PropertyGroup>
 		<TargetFramework>net10.0</TargetFramework>
 	</PropertyGroup>

Directory.Packages.props

@@ -22,7 +22,6 @@
     </PackageVersion>
     <PackageVersion Include="Dapper" Version="2.1.66" />
     <PackageVersion Include="DocumentFormat.OpenXml" Version="3.3.0" />
-    <PackageVersion Include="DotNetEnv" Version="3.1.1" />
     <PackageVersion Include="Microsoft.AspNetCore.Authentication.JwtBearer" Version="10.0.0" />
     <PackageVersion Include="Microsoft.AspNetCore.Authentication.OpenIdConnect" Version="10.0.0" />
     <PackageVersion Include="Microsoft.AspNetCore.Components" Version="10.0.0" />

README.md

@@ -13,7 +13,7 @@ HMPPS Creating Future Opportunities (CFO) - Data Management System (DMS). It is
 ## Architecture
 CFO DMS is built as a microservices architecture using .NET Aspire for orchestration. Data flows through the following pipeline:
 
-**File Ingestion → Parsing/Cleaning → Staging → Import → Running Picture → Matching → Clustering**
+**File Ingestion → Parsing/Cleaning → Staging → Import → Running Picture → Blocking/Matching → Clustering**
 
 1. **FileSync** monitors MinIO/S3/FileSystem storage and syncs incoming files
 2. **Parsers/Cleaners** (Offloc, Delius) transform raw PNOMIS and NDelius files into structured records in staging databases

dms.sln

@@ -1,4 +1,4 @@
-
+
 Microsoft Visual Studio Solution File, Format Version 12.00
 # Visual Studio Version 18
 VisualStudioVersion = 18.0.11116.177
@@ -38,9 +38,6 @@ EndProject
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{D3DAF78A-03D7-42EC-8B73-4B4622E61EF8}"
 	ProjectSection(SolutionItems) = preProject
 		.gitignore = .gitignore
-		base.development.env = base.development.env
-		development.docker.env = development.docker.env
-		development.local.env = development.local.env
 		Directory.Build.props = Directory.Build.props
 		Directory.Packages.props = Directory.Packages.props
 		NuGet.config = NuGet.config

src/API/API.csproj

@@ -16,6 +16,10 @@
         <PackageReference Include="Microsoft.AspNetCore.OpenApi" />
         <PackageReference Include="Microsoft.Identity.Web" />
         <PackageReference Include="NSwag.AspNetCore" />
+        <PackageReference Include="Serilog.Extensions.Hosting" />
+        <PackageReference Include="Serilog.Settings.Configuration" />
+        <PackageReference Include="Serilog.Sinks.Console" />
+        <PackageReference Include="Serilog.Sinks.File" />
     </ItemGroup>
 
     <ItemGroup>

src/API/Program.cs

@@ -4,15 +4,13 @@
 using Infrastructure.Middlewares;
 using Microsoft.AspNetCore.Authentication.JwtBearer;
 using Microsoft.Identity.Web;
-using Namotion.Reflection;
 using OpenApiContact = NSwag.OpenApiContact;
 using OpenApiInfo = NSwag.OpenApiInfo;
 using OpenApiSecurityScheme = NSwag.OpenApiSecurityScheme;
 
-
 var builder = WebApplication.CreateBuilder(args);
 
-IConfiguration configuration = builder.Configuration;
+builder.UseDmsSerilog();
 
 // Add service defaults & Aspire components.
 builder.AddServiceDefaults();
@@ -96,7 +94,7 @@
 // Configure the HTTP request pipeline.
 app.UseExceptionHandler();
 
-if (configuration["IsDevelopment"] is not null && configuration.GetValue<bool>("IsDevelopment"))
+if (builder.Configuration["IsDevelopment"] is not null && builder.Configuration.GetValue<bool>("IsDevelopment"))
 {
     app.RegisterDevelopmentEndpoints();
 

src/API/appsettings.json

@@ -1,60 +1,42 @@
 {
-  "Logging": {
-    "LogLevel": {
-      "Default": "Information",
-      "Microsoft.AspNetCore": "Warning"
-    }
-  },
-  "AllowedHosts": "*",
-  "Authentication": {
-    "ApiKey": "3A27A442D05942689AF3800E0FE8B242"
-  },
-  "AzureAd": {
-    "TenantId": "c6874728-71e6-41fe-a9e1-2e8c36776ad8",
-    "Instance": "https://login.microsoftonline.com/",
-    "ClientId": "",
-    "Scopes": {
-      "Read": "dms.read",
-      "Write": "dms.write"
-    }
-  },
   "ConnectionStrings": {
-    "AuditDb": "Data Source=localhost;Initial Catalog=AuditDb;User Id=sa;Password=YourStrong@Passw0rd;TrustServerCertificate=true",
-    "ClusterDb": "Data Source=localhost;Initial Catalog=ClusterDb;User Id=sa;Password=YourStrong@Passw0rd;TrustServerCertificate=true",
-    "DeliusRunningPictureDb": "Data Source=localhost;Initial Catalog=DeliusRunningPictureDb;User Id=sa;Password=YourStrong@Passw0rd;TrustServerCertificate=true",
-    "OfflocRunningPictureDb": "Data Source=localhost;Initial Catalog=OfflocRunningPictureDb;User Id=sa;Password=YourStrong@Passw0rd;TrustServerCertificate=true"
+    "AuditDb": "",
+    "DeliusRunningPictureDb": "",
+    "OfflocRunningPictureDb": "",
+    "ClusterDb": ""
   },
   "Serilog": {
-    "Using": [
-      "Serilog.Sinks.Console"
-    ],
     "MinimumLevel": {
-      "Default": "Debug",
+      "Default": "Information",
       "Override": {
+        "Microsoft.AspNetCore": "Warning",
         "Microsoft": "Information",
-        "System": "Information"
+        "System": "Warning"
       }
     },
     "WriteTo": [
-      {
-        "Name": "Console"
-      },
-      {
+      { "Name": "Console" },
+      { 
         "Name": "File",
         "Args": {
-          "path": "../../logs/log-.txt",
-          "rollingInterval": "Day",
-          "restrictedToMinimumLevel": "Debug"
+          "path": "./logs/API-.txt",
+          "rollingInterval": "Day"
         }
       }
     ],
-    "Enrich": [
-      "FromLogContext",
-      "WithMachineName",
-      "WithThreadId"
-    ],
-    "Properties": {
-      "Application": "DMS"
+    "Enrich": ["FromLogContext", "WithMachineName", "WithProcessId", "WithThreadId"]
+  },
+  "AllowedHosts": "*",
+  "Authentication": {
+    "ApiKey": "3A27A442D05942689AF3800E0FE8B242"
+  },
+  "AzureAd": {
+    "TenantId": "c6874728-71e6-41fe-a9e1-2e8c36776ad8",
+    "Instance": "https://login.microsoftonline.com/",
+    "ClientId": "",
+    "Scopes": {
+      "Read": "dms.read",
+      "Write": "dms.write"
     }
   },
   "IsDevelopment": "False"

src/Aspire/Aspire.AppHost/Extensions/AppExtensions.cs

@@ -10,7 +10,7 @@ public static IResourceBuilder<ProjectResource> AddDmsApi(
         IResourceBuilder<ParameterResource> apiKey,
         IResourceBuilder<ParameterResource> isDevelopment)
     {
-        return builder.AddProject<API>("api")
+        return builder.AddProject<API>("Api")
             .WithDmsDatabaseReference(databases.OfflocRunningPicture)
             .WithDmsDatabaseReference(databases.DeliusRunningPicture)
             .WithDmsDatabaseReference(databases.Cluster)
@@ -23,7 +23,7 @@ public static IResourceBuilder<ProjectResource> AddDmsVisualiser(
         this IDistributedApplicationBuilder builder,
         IResourceBuilder<ProjectResource> apiService)
     {
-        return builder.AddProject<Visualiser>("visualiser")
+        return builder.AddProject<Visualiser>("Visualiser")
             .WithReference(apiService).WaitFor(apiService);
     }
 

src/Blocking/Blocking.csproj

@@ -4,23 +4,10 @@
     <OutputType>Exe</OutputType>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
-    <DockerDefaultTargetOS>Linux</DockerDefaultTargetOS>
-    <DockerfileContext>..\..\..</DockerfileContext>
   </PropertyGroup>
 
   <ItemGroup>
-    <Compile Remove="PythonContext.cs" />
-    <Compile Remove="PythonInterop.cs" />
-    <Compile Remove="PythonResultWrapper.cs" />
-    <Compile Remove="PythonScriptConfig.cs" />
-  </ItemGroup>
-
-  <ItemGroup>
-    <None Remove="BlockingScript.py" />
-  </ItemGroup>
-
-  <ItemGroup>
-	<PackageReference Include="Microsoft.Data.SqlClient" />
+	  <PackageReference Include="Microsoft.Data.SqlClient" />
     <PackageReference Include="Newtonsoft.Json" />
   </ItemGroup>