Updates to the itr journey to display field level warnings (#2706)

### Context

We need to inform users when a record has not been fully created /
updated from an ITR import.
At the moment these records would show as having the status `Success` in
the db / UI.

This change introduces the concept of a Warning status when data in a
row has been partially used to create or update record.

Author: hortha

TeachingRecordSystem/src/TeachingRecordSystem.Core/DataStore/Postgres/Mappings/IntegrationTransactionMapping.cs

@@ -13,6 +13,7 @@ public void Configure(EntityTypeBuilder<IntegrationTransaction> builder)
         builder.Property(p => p.FileName).IsRequired();
         builder.Property(p => p.ImportStatus).IsRequired();
         builder.Property(p => p.SuccessCount).IsRequired();
+        builder.Property(p => p.WarningCount).IsRequired();
         builder.Property(p => p.TotalCount).IsRequired();
         builder.Property(p => p.DuplicateCount).IsRequired();
         builder.Property(p => p.FailureCount).IsRequired();

TeachingRecordSystem/src/TeachingRecordSystem.Core/DataStore/Postgres/Migrations/20251028165543_AddWarningCountToIntegrationTransaction.Designer.cs

@@ -0,0 +1,29 @@
+using Microsoft.EntityFrameworkCore.Migrations;
+
+#nullable disable
+
+namespace TeachingRecordSystem.Core.DataStore.Postgres.Migrations
+{
+    /// <inheritdoc />
+    public partial class AddWarningCountToIntegrationTransaction : Migration
+    {
+        /// <inheritdoc />
+        protected override void Up(MigrationBuilder migrationBuilder)
+        {
+            migrationBuilder.AddColumn<int>(
+                name: "warning_count",
+                table: "integration_transactions",
+                type: "integer",
+                nullable: false,
+                defaultValue: 0);
+        }
+
+        /// <inheritdoc />
+        protected override void Down(MigrationBuilder migrationBuilder)
+        {
+            migrationBuilder.DropColumn(
+                name: "warning_count",
+                table: "integration_transactions");
+        }
+    }
+}

TeachingRecordSystem/src/TeachingRecordSystem.Core/DataStore/Postgres/Migrations/20251028165543_AddWarningCountToIntegrationTransaction.cs

@@ -3722,6 +3722,10 @@ protected override void BuildModel(ModelBuilder modelBuilder)
                         .HasColumnType("integer")
                         .HasColumnName("total_count");
 
+                    b.Property<int>("WarningCount")
+                        .HasColumnType("integer")
+                        .HasColumnName("warning_count");
+
                     b.HasKey("IntegrationTransactionId")
                         .HasName("pk_integration_transactions");
 

TeachingRecordSystem/src/TeachingRecordSystem.Core/DataStore/Postgres/Migrations/TrsDbContextModelSnapshot.cs

@@ -7,6 +7,7 @@ public class IntegrationTransaction
     public required IntegrationTransactionImportStatus ImportStatus { get; set; }
     public required int TotalCount { get; set; }
     public required int SuccessCount { get; set; }
+    public required int WarningCount { get; set; }
     public required int FailureCount { get; set; }
     public required int DuplicateCount { get; set; }
     public required string FileName { get; set; }

TeachingRecordSystem/src/TeachingRecordSystem.Core/DataStore/Postgres/Models/IntegrationTransaction.cs

@@ -0,0 +1,43 @@
+using TeachingRecordSystem.Core.DataStore.Postgres;
+
+namespace TeachingRecordSystem.Core.Jobs;
+
+public class BackfillCapitaImportWarningStatusesJob(TrsDbContext dbContext)
+{
+    public async Task ExecuteAsync(bool dryRun, CancellationToken cancellationToken)
+    {
+        await using var transaction = await dbContext.Database.BeginTransactionAsync(cancellationToken);
+
+        var integrationTransactionsToBackfill = await dbContext.IntegrationTransactions
+            .Include(it => it.IntegrationTransactionRecords)
+            .Where(it => it.InterfaceType == IntegrationTransactionInterfaceType.CapitaImport)
+            .ToListAsync(cancellationToken);
+
+        foreach (var integrationTransaction in integrationTransactionsToBackfill)
+        {
+            int warningCount = integrationTransaction.WarningCount;
+            int successCount = integrationTransaction.SuccessCount;
+
+            foreach (var record in integrationTransaction.IntegrationTransactionRecords!.Where(r => r.Status == IntegrationTransactionRecordStatus.Success && !string.IsNullOrEmpty(r.FailureMessage)))
+            {
+                record.Status = IntegrationTransactionRecordStatus.Warning;
+                warningCount++;
+                successCount--;
+            }
+
+            integrationTransaction.WarningCount = warningCount;
+            integrationTransaction.SuccessCount = successCount;
+
+            await dbContext.SaveChangesAsync(cancellationToken);
+        }
+
+        if (dryRun)
+        {
+            await transaction.RollbackAsync(cancellationToken);
+        }
+        else
+        {
+            await transaction.CommitAsync(cancellationToken);
+        }
+    }
+}

TeachingRecordSystem/src/TeachingRecordSystem.Core/Jobs/BackfillCapitaImportWarningStatusesJob.cs

@@ -32,6 +32,7 @@ public async Task<long> ExecuteAsync(CancellationToken cancellationToken)
             ImportStatus = IntegrationTransactionImportStatus.InProgress,
             TotalCount = 0,
             SuccessCount = 0,
+            WarningCount = 0,
             FailureCount = 0,
             DuplicateCount = 0,
             FileName = fileName,

TeachingRecordSystem/src/TeachingRecordSystem.Core/Jobs/CapitaExportAmendJob.cs

@@ -30,6 +30,7 @@ public async Task<long> ExecuteAsync(CancellationToken cancellationToken)
             ImportStatus = IntegrationTransactionImportStatus.InProgress,
             TotalCount = 0,
             SuccessCount = 0,
+            WarningCount = 0,
             FailureCount = 0,
             DuplicateCount = 0,
             FileName = fileName,

TeachingRecordSystem/src/TeachingRecordSystem.Core/Jobs/CapitaExportNewJob.cs

@@ -78,6 +78,7 @@ public async Task<long> ImportAsync(StreamReader reader, string fileName)
         var records = csv.GetRecords<CapitaImportRecord>().ToList();
         var totalRowCount = 0;
         var successCount = 0;
+        var warningCount = 0;
         var duplicateRowCount = 0;
         var failureRowCount = 0;
         var failureMessage = new StringBuilder();
@@ -89,6 +90,7 @@ public async Task<long> ImportAsync(StreamReader reader, string fileName)
             ImportStatus = IntegrationTransactionImportStatus.InProgress,
             TotalCount = 0,
             SuccessCount = 0,
+            WarningCount = 0,
             FailureCount = 0,
             DuplicateCount = 0,
             FileName = fileName,
@@ -108,6 +110,7 @@ public async Task<long> ImportAsync(StreamReader reader, string fileName)
                 var personId = default(Guid?);
                 var recordStatus = IntegrationTransactionRecordStatus.Success;
                 var potentialDuplicate = false;
+                var hasWarnings = warnings.Any();
                 var rowFailureMessage = new StringBuilder();
                 rowFailureMessage.Append(string.Concat(errors.Select(e => e + ",")));
                 rowFailureMessage.Append(string.Concat(warnings.Select(e => e + ",")));
@@ -224,20 +227,31 @@ public async Task<long> ImportAsync(StreamReader reader, string fileName)
                         else if (ni is not null && person.NationalInsuranceNumber is not null && !person.NationalInsuranceNumber.Equals(row.NINumber))
                         {
                             rowFailureMessage.Append($"Warning: Attempted to update NationalInsuranceNumber from {person.NationalInsuranceNumber} to {row.NINumber}");
+                            hasWarnings = true;
                         }
 
                         // Gender is different to incomming record.
                         if (person.Gender is not null && (int?)person.Gender != row.Gender)
                         {
                             rowFailureMessage.Append($"Warning: Attempted to update gender from {person.Gender} to {(Gender?)row.Gender},");
+                            hasWarnings = true;
                         }
 
                         // lastname is different to incomming record
                         if (row.LastName is not null && !person.LastName.Equals(row.LastName, StringComparison.OrdinalIgnoreCase))
                         {
                             rowFailureMessage.Append($"Warning: Attempted to update lastname from {person.LastName} to {row.LastName},");
+                            hasWarnings = true;
+                        }
+                        if (hasWarnings)
+                        {
+                            recordStatus = IntegrationTransactionRecordStatus.Warning;
+                            warningCount++;
+                        }
+                        else
+                        {
+                            successCount++;
                         }
-                        successCount++;
                     }
                 }
 
@@ -266,6 +280,7 @@ public async Task<long> ImportAsync(StreamReader reader, string fileName)
         // mark job as complete
         integrationJob.TotalCount = totalRowCount;
         integrationJob.SuccessCount = successCount;
+        integrationJob.WarningCount = warningCount;
         integrationJob.FailureCount = failureRowCount;
         integrationJob.DuplicateCount = duplicateRowCount;
         integrationJob.ImportStatus = IntegrationTransactionImportStatus.Success;

TeachingRecordSystem/src/TeachingRecordSystem.Core/Jobs/CapitaImportJob.cs

@@ -38,6 +38,7 @@ public async Task<InductionImportResult> ImportAsync(StreamReader csvReaderStrea
                 ImportStatus = IntegrationTransactionImportStatus.InProgress,
                 TotalCount = 0,
                 SuccessCount = 0,
+                WarningCount = 0,
                 FailureCount = 0,
                 DuplicateCount = 0,
                 FileName = fileName,