diff --git a/src/ServiceLayer.Common/Data/Migrations/20250527094020_ValidationErrors.Designer.cs b/src/ServiceLayer.Common/Data/Migrations/20250527094020_ValidationErrors.Designer.cs new file mode 100644 index 0000000..db67028 --- /dev/null +++ b/src/ServiceLayer.Common/Data/Migrations/20250527094020_ValidationErrors.Designer.cs @@ -0,0 +1,225 @@ +// +using System; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Metadata; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using ServiceLayer.Data; + +#nullable disable + +namespace ServiceLayer.Mesh.Migrations +{ + [DbContext(typeof(ServiceLayerDbContext))] + [Migration("20250527094020_ValidationErrors")] + partial class ValidationErrors + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "9.0.5") + .HasAnnotation("Relational:MaxIdentifierLength", 128); + + SqlServerModelBuilderExtensions.UseIdentityColumns(modelBuilder); + + modelBuilder.Entity("ServiceLayer.Data.Models.MeshFile", b => + { + b.Property("FileId") + .HasMaxLength(255) + .HasColumnType("nvarchar(255)"); + + b.Property("BlobPath") + .HasMaxLength(1024) + .HasColumnType("nvarchar(1024)"); + + b.Property("FileType") + .IsRequired() + .HasMaxLength(50) + .HasColumnType("nvarchar(50)"); + + b.Property("FirstSeenUtc") + .HasColumnType("datetime2"); + + b.Property("LastUpdatedUtc") + .HasColumnType("datetime2"); + + b.Property("MailboxId") + .IsRequired() + .HasMaxLength(50) + .HasColumnType("nvarchar(50)"); + + b.Property("Status") + .IsRequired() + .HasMaxLength(20) + .HasColumnType("nvarchar(20)"); + + b.Property("ValidationErrors") + .HasColumnType("nvarchar(max)"); + + b.HasKey("FileId"); + + b.ToTable("MeshFiles"); + }); + + modelBuilder.Entity("ServiceLayer.Data.Models.NbssAppointmentEvent", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("Action") + .IsRequired() + .HasMaxLength(1) + .HasColumnType("char(1)"); + + b.Property("ActionTimestamp") + .HasColumnType("datetime2(0)"); + + b.Property("AppointmenId") + .IsRequired() + .HasMaxLength(27) + .HasColumnType("varchar(27)"); + + b.Property("AppointmentDateTime") + .HasColumnType("datetime2(0)"); + + b.Property("AppointmentType") + .IsRequired() + .HasMaxLength(1) + .HasColumnType("char(1)"); + + b.Property("AttendedNotScreened") + .HasMaxLength(1) + .HasColumnType("char(1)"); + + b.Property("BSO") + .IsRequired() + .HasMaxLength(3) + .HasColumnType("char(3)"); + + b.Property("BatchId") + .IsRequired() + .HasMaxLength(9) + .HasColumnType("varchar(9)"); + + b.Property("BookedBy") + .IsRequired() + .HasMaxLength(1) + .HasColumnType("char(1)"); + + b.Property("CancelledBy") + .HasMaxLength(1) + .HasColumnType("char(1)"); + + b.Property("ClinicAddressLine1") + .IsRequired() + .HasMaxLength(30) + .HasColumnType("varchar(30)"); + + b.Property("ClinicAddressLine2") + .IsRequired() + .HasMaxLength(30) + .HasColumnType("varchar(30)"); + + b.Property("ClinicAddressLine3") + .IsRequired() + .HasMaxLength(30) + .HasColumnType("varchar(30)"); + + b.Property("ClinicAddressLine4") + .IsRequired() + .HasMaxLength(30) + .HasColumnType("varchar(30)"); + + b.Property("ClinicAddressLine5") + .IsRequired() + .HasMaxLength(30) + .HasColumnType("varchar(30)"); + + b.Property("ClinicCode") + .IsRequired() + .HasMaxLength(5) + .HasColumnType("varchar(5)"); + + b.Property("ClinicName") + .IsRequired() + .HasMaxLength(40) + .HasColumnType("varchar(40)"); + + b.Property("ClinicNameOnLetters") + .IsRequired() + .HasMaxLength(50) + .HasColumnType("varchar(50)"); + + b.Property("ClinicPostcode") + .IsRequired() + .HasMaxLength(8) + .HasColumnType("varchar(8)"); + + b.Property("EpisodeStart") + .HasColumnType("date"); + + b.Property("EpisodeType") + .IsRequired() + .HasMaxLength(1) + .HasColumnType("char(1)"); + + b.Property("ExtractId") + .IsRequired() + .HasMaxLength(8) + .HasColumnType("char(8)"); + + b.Property("HoldingClinic") + .HasMaxLength(1) + .HasColumnType("char(1)"); + + b.Property("Location") + .IsRequired() + .HasMaxLength(5) + .HasColumnType("varchar(5)"); + + b.Property("MeshFileId") + .IsRequired() + .HasMaxLength(255) + .HasColumnType("nvarchar(255)"); + + b.Property("NhsNumber") + .IsRequired() + .HasMaxLength(10) + .HasColumnType("char(10)"); + + b.Property("ScreeningAppointmentNumber") + .HasColumnType("tinyint"); + + b.Property("Sequence") + .IsRequired() + .HasMaxLength(6) + .HasColumnType("char(6)"); + + b.Property("Status") + .IsRequired() + .HasMaxLength(1) + .HasColumnType("char(1)"); + + b.HasKey("Id"); + + b.HasIndex("MeshFileId"); + + b.ToTable("NbssAppointmentEvents"); + }); + + modelBuilder.Entity("ServiceLayer.Data.Models.NbssAppointmentEvent", b => + { + b.HasOne("ServiceLayer.Data.Models.MeshFile", null) + .WithMany() + .HasForeignKey("MeshFileId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/src/ServiceLayer.Common/Data/Migrations/20250527094020_ValidationErrors.cs b/src/ServiceLayer.Common/Data/Migrations/20250527094020_ValidationErrors.cs new file mode 100644 index 0000000..c5c2390 --- /dev/null +++ b/src/ServiceLayer.Common/Data/Migrations/20250527094020_ValidationErrors.cs @@ -0,0 +1,28 @@ +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace ServiceLayer.Mesh.Migrations +{ + /// + public partial class ValidationErrors : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.AddColumn( + name: "ValidationErrors", + table: "MeshFiles", + type: "nvarchar(max)", + nullable: true); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropColumn( + name: "ValidationErrors", + table: "MeshFiles"); + } + } +} diff --git a/src/ServiceLayer.Common/Data/Migrations/ServiceLayerDbContextModelSnapshot.cs b/src/ServiceLayer.Common/Data/Migrations/ServiceLayerDbContextModelSnapshot.cs index 4a05938..6a7f117 100644 --- a/src/ServiceLayer.Common/Data/Migrations/ServiceLayerDbContextModelSnapshot.cs +++ b/src/ServiceLayer.Common/Data/Migrations/ServiceLayerDbContextModelSnapshot.cs @@ -53,6 +53,9 @@ protected override void BuildModel(ModelBuilder modelBuilder) .HasMaxLength(20) .HasColumnType("nvarchar(20)"); + b.Property("ValidationErrors") + .HasColumnType("nvarchar(max)"); + b.HasKey("FileId"); b.ToTable("MeshFiles"); diff --git a/src/ServiceLayer.Common/Data/Models/MeshFile.cs b/src/ServiceLayer.Common/Data/Models/MeshFile.cs index 1963555..3ce8f3a 100644 --- a/src/ServiceLayer.Common/Data/Models/MeshFile.cs +++ b/src/ServiceLayer.Common/Data/Models/MeshFile.cs @@ -16,4 +16,6 @@ public class MeshFile public string? BlobPath { get; set; } public DateTime FirstSeenUtc { get; set; } public DateTime LastUpdatedUtc { get; set; } + // ReSharper disable once EntityFramework.ModelValidation.UnlimitedStringLength + public string? ValidationErrors { get; set; } } diff --git a/src/ServiceLayer.Mesh/Configuration/AppConfiguration.cs b/src/ServiceLayer.Mesh/Configuration/AppConfiguration.cs index 08799cd..fa9e348 100644 --- a/src/ServiceLayer.Mesh/Configuration/AppConfiguration.cs +++ b/src/ServiceLayer.Mesh/Configuration/AppConfiguration.cs @@ -4,7 +4,6 @@ namespace ServiceLayer.Mesh.Configuration; public class AppConfiguration : IFileDiscoveryFunctionConfiguration, - IFileExtractFunctionConfiguration, IFileExtractQueueClientConfiguration, IFileTransformQueueClientConfiguration, IFileTransformFunctionConfiguration, diff --git a/src/ServiceLayer.Mesh/Configuration/IFileExtractFunctionConfiguration.cs b/src/ServiceLayer.Mesh/Configuration/IFileExtractFunctionConfiguration.cs deleted file mode 100644 index 85ed116..0000000 --- a/src/ServiceLayer.Mesh/Configuration/IFileExtractFunctionConfiguration.cs +++ /dev/null @@ -1,6 +0,0 @@ -namespace ServiceLayer.Mesh.Configuration; - -public interface IFileExtractFunctionConfiguration -{ - string NbssMeshMailboxId { get; } -} diff --git a/src/ServiceLayer.Mesh/Extensions/ApiErrorResponseExtensions.cs b/src/ServiceLayer.Mesh/Extensions/ApiErrorResponseExtensions.cs new file mode 100644 index 0000000..5731908 --- /dev/null +++ b/src/ServiceLayer.Mesh/Extensions/ApiErrorResponseExtensions.cs @@ -0,0 +1,11 @@ +using NHS.MESH.Client.Models; + +namespace ServiceLayer.Mesh.Extensions; + +public static class ApiErrorResponseExtensions +{ + public static string ToFormattedString(this APIErrorResponse? error) + { + return error == null ? "Unknown error" : $"ErrorEvent: {error.ErrorEvent ?? "N/A"}, ErrorCode: {error.ErrorCode ?? "N/A"}, ErrorDescription: {error.ErrorDescription}"; + } +} diff --git a/src/ServiceLayer.Mesh/FileTransformerBase.cs b/src/ServiceLayer.Mesh/FileTransformerBase.cs new file mode 100644 index 0000000..7dd9764 --- /dev/null +++ b/src/ServiceLayer.Mesh/FileTransformerBase.cs @@ -0,0 +1,10 @@ +using ServiceLayer.Data.Models; + +namespace ServiceLayer.Mesh; + +public abstract class FileTransformerBase : IFileTransformer +{ + protected abstract MeshFileType HandlesFileType { get; } + public virtual bool CanHandle(MeshFileType fileType) => fileType == HandlesFileType; + public abstract Task> TransformFileAsync(Stream stream, MeshFile metaData); +} diff --git a/src/ServiceLayer.Mesh/FileTypes/NbssAppointmentEvents/FileTransformer.cs b/src/ServiceLayer.Mesh/FileTypes/NbssAppointmentEvents/FileTransformer.cs index da61b08..ce2897a 100644 --- a/src/ServiceLayer.Mesh/FileTypes/NbssAppointmentEvents/FileTransformer.cs +++ b/src/ServiceLayer.Mesh/FileTypes/NbssAppointmentEvents/FileTransformer.cs @@ -3,31 +3,23 @@ namespace ServiceLayer.Mesh.FileTypes.NbssAppointmentEvents; -// TODO - NBSS appointment file specific implementation of IFileTransformer. To orchestrate parsing, validation and staging of data (delegated to separate classes) -public class FileTransformer : IFileTransformer +public class FileTransformer( + IFileParser fileParser, + IValidationRunner validationRunner, + IStagingPersister stagingPersister) + : FileTransformerBase { - private readonly IFileParser _fileParser; - private readonly IValidationRunner _validationRunner; - private readonly IStagingPersister _stagingPersister; + protected override MeshFileType HandlesFileType => MeshFileType.NbssAppointmentEvents; - public FileTransformer(IFileParser fileParser, IValidationRunner validationRunner, IStagingPersister stagingPersister) - { - _fileParser = fileParser; - _validationRunner = validationRunner; - _stagingPersister = stagingPersister; - } - - public MeshFileType HandlesFileType => MeshFileType.NbssAppointmentEvents; - - public async Task> TransformFileAsync(Stream stream, MeshFile metaData) + public override async Task> TransformFileAsync(Stream stream, MeshFile metaData) { // TODO - wrap this parsing in a try-catch and return a List in case of any unforeseen parsing issues (file is totally unlike anything we expect) - var parsed = _fileParser.Parse(stream); + var parsed = fileParser.Parse(stream); - var validationErrors = _validationRunner.Validate(parsed); + var validationErrors = validationRunner.Validate(parsed); if (!validationErrors.Any()) { - await _stagingPersister.WriteStagedData(parsed, metaData); + await stagingPersister.WriteStagedData(parsed, metaData); } return validationErrors; diff --git a/src/ServiceLayer.Mesh/FileTypes/NbssAppointmentEvents/ServiceCollectionExtensions.cs b/src/ServiceLayer.Mesh/FileTypes/NbssAppointmentEvents/ServiceCollectionExtensions.cs new file mode 100644 index 0000000..2e05d24 --- /dev/null +++ b/src/ServiceLayer.Mesh/FileTypes/NbssAppointmentEvents/ServiceCollectionExtensions.cs @@ -0,0 +1,33 @@ +using Microsoft.Extensions.DependencyInjection; +using ServiceLayer.Mesh.FileTypes.NbssAppointmentEvents.Validation; + +namespace ServiceLayer.Mesh.FileTypes.NbssAppointmentEvents; + +public static class ServiceCollectionExtensions +{ + public static IServiceCollection ConfigureNbssAppointmentEvents(this IServiceCollection services) + { + services.AddTransient(); + services.AddTransient(); + services.AddTransient(); + services.AddSingleton(); + services.RegisterValidators(); + + return services; + } + + private static IServiceCollection RegisterValidators(this IServiceCollection services) + { + foreach (var recordValidator in ValidatorRegistry.GetAllRecordValidators()) + { + services.AddSingleton(_ => recordValidator); + } + + foreach (var fileValidator in ValidatorRegistry.GetAllFileValidators()) + { + services.AddSingleton(_ => fileValidator); + } + + return services; + } +} diff --git a/src/ServiceLayer.Mesh/Functions/FileDiscoveryFunction.cs b/src/ServiceLayer.Mesh/Functions/FileDiscoveryFunction.cs index ffddeac..573b06b 100644 --- a/src/ServiceLayer.Mesh/Functions/FileDiscoveryFunction.cs +++ b/src/ServiceLayer.Mesh/Functions/FileDiscoveryFunction.cs @@ -19,10 +19,11 @@ public class FileDiscoveryFunction( [Function("FileDiscoveryFunction")] public async Task Run([TimerTrigger("%FileDiscoveryTimerExpression%")] TimerInfo myTimer) { - logger.LogInformation("{functionName} started at: {time}", nameof(FileDiscoveryFunction), DateTime.UtcNow); + logger.LogInformation("{FunctionName} started.", nameof(FileDiscoveryFunction)); var response = await meshInboxService.GetMessagesAsync(configuration.NbssMeshMailboxId); + // TODO - check if response.IsSuccessful before proceeding to dereference the Response.Messages foreach (var messageId in response.Response.Messages) { await using var transaction = await serviceLayerDbContext.Database.BeginTransactionAsync(); diff --git a/src/ServiceLayer.Mesh/Functions/FileExtractFunction.cs b/src/ServiceLayer.Mesh/Functions/FileExtractFunction.cs index e0d7ed4..5fbcea7 100644 --- a/src/ServiceLayer.Mesh/Functions/FileExtractFunction.cs +++ b/src/ServiceLayer.Mesh/Functions/FileExtractFunction.cs @@ -5,7 +5,7 @@ using NHS.MESH.Client.Contracts.Services; using ServiceLayer.Data; using ServiceLayer.Data.Models; -using ServiceLayer.Mesh.Configuration; +using ServiceLayer.Mesh.Extensions; using ServiceLayer.Mesh.Messaging; using ServiceLayer.Mesh.Storage; @@ -13,7 +13,6 @@ namespace ServiceLayer.Mesh.Functions; public class FileExtractFunction( ILogger logger, - IFileExtractFunctionConfiguration configuration, IMeshInboxService meshInboxService, ServiceLayerDbContext serviceLayerDbContext, IFileTransformQueueClient fileTransformQueueClient, @@ -23,7 +22,7 @@ public class FileExtractFunction( [Function("FileExtractFunction")] public async Task Run([QueueTrigger("%FileExtractQueueName%")] FileExtractQueueMessage message) { - logger.LogInformation("{functionName} started.", nameof(FileExtractFunction)); + logger.LogInformation("{FunctionName} started. Processing fileId: {FileId}", nameof(FileExtractFunction), message.FileId); await using var transaction = await serviceLayerDbContext.Database.BeginTransactionAsync(); @@ -38,7 +37,7 @@ public async Task Run([QueueTrigger("%FileExtractQueueName%")] FileExtractQueueM try { - await ProcessFileExtraction(file, message); + await ProcessFileExtraction(file); } catch (Exception ex) { @@ -84,20 +83,20 @@ private async Task UpdateFileStatusForExtraction(MeshFile file) await serviceLayerDbContext.SaveChangesAsync(); } - private async Task ProcessFileExtraction(MeshFile file, FileExtractQueueMessage message) + private async Task ProcessFileExtraction(MeshFile file) { - var meshResponse = await meshInboxService.GetMessageByIdAsync(configuration.NbssMeshMailboxId, file.FileId); + var meshResponse = await meshInboxService.GetMessageByIdAsync(file.MailboxId, file.FileId); if (!meshResponse.IsSuccessful) { - throw new InvalidOperationException($"Mesh extraction failed: {meshResponse.Error}"); + throw new InvalidOperationException($"Mesh extraction failed: [ {meshResponse.Error.ToFormattedString()} ]"); } var blobPath = await meshFileBlobStore.UploadAsync(file, meshResponse.Response.FileAttachment.Content); - var meshAcknowledgementResponse = await meshInboxService.AcknowledgeMessageByIdAsync(configuration.NbssMeshMailboxId, message.FileId); + var meshAcknowledgementResponse = await meshInboxService.AcknowledgeMessageByIdAsync(file.MailboxId, file.FileId); if (!meshAcknowledgementResponse.IsSuccessful) { - logger.LogWarning("Mesh acknowledgement failed: {error}.\nThis is not a fatal error so processing will continue.", meshAcknowledgementResponse.Error); + logger.LogWarning("Mesh acknowledgement failed: [ {ToFormattedString} ].\nThis is not a fatal error so processing will continue.", meshAcknowledgementResponse.Error.ToFormattedString()); } file.BlobPath = blobPath; diff --git a/src/ServiceLayer.Mesh/Functions/FileRetryFunction.cs b/src/ServiceLayer.Mesh/Functions/FileRetryFunction.cs index 439bd4a..36ca269 100644 --- a/src/ServiceLayer.Mesh/Functions/FileRetryFunction.cs +++ b/src/ServiceLayer.Mesh/Functions/FileRetryFunction.cs @@ -18,7 +18,7 @@ public class FileRetryFunction( [Function("FileRetryFunction")] public async Task Run([TimerTrigger("%FileRetryTimerExpression%")] TimerInfo myTimer) { - logger.LogInformation("FileRetryFunction started"); + logger.LogInformation("{functionName} started.", nameof(FileRetryFunction)); var staleDateTimeUtc = DateTime.UtcNow.AddHours(-configuration.StaleHours); diff --git a/src/ServiceLayer.Mesh/Functions/FileTransformFunction.cs b/src/ServiceLayer.Mesh/Functions/FileTransformFunction.cs index cc1a955..cd61816 100644 --- a/src/ServiceLayer.Mesh/Functions/FileTransformFunction.cs +++ b/src/ServiceLayer.Mesh/Functions/FileTransformFunction.cs @@ -1,3 +1,5 @@ +using System.Text.Json; +using System.Text.Json.Serialization; using Google.Protobuf.WellKnownTypes; using Microsoft.Azure.Functions.Worker; using Microsoft.EntityFrameworkCore; @@ -5,7 +7,6 @@ using ServiceLayer.Data; using ServiceLayer.Data.Models; using ServiceLayer.Mesh.Configuration; -using ServiceLayer.Mesh.FileTypes.NbssAppointmentEvents; using ServiceLayer.Mesh.Messaging; using ServiceLayer.Mesh.Storage; @@ -13,25 +14,30 @@ namespace ServiceLayer.Mesh.Functions; public class FileTransformFunction( ILogger logger, + IFileTransformFunctionConfiguration configuration, ServiceLayerDbContext serviceLayerDbContext, + IFileTransformQueueClient fileTransformQueueClient, IMeshFilesBlobStore meshFileBlobStore, - IFileTransformFunctionConfiguration configuration, - IFileParser fileParser) + IEnumerable fileTransformers) { + private static readonly JsonSerializerOptions ValidationErrorJsonOptions = new() + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, + WriteIndented = true, + Converters = { new JsonStringEnumConverter(JsonNamingPolicy.CamelCase) } + }; + [Function("FileTransformFunction")] public async Task Run([QueueTrigger("%FileTransformQueueName%")] FileTransformQueueMessage message) { - await using var transaction = await serviceLayerDbContext.Database.BeginTransactionAsync(); - - var file = await serviceLayerDbContext.MeshFiles.FirstOrDefaultAsync(f => f.FileId == message.FileId); + logger.LogInformation("{FunctionName} started. Processing fileId: {FileId}", nameof(FileTransformFunction), + message.FileId); - if (file == null) - { - logger.LogWarning("File with id: {FileId} not found in MeshFiles table.", message.FileId); - return; - } + await using var transaction = await serviceLayerDbContext.Database.BeginTransactionAsync(); - if (!IsFileSuitableForTransformation(file)) + var file = await GetFileAsync(message.FileId); + if (file == null || !IsFileSuitableForTransformation(file)) { return; } @@ -39,20 +45,27 @@ public async Task Run([QueueTrigger("%FileTransformQueueName%")] FileTransformQu await UpdateFileStatusForTransformation(file); await transaction.CommitAsync(); - var fileContent = await meshFileBlobStore.DownloadAsync(file); - - var parsedfile = fileParser.Parse(fileContent); - - // TODO - take dependency on IEnumerable. - // After initial common checks against database, find the appropriate implementation of IFileTransformer - // to handle the functionality that differs between file types. + try + { + await ProcessFileTransformation(file); + } + catch (Exception ex) + { + await HandleTransformationError(file, message, ex); + } } - private async Task UpdateFileStatusForTransformation(MeshFile file) + private async Task GetFileAsync(string fileId) { - file.Status = MeshFileStatus.Transforming; - file.LastUpdatedUtc = DateTime.UtcNow; - await serviceLayerDbContext.SaveChangesAsync(); + var file = await serviceLayerDbContext.MeshFiles + .FirstOrDefaultAsync(f => f.FileId == fileId); + + if (file == null) + { + logger.LogWarning("File with id: {FileId} not found in MeshFiles table.", fileId); + } + + return file; } private bool IsFileSuitableForTransformation(MeshFile file) @@ -70,6 +83,55 @@ private bool IsFileSuitableForTransformation(MeshFile file) file.LastUpdatedUtc.ToTimestamp()); return false; } + return true; } + + private async Task UpdateFileStatusForTransformation(MeshFile file) + { + file.Status = MeshFileStatus.Transforming; + file.LastUpdatedUtc = DateTime.UtcNow; + await serviceLayerDbContext.SaveChangesAsync(); + } + + private async Task ProcessFileTransformation(MeshFile file) + { + var transformer = GetTransformerFor(file.FileType); + var fileContent = await meshFileBlobStore.DownloadAsync(file); + + var validationErrors = await transformer.TransformFileAsync(fileContent, file); + + if (validationErrors.Any()) + { + file.ValidationErrors = JsonSerializer.Serialize(validationErrors, ValidationErrorJsonOptions); + throw new InvalidOperationException("Validation errors encountered"); + } + + file.Status = MeshFileStatus.Transformed; + file.LastUpdatedUtc = DateTime.UtcNow; + await serviceLayerDbContext.SaveChangesAsync(); + } + + private IFileTransformer GetTransformerFor(MeshFileType type) + { + try + { + return fileTransformers.SingleOrDefault(t => t.CanHandle(type)) + ?? throw new InvalidOperationException($"No transformer registered to handle file type: {type}"); + } + catch (InvalidOperationException ex) when (ex.Message.Contains("more than one")) + { + throw new InvalidOperationException( + $"Multiple transformers found for file type: {type}. This is likely a configuration error.", ex); + } + } + + private async Task HandleTransformationError(MeshFile file, FileTransformQueueMessage message, Exception ex) + { + logger.LogError(ex, "An exception occurred during file transformation for fileId: {FileId}", message.FileId); + file.Status = MeshFileStatus.FailedTransform; + file.LastUpdatedUtc = DateTime.UtcNow; + await serviceLayerDbContext.SaveChangesAsync(); + await fileTransformQueueClient.SendToPoisonQueueAsync(message); + } } diff --git a/src/ServiceLayer.Mesh/Functions/MeshHandshakeFunction.cs b/src/ServiceLayer.Mesh/Functions/MeshHandshakeFunction.cs index f6978e2..24e8b54 100644 --- a/src/ServiceLayer.Mesh/Functions/MeshHandshakeFunction.cs +++ b/src/ServiceLayer.Mesh/Functions/MeshHandshakeFunction.cs @@ -1,8 +1,8 @@ -using Azure; using Microsoft.Azure.Functions.Worker; using Microsoft.Extensions.Logging; using NHS.MESH.Client.Contracts.Services; using ServiceLayer.Mesh.Configuration; +using ServiceLayer.Mesh.Extensions; namespace ServiceLayer.Mesh.Functions { @@ -14,7 +14,7 @@ public class MeshHandshakeFunction( [Function("MeshHandshakeFunction")] public async Task Run([TimerTrigger("%MeshHandshakeTimerExpression%")] TimerInfo myTimer) { - logger.LogInformation("{FunctionName} started", nameof(MeshHandshakeFunction)); + logger.LogInformation("{FunctionName} started.", nameof(MeshHandshakeFunction)); try { @@ -22,18 +22,17 @@ public async Task Run([TimerTrigger("%MeshHandshakeTimerExpression%")] TimerInfo if (response.IsSuccessful) { - logger.LogInformation("Mesh handshake completed successfully for mailbox {MailboxId}. Status: {Status}", - response.Response.MailboxId, response.IsSuccessful); + logger.LogInformation( + "Mesh handshake completed successfully for mailbox {ConfigurationNbssMeshMailboxId}.", configuration.NbssMeshMailboxId); } else { - logger.LogWarning("Mesh handshake failed for mailbox {MailboxId}. Error: {Error}", - configuration.NbssMeshMailboxId, response.Error); + logger.LogWarning("Mesh handshake failed for mailbox {ConfigurationNbssMeshMailboxId}: [ {ToFormattedString} ]", configuration.NbssMeshMailboxId, response.Error.ToFormattedString()); } } catch (Exception ex) { - logger.LogError(ex, "An error occurred during mesh handshake for mailbox {MailboxId}.", configuration.NbssMeshMailboxId); + logger.LogWarning(ex, "An error occurred during mesh handshake for mailbox {MailboxId}.", configuration.NbssMeshMailboxId); } } } diff --git a/src/ServiceLayer.Mesh/IFileTransformer.cs b/src/ServiceLayer.Mesh/IFileTransformer.cs index fdcba73..81a11cd 100644 --- a/src/ServiceLayer.Mesh/IFileTransformer.cs +++ b/src/ServiceLayer.Mesh/IFileTransformer.cs @@ -4,6 +4,6 @@ namespace ServiceLayer.Mesh; public interface IFileTransformer { - MeshFileType HandlesFileType { get; } + bool CanHandle(MeshFileType fileType); Task> TransformFileAsync(Stream stream, MeshFile metaData); } diff --git a/src/ServiceLayer.Mesh/Program.cs b/src/ServiceLayer.Mesh/Program.cs index 110688d..3c2c604 100644 --- a/src/ServiceLayer.Mesh/Program.cs +++ b/src/ServiceLayer.Mesh/Program.cs @@ -58,7 +58,6 @@ services.AddSingleton(); services.AddSingleton(); - services.AddSingleton(); services.AddSingleton(provider => { @@ -70,12 +69,13 @@ services.AddSingleton(); services.AddTransient(); - services.AddTransient(); services.AddTransient(); services.AddTransient(); services.AddTransient(); services.AddTransient(); services.AddTransient(); + + services.ConfigureNbssAppointmentEvents(); }); diff --git a/src/ServiceLayer.sln b/src/ServiceLayer.sln index e37cf51..14cb8a0 100644 --- a/src/ServiceLayer.sln +++ b/src/ServiceLayer.sln @@ -19,6 +19,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ServiceLayer.Common", "Serv EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ServiceLayer.Common.Tests", "..\tests\ServiceLayer.Common.Tests\ServiceLayer.Common.Tests.csproj", "{9642EC3A-9BC3-4557-ACD1-807FE764D5F6}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ServiceLayer.TestUtilities", "..\tests\ServiceLayer.TestUtilities\ServiceLayer.TestUtilities.csproj", "{43AE0301-63CA-4254-9580-4FB07C9D1920}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -113,6 +115,18 @@ Global {9642EC3A-9BC3-4557-ACD1-807FE764D5F6}.Release|x64.Build.0 = Release|Any CPU {9642EC3A-9BC3-4557-ACD1-807FE764D5F6}.Release|x86.ActiveCfg = Release|Any CPU {9642EC3A-9BC3-4557-ACD1-807FE764D5F6}.Release|x86.Build.0 = Release|Any CPU + {43AE0301-63CA-4254-9580-4FB07C9D1920}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {43AE0301-63CA-4254-9580-4FB07C9D1920}.Debug|Any CPU.Build.0 = Debug|Any CPU + {43AE0301-63CA-4254-9580-4FB07C9D1920}.Debug|x64.ActiveCfg = Debug|Any CPU + {43AE0301-63CA-4254-9580-4FB07C9D1920}.Debug|x64.Build.0 = Debug|Any CPU + {43AE0301-63CA-4254-9580-4FB07C9D1920}.Debug|x86.ActiveCfg = Debug|Any CPU + {43AE0301-63CA-4254-9580-4FB07C9D1920}.Debug|x86.Build.0 = Debug|Any CPU + {43AE0301-63CA-4254-9580-4FB07C9D1920}.Release|Any CPU.ActiveCfg = Release|Any CPU + {43AE0301-63CA-4254-9580-4FB07C9D1920}.Release|Any CPU.Build.0 = Release|Any CPU + {43AE0301-63CA-4254-9580-4FB07C9D1920}.Release|x64.ActiveCfg = Release|Any CPU + {43AE0301-63CA-4254-9580-4FB07C9D1920}.Release|x64.Build.0 = Release|Any CPU + {43AE0301-63CA-4254-9580-4FB07C9D1920}.Release|x86.ActiveCfg = Release|Any CPU + {43AE0301-63CA-4254-9580-4FB07C9D1920}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -121,6 +135,7 @@ Global {BA052DAE-6FD1-483A-A0AF-DCBCF9E38C72} = {0AB3BF05-4346-4AA6-1389-037BE0695223} {E5EF5B92-52DA-4EF3-956B-8AEE3D333428} = {0AB3BF05-4346-4AA6-1389-037BE0695223} {9642EC3A-9BC3-4557-ACD1-807FE764D5F6} = {0AB3BF05-4346-4AA6-1389-037BE0695223} + {43AE0301-63CA-4254-9580-4FB07C9D1920} = {0AB3BF05-4346-4AA6-1389-037BE0695223} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {EEE06B13-019F-4618-A6EB-FD834B6EA7D7} diff --git a/tests/ServiceLayer.Common.Tests/ServiceLayer.Common.Tests.csproj b/tests/ServiceLayer.Common.Tests/ServiceLayer.Common.Tests.csproj index 2394e71..7bb3d6b 100644 --- a/tests/ServiceLayer.Common.Tests/ServiceLayer.Common.Tests.csproj +++ b/tests/ServiceLayer.Common.Tests/ServiceLayer.Common.Tests.csproj @@ -13,6 +13,7 @@ runtime; build; native; contentfiles; analyzers; buildtransitive + diff --git a/tests/ServiceLayer.Mesh.Tests/Extensions/ApiErrorResponseExtensionsTests.cs b/tests/ServiceLayer.Mesh.Tests/Extensions/ApiErrorResponseExtensionsTests.cs new file mode 100644 index 0000000..e3985f0 --- /dev/null +++ b/tests/ServiceLayer.Mesh.Tests/Extensions/ApiErrorResponseExtensionsTests.cs @@ -0,0 +1,89 @@ +using ServiceLayer.Mesh.Extensions; + +namespace ServiceLayer.Mesh.Tests.Extensions; + +using Xunit; +using NHS.MESH.Client.Models; + +public class ApiErrorResponseExtensionsTests +{ + [Fact] + public void ToFormattedString_ReturnsFormattedString_WhenAllFieldsArePresent() + { + var error = new APIErrorResponse + { + ErrorEvent = "SomeEvent", + ErrorCode = "123", + ErrorDescription = "Something went awry" + }; + + var result = error.ToFormattedString(); + + Assert.Equal("ErrorEvent: SomeEvent, ErrorCode: 123, ErrorDescription: Something went awry", result); + } + + [Fact] + public void ToFormattedString_HandlesNullErrorEvent() + { + var error = new APIErrorResponse + { + ErrorEvent = null, + ErrorCode = "123", + ErrorDescription = "Something went awry" + }; + + var result = error.ToFormattedString(); + + Assert.Equal("ErrorEvent: N/A, ErrorCode: 123, ErrorDescription: Something went awry", result); + } + + [Fact] + public void ToFormattedString_HandlesNullErrorCode() + { + var error = new APIErrorResponse + { + ErrorEvent = "SomeEvent", + ErrorCode = null, + ErrorDescription = "Something went awry" + }; + + var result = error.ToFormattedString(); + + Assert.Equal("ErrorEvent: SomeEvent, ErrorCode: N/A, ErrorDescription: Something went awry", result); + } + + [Fact] + public void ToFormattedString_HandlesNullErrorDescription() + { + var error = new APIErrorResponse + { + ErrorEvent = "SomeEvent", + ErrorCode = "123", + ErrorDescription = null! + }; + + var result = error.ToFormattedString(); + + Assert.Equal("ErrorEvent: SomeEvent, ErrorCode: 123, ErrorDescription: ", result); + } + + [Fact] + public void ToFormattedString_HandlesAllFieldsNull() + { + var error = new APIErrorResponse(); + + var result = error.ToFormattedString(); + + Assert.Equal("ErrorEvent: N/A, ErrorCode: N/A, ErrorDescription: ", result); + } + + [Fact] + public void ToFormattedString_ReturnsFallback_WhenErrorIsNull() + { + APIErrorResponse? error = null; + + var result = error.ToFormattedString(); + + Assert.Equal("Unknown error", result); + } +} diff --git a/tests/ServiceLayer.Mesh.Tests/Functions/FileDiscoveryFunctionTests.cs b/tests/ServiceLayer.Mesh.Tests/Functions/FileDiscoveryFunctionTests.cs index cca2557..a80995a 100644 --- a/tests/ServiceLayer.Mesh.Tests/Functions/FileDiscoveryFunctionTests.cs +++ b/tests/ServiceLayer.Mesh.Tests/Functions/FileDiscoveryFunctionTests.cs @@ -1,9 +1,6 @@ -using Microsoft.EntityFrameworkCore; -using Microsoft.Extensions.Logging; -using Moq; +using Moq; using NHS.MESH.Client.Contracts.Services; using NHS.MESH.Client.Models; -using ServiceLayer.Data; using ServiceLayer.Data.Models; using ServiceLayer.Mesh.Configuration; using ServiceLayer.Mesh.Functions; @@ -11,36 +8,25 @@ namespace ServiceLayer.Mesh.Tests.Functions; -public class FileDiscoveryFunctionTests +public class FileDiscoveryFunctionTests : FunctionTestBase { - private readonly Mock> _loggerMock; private readonly Mock _meshInboxServiceMock; - private readonly ServiceLayerDbContext _dbContext; private readonly Mock _queueClientMock; private readonly FileDiscoveryFunction _function; public FileDiscoveryFunctionTests() { - _loggerMock = new Mock>(); _meshInboxServiceMock = new Mock(); _queueClientMock = new Mock(); - var options = new DbContextOptionsBuilder() - .UseInMemoryDatabase(databaseName: Guid.NewGuid().ToString()) - .ConfigureWarnings(warnings => - warnings.Ignore(Microsoft.EntityFrameworkCore.Diagnostics.InMemoryEventId.TransactionIgnoredWarning)) - .Options; - - _dbContext = new ServiceLayerDbContext(options); - var functionConfiguration = new Mock(); functionConfiguration.Setup(c => c.NbssMeshMailboxId).Returns("test-mailbox"); _function = new FileDiscoveryFunction( - _loggerMock.Object, + LoggerMock.Object, functionConfiguration.Object, _meshInboxServiceMock.Object, - _dbContext, + DbContext, _queueClientMock.Object ); } @@ -54,16 +40,14 @@ public async Task Run_AddsNewMessageToDbAndQueue() _meshInboxServiceMock.Setup(s => s.GetMessagesAsync("test-mailbox")) .ReturnsAsync(new MeshResponse { - Response = new CheckInboxResponse { Messages = new[] { testMessageId } } + Response = new CheckInboxResponse { Messages = [testMessageId] } }); // Act await _function.Run(null); // Assert - var meshFile = _dbContext.MeshFiles.FirstOrDefault(f => f.FileId == testMessageId); - Assert.NotNull(meshFile); - Assert.Equal(MeshFileStatus.Discovered, meshFile.Status); + var meshFile = AssertFileUpdated(testMessageId, MeshFileStatus.Discovered); Assert.Equal("test-mailbox", meshFile.MailboxId); // TODO - replace the It.IsAny with a more specific matcher, or use a callback @@ -74,29 +58,19 @@ public async Task Run_AddsNewMessageToDbAndQueue() public async Task Run_DoesNotAddDuplicateMessageOrQueueIt() { // Arrange - var duplicateMessageId = "existing-message"; - _dbContext.MeshFiles.Add(new MeshFile - { - FileId = duplicateMessageId, - FileType = MeshFileType.NbssAppointmentEvents, - MailboxId = "test-mailbox", - Status = MeshFileStatus.Discovered, - FirstSeenUtc = DateTime.UtcNow, - LastUpdatedUtc = DateTime.UtcNow - }); - await _dbContext.SaveChangesAsync(); + var existingFile = SaveMeshFile(MeshFileStatus.Discovered); _meshInboxServiceMock.Setup(s => s.GetMessagesAsync("test-mailbox")) .ReturnsAsync(new MeshResponse { - Response = new CheckInboxResponse { Messages = new[] { duplicateMessageId } } + Response = new CheckInboxResponse { Messages = [existingFile.FileId] } }); // Act await _function.Run(null); // Assert - var count = _dbContext.MeshFiles.Count(f => f.FileId == duplicateMessageId); + var count = DbContext.MeshFiles.Count(f => f.FileId == existingFile.FileId); Assert.Equal(1, count); _queueClientMock.Verify(q => q.EnqueueFileExtractAsync(It.IsAny()), Times.Never); @@ -116,7 +90,7 @@ public async Task Run_NoMessagesInInbox_DoesNothing() await _function.Run(null); // Assert - Assert.Empty(_dbContext.MeshFiles); + Assert.Empty(DbContext.MeshFiles); _queueClientMock.Verify(q => q.EnqueueFileExtractAsync(It.IsAny()), Times.Never); } @@ -138,10 +112,8 @@ public async Task Run_MultipleMessagesInInbox_AllAreProcessed() // Assert foreach (var id in messageIds) { - var meshFile = _dbContext.MeshFiles.FirstOrDefault(f => f.FileId == id); - Assert.NotNull(meshFile); - Assert.Equal(MeshFileStatus.Discovered, meshFile.Status); - Assert.Equal("test-mailbox", meshFile.MailboxId); + var savedFile = AssertFileUpdated(id, MeshFileStatus.Discovered); + Assert.Equal("test-mailbox", savedFile.MailboxId); } // TODO - replace the It.IsAny with more specific matcher, or use a callback to capture the arguments and check the file IDs diff --git a/tests/ServiceLayer.Mesh.Tests/Functions/FileExtractFunctionTests.cs b/tests/ServiceLayer.Mesh.Tests/Functions/FileExtractFunctionTests.cs index 9b18d12..c19697e 100644 --- a/tests/ServiceLayer.Mesh.Tests/Functions/FileExtractFunctionTests.cs +++ b/tests/ServiceLayer.Mesh.Tests/Functions/FileExtractFunctionTests.cs @@ -1,52 +1,35 @@ using Google.Protobuf.WellKnownTypes; -using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Logging; using Moq; using NHS.MESH.Client.Contracts.Services; using NHS.MESH.Client.Models; -using ServiceLayer.Data; using ServiceLayer.Data.Models; -using ServiceLayer.Mesh.Configuration; using ServiceLayer.Mesh.Functions; using ServiceLayer.Mesh.Messaging; using ServiceLayer.Mesh.Storage; +using ServiceLayer.TestUtilities; namespace ServiceLayer.Mesh.Tests.Functions; -public class FileExtractFunctionTests +public class FileExtractFunctionTests : FunctionTestBase { - private readonly Mock> _loggerMock; private readonly Mock _meshInboxServiceMock; private readonly Mock _fileTransformQueueClientMock; private readonly Mock _fileExtractQueueClientMock; private readonly Mock _blobStoreMock; - private readonly ServiceLayerDbContext _dbContext; private readonly FileExtractFunction _function; public FileExtractFunctionTests() { - _loggerMock = new Mock>(); _meshInboxServiceMock = new Mock(); _fileExtractQueueClientMock = new Mock(); _fileTransformQueueClientMock = new Mock(); _blobStoreMock = new Mock(); - var options = new DbContextOptionsBuilder() - .UseInMemoryDatabase(Guid.NewGuid().ToString()) - .ConfigureWarnings(warnings => - warnings.Ignore(Microsoft.EntityFrameworkCore.Diagnostics.InMemoryEventId.TransactionIgnoredWarning)) - .Options; - - _dbContext = new ServiceLayerDbContext(options); - - var functionConfiguration = new Mock(); - functionConfiguration.Setup(c => c.NbssMeshMailboxId).Returns("test-mailbox"); - _function = new FileExtractFunction( - _loggerMock.Object, - functionConfiguration.Object, + LoggerMock.Object, _meshInboxServiceMock.Object, - _dbContext, + DbContext, _fileTransformQueueClientMock.Object, _fileExtractQueueClientMock.Object, _blobStoreMock.Object @@ -63,87 +46,35 @@ public async Task Run_FileNotFound_ExitsSilently() await _function.Run(message); // Assert - _loggerMock.Verify( - x => x.Log( - LogLevel.Warning, - It.IsAny(), - It.Is((v, t) => v.ToString() == $"File with id: {message.FileId} not found in MeshFiles table."), - null, - It.IsAny>() - ), Times.Once); - - Assert.Equal(0, _dbContext.MeshFiles.Count()); - _meshInboxServiceMock.Verify(x => x.GetHeadMessageByIdAsync(It.IsAny(), It.IsAny()), Times.Never); - _blobStoreMock.Verify(x => x.UploadAsync(It.IsAny(), It.IsAny()), Times.Never); - _fileTransformQueueClientMock.Verify(x => x.EnqueueFileTransformAsync(It.IsAny()), Times.Never); - _fileTransformQueueClientMock.Verify(x => x.SendToPoisonQueueAsync(It.IsAny()), Times.Never); - } - - [Fact] - public async Task Run_FileStatusInvalid_ExitsSilently() - { - // Arrange - var file = new MeshFile - { - FileType = MeshFileType.NbssAppointmentEvents, - MailboxId = "test-mailbox", - FileId = "file-1", - Status = MeshFileStatus.Transforming, // Not eligible - LastUpdatedUtc = DateTime.UtcNow - }; - _dbContext.MeshFiles.Add(file); - await _dbContext.SaveChangesAsync(); - - var message = new FileExtractQueueMessage { FileId = "file-1" }; - - // Act - await _function.Run(message); - - // Assert - _loggerMock.Verify( - x => x.Log( - LogLevel.Warning, - It.IsAny(), - It.Is((v, t) => v.ToString() == $"File with id: {message.FileId} found in MeshFiles table but is not suitable for extraction. Status: {file.Status}, LastUpdatedUtc: {file.LastUpdatedUtc.ToTimestamp()}."), - null, - It.IsAny>() - ), Times.Once); + LoggerMock.VerifyLogger(LogLevel.Warning, + $"File with id: {message.FileId} not found in MeshFiles table."); + Assert.Equal(0, DbContext.MeshFiles.Count()); _meshInboxServiceMock.Verify(x => x.GetHeadMessageByIdAsync(It.IsAny(), It.IsAny()), Times.Never); _blobStoreMock.Verify(x => x.UploadAsync(It.IsAny(), It.IsAny()), Times.Never); _fileTransformQueueClientMock.Verify(x => x.EnqueueFileTransformAsync(It.IsAny()), Times.Never); _fileTransformQueueClientMock.Verify(x => x.SendToPoisonQueueAsync(It.IsAny()), Times.Never); } - [Fact] - public async Task Run_FileStatusExtractingButNotTimedOut_ExitsSilently() + [Theory] + [InlineData(MeshFileStatus.Extracted)] + [InlineData(MeshFileStatus.Extracting)] + [InlineData(MeshFileStatus.Transforming)] + [InlineData(MeshFileStatus.Transformed)] + [InlineData(MeshFileStatus.FailedExtract)] + [InlineData(MeshFileStatus.FailedTransform)] + public async Task Run_FileStatusInvalid_ExitsSilently(MeshFileStatus invalidStatus) { // Arrange - var file = new MeshFile - { - FileType = MeshFileType.NbssAppointmentEvents, - MailboxId = "test-mailbox", - FileId = "file-2", - Status = MeshFileStatus.Extracting, - LastUpdatedUtc = DateTime.UtcNow // Not timed out - }; - _dbContext.MeshFiles.Add(file); - await _dbContext.SaveChangesAsync(); - - var message = new FileExtractQueueMessage { FileId = "file-2" }; + var file = SaveMeshFile(invalidStatus); + var message = new FileExtractQueueMessage { FileId = file.FileId }; // Act await _function.Run(message); // Assert - _loggerMock.Verify( - x => x.Log( - LogLevel.Warning, - It.IsAny(), - It.Is((v, t) => v.ToString() == $"File with id: {message.FileId} found in MeshFiles table but is not suitable for extraction. Status: {file.Status}, LastUpdatedUtc: {file.LastUpdatedUtc.ToTimestamp()}."), - null, - It.IsAny>() - ), Times.Once); + LoggerMock.VerifyLogger(LogLevel.Warning, + $"File with id: {message.FileId} found in MeshFiles table but is not suitable for extraction. Status: {file.Status}, LastUpdatedUtc: {file.LastUpdatedUtc.ToTimestamp()}."); _meshInboxServiceMock.Verify(x => x.GetHeadMessageByIdAsync(It.IsAny(), It.IsAny()), Times.Never); _blobStoreMock.Verify(x => x.UploadAsync(It.IsAny(), It.IsAny()), Times.Never); @@ -151,21 +82,13 @@ public async Task Run_FileStatusExtractingButNotTimedOut_ExitsSilently() _fileTransformQueueClientMock.Verify(x => x.SendToPoisonQueueAsync(It.IsAny()), Times.Never); } - [Fact] - public async Task Run_FileValid_FileUploadedToBlobAndAcknowledgedAndEnqueued() + [Theory] + [InlineData(MeshFileStatus.Discovered, 0)] + [InlineData(MeshFileStatus.Extracting, 13)] + public async Task Run_FileValid_FileUploadedToBlobAndAcknowledgedAndEnqueued(MeshFileStatus validStatus, int hoursOld) { // Arrange - var originalLastUpdatedUtc = DateTime.UtcNow.AddHours(-1); - var file = new MeshFile - { - FileType = MeshFileType.NbssAppointmentEvents, - MailboxId = "test-mailbox", - FileId = "file-3", - Status = MeshFileStatus.Discovered, - LastUpdatedUtc = originalLastUpdatedUtc - }; - _dbContext.MeshFiles.Add(file); - await _dbContext.SaveChangesAsync(); + var file = SaveMeshFile(validStatus, hoursOld); var content = new byte[] { 1, 2, 3 }; const string blobPath = "directory/fileName"; @@ -195,80 +118,58 @@ public async Task Run_FileValid_FileUploadedToBlobAndAcknowledgedAndEnqueued() _blobStoreMock.Verify(b => b.UploadAsync(It.Is(f => f.FileId == file.FileId), content), Times.Once); _meshInboxServiceMock.Verify(m => m.AcknowledgeMessageByIdAsync(file.MailboxId, file.FileId), Times.Once); _fileTransformQueueClientMock.Verify(q => q.EnqueueFileTransformAsync(file), Times.Once); - var updatedFile = _dbContext.MeshFiles.First(); + + var updatedFile = AssertFileUpdated(file.FileId, MeshFileStatus.Extracted); Assert.Equal(blobPath, updatedFile.BlobPath); - Assert.Equal(MeshFileStatus.Extracted, updatedFile.Status); - Assert.True(updatedFile.LastUpdatedUtc > originalLastUpdatedUtc); } [Fact] public async Task Run_GetMessageFails_ErrorLoggedAndFileSentToPoisonQueue() { // Arrange - var fileId = "file-4"; - var originalLastUpdatedUtc = DateTime.UtcNow.AddHours(-1); - var file = new MeshFile - { - FileType = MeshFileType.NbssAppointmentEvents, - MailboxId = "test-mailbox", - FileId = fileId, - Status = MeshFileStatus.Discovered, - LastUpdatedUtc = originalLastUpdatedUtc - }; - _dbContext.MeshFiles.Add(file); - await _dbContext.SaveChangesAsync(); - - _meshInboxServiceMock.Setup(s => s.GetMessageByIdAsync(file.MailboxId, fileId)) + var file = SaveMeshFile(MeshFileStatus.Discovered); + + _meshInboxServiceMock.Setup(s => s.GetMessageByIdAsync(file.MailboxId, file.FileId)) .ReturnsAsync(new MeshResponse { - IsSuccessful = false + IsSuccessful = false, + Error = new APIErrorResponse + { + ErrorCode = "code", + ErrorDescription = "description", + ErrorEvent = "event" + } }); - var message = new FileExtractQueueMessage { FileId = fileId }; + var message = new FileExtractQueueMessage { FileId = file.FileId }; // Act await _function.Run(message); // Assert - _loggerMock.Verify( - x => x.Log( - LogLevel.Error, - It.IsAny(), - It.Is((v, t) => v.ToString() == $"An exception occurred during file extraction for fileId: {fileId}"), - It.Is(e => e.Message.StartsWith("Mesh extraction failed: ")), - It.IsAny>() - ), Times.Once); + LoggerMock.VerifyLogger(LogLevel.Error, + $"An exception occurred during file extraction for fileId: {file.FileId}", + e => e is InvalidOperationException && e.Message.StartsWith("Mesh extraction failed: [ ErrorEvent: event, ErrorCode: code, ErrorDescription: description ]") + ); + _blobStoreMock.Verify(b => b.UploadAsync(It.IsAny(), It.IsAny()), Times.Never); _meshInboxServiceMock.Verify(m => m.AcknowledgeMessageByIdAsync(file.MailboxId, file.FileId), Times.Never); _fileTransformQueueClientMock.Verify(q => q.EnqueueFileTransformAsync(It.IsAny()), Times.Never); _fileExtractQueueClientMock.Verify(q => q.SendToPoisonQueueAsync(message), Times.Once); - var updatedFile = _dbContext.MeshFiles.First(); + var updatedFile = AssertFileUpdated(file.FileId, MeshFileStatus.FailedExtract); Assert.Null(updatedFile.BlobPath); - Assert.Equal(MeshFileStatus.FailedExtract, updatedFile.Status); - Assert.True(updatedFile.LastUpdatedUtc > originalLastUpdatedUtc); } [Fact] public async Task Run_AcknowledgeMessageFails_WarningLoggedAndProcessingContinuesAsNormal() { // Arrange - var fileId = "file-4"; - var originalLastUpdatedUtc = DateTime.UtcNow.AddHours(-1); - var file = new MeshFile - { - FileType = MeshFileType.NbssAppointmentEvents, - MailboxId = "test-mailbox", - FileId = fileId, - Status = MeshFileStatus.Discovered, - LastUpdatedUtc = originalLastUpdatedUtc - }; - _dbContext.MeshFiles.Add(file); - await _dbContext.SaveChangesAsync(); + var file = SaveMeshFile(MeshFileStatus.Discovered); var content = new byte[] { 1, 2, 3 }; const string blobPath = "directory/fileName"; - _meshInboxServiceMock.Setup(s => s.GetMessageByIdAsync(file.MailboxId, fileId)) + _meshInboxServiceMock.Setup(s => s.GetMessageByIdAsync(file.MailboxId, file.FileId)) .ReturnsAsync(new MeshResponse { IsSuccessful = true, @@ -281,32 +182,29 @@ public async Task Run_AcknowledgeMessageFails_WarningLoggedAndProcessingContinue _meshInboxServiceMock.Setup(s => s.AcknowledgeMessageByIdAsync(file.MailboxId, file.FileId)) .ReturnsAsync(new MeshResponse { - IsSuccessful = false + IsSuccessful = false, + Error = new APIErrorResponse + { + ErrorCode = "code", + ErrorDescription = "description", + ErrorEvent = "event" + } }); - var message = new FileExtractQueueMessage { FileId = fileId }; + var message = new FileExtractQueueMessage { FileId = file.FileId }; // Act await _function.Run(message); // Assert - _loggerMock.Verify( - x => x.Log( - LogLevel.Warning, - It.IsAny(), - It.Is((v, t) => - v.ToString().StartsWith("Mesh acknowledgement failed: ") && - v.ToString().EndsWith("This is not a fatal error so processing will continue.")), - null, - It.IsAny>() - ), Times.Once); + LoggerMock.VerifyLogger(LogLevel.Warning, + "Mesh acknowledgement failed: [ ErrorEvent: event, ErrorCode: code, ErrorDescription: description ].\nThis is not a fatal error so processing will continue."); + _blobStoreMock.Verify(b => b.UploadAsync(file, content), Times.Once); _meshInboxServiceMock.Verify(m => m.AcknowledgeMessageByIdAsync(file.MailboxId, file.FileId), Times.Once); _fileTransformQueueClientMock.Verify(q => q.EnqueueFileTransformAsync(file), Times.Once); _fileExtractQueueClientMock.Verify(q => q.SendToPoisonQueueAsync(message), Times.Never); - var updatedFile = _dbContext.MeshFiles.First(); + var updatedFile = AssertFileUpdated(file.FileId, MeshFileStatus.Extracted); Assert.Equal(blobPath, updatedFile.BlobPath); - Assert.Equal(MeshFileStatus.Extracted, updatedFile.Status); - Assert.True(updatedFile.LastUpdatedUtc > originalLastUpdatedUtc); } } diff --git a/tests/ServiceLayer.Mesh.Tests/Functions/FileRetryFunctionTests.cs b/tests/ServiceLayer.Mesh.Tests/Functions/FileRetryFunctionTests.cs index c1e07bb..862d423 100644 --- a/tests/ServiceLayer.Mesh.Tests/Functions/FileRetryFunctionTests.cs +++ b/tests/ServiceLayer.Mesh.Tests/Functions/FileRetryFunctionTests.cs @@ -1,7 +1,4 @@ -using Microsoft.EntityFrameworkCore; -using Microsoft.Extensions.Logging; using Moq; -using ServiceLayer.Data; using ServiceLayer.Data.Models; using ServiceLayer.Mesh.Functions; using ServiceLayer.Mesh.Messaging; @@ -9,38 +6,26 @@ namespace ServiceLayer.Mesh.Tests.Functions; -public class FileRetryFunctionTests +public class FileRetryFunctionTests : FunctionTestBase { - private readonly Mock> _loggerMock; private readonly Mock _fileExtractQueueClientMock; private readonly Mock _fileTransformQueueClientMock; - private readonly Mock _configuration; - private readonly ServiceLayerDbContext _dbContext; private readonly FileRetryFunction _function; public FileRetryFunctionTests() { - _loggerMock = new Mock>(); _fileExtractQueueClientMock = new Mock(); _fileTransformQueueClientMock = new Mock(); - _configuration = new Mock(); - var options = new DbContextOptionsBuilder() - .UseInMemoryDatabase(databaseName: Guid.NewGuid().ToString()) - .ConfigureWarnings(warnings => - warnings.Ignore(Microsoft.EntityFrameworkCore.Diagnostics.InMemoryEventId.TransactionIgnoredWarning)) - .Options; - - _dbContext = new ServiceLayerDbContext(options); - - _configuration.Setup(c => c.StaleHours).Returns(12); + var configurationMock = new Mock(); + configurationMock.Setup(c => c.StaleHours).Returns(12); _function = new FileRetryFunction( - _loggerMock.Object, - _dbContext, + LoggerMock.Object, + DbContext, _fileExtractQueueClientMock.Object, _fileTransformQueueClientMock.Object, - _configuration.Object + configurationMock.Object ); } @@ -49,27 +34,17 @@ public FileRetryFunctionTests() [InlineData(MeshFileStatus.Extracting)] public async Task Run_EnqueuesDiscoveredOrExtractingFilesOlderThan12Hours(MeshFileStatus testStatus) { - var file = new MeshFile - { - FileType = MeshFileType.NbssAppointmentEvents, - MailboxId = "test-mailbox", - FileId = "file-1", - Status = testStatus, - LastUpdatedUtc = DateTime.UtcNow.AddHours(-13) - }; - - _dbContext.MeshFiles.Add(file); - await _dbContext.SaveChangesAsync(); + // Arrange + var file = SaveMeshFile(testStatus, 13); // Act await _function.Run(null); // Assert - _fileExtractQueueClientMock.Verify(q => q.EnqueueFileExtractAsync(It.Is(f => f.FileId == "file-1")), Times.Once); - _fileTransformQueueClientMock.Verify(q => q.EnqueueFileTransformAsync(It.Is(f => f.FileId == "file-1")), Times.Never); + _fileExtractQueueClientMock.Verify(q => q.EnqueueFileExtractAsync(It.Is(f => f.FileId == file.FileId)), Times.Once); + _fileTransformQueueClientMock.Verify(q => q.EnqueueFileTransformAsync(It.Is(f => f.FileId == file.FileId)), Times.Never); - var updatedFile = await _dbContext.MeshFiles.FindAsync("file-1"); - Assert.True(updatedFile!.LastUpdatedUtc > DateTime.UtcNow.AddMinutes(-1)); + AssertFileUpdated(file.FileId, testStatus); } [Theory] @@ -77,28 +52,17 @@ public async Task Run_EnqueuesDiscoveredOrExtractingFilesOlderThan12Hours(MeshFi [InlineData(MeshFileStatus.Transforming)] public async Task Run_EnqueuesExtractedOrTransformingFilesOlderThan12Hours(MeshFileStatus testStatus) { - var file = new MeshFile - { - FileType = MeshFileType.NbssAppointmentEvents, - MailboxId = "test-mailbox", - FileId = "file-1", - Status = testStatus, - LastUpdatedUtc = DateTime.UtcNow.AddHours(-13) - }; - - _dbContext.MeshFiles.Add(file); - await _dbContext.SaveChangesAsync(); + // Arrange + var file = SaveMeshFile(testStatus, 13); // Act await _function.Run(null); // Assert - _fileTransformQueueClientMock.Verify(q => q.EnqueueFileTransformAsync(It.Is(f => f.FileId == "file-1")), Times.Once); - _fileExtractQueueClientMock.Verify(q => q.EnqueueFileExtractAsync(It.Is(f => f.FileId == "file-1")), Times.Never); - + _fileTransformQueueClientMock.Verify(q => q.EnqueueFileTransformAsync(It.Is(f => f.FileId == file.FileId)), Times.Once); + _fileExtractQueueClientMock.Verify(q => q.EnqueueFileExtractAsync(It.Is(f => f.FileId == file.FileId)), Times.Never); - var updatedFile = await _dbContext.MeshFiles.FindAsync("file-1"); - Assert.True(updatedFile!.LastUpdatedUtc > DateTime.UtcNow.AddMinutes(-1)); + AssertFileUpdated(file.FileId, testStatus); } [Theory] @@ -109,18 +73,7 @@ public async Task Run_EnqueuesExtractedOrTransformingFilesOlderThan12Hours(MeshF public async Task Run_SkipsFreshFiles(MeshFileStatus testStatus) { // Arrange - var lastUpdatedUtc = DateTime.UtcNow.AddHours(-1); - - var file = new MeshFile - { - FileType = MeshFileType.NbssAppointmentEvents, - MailboxId = "test-mailbox", - FileId = "file-2", - Status = testStatus, - LastUpdatedUtc = lastUpdatedUtc - }; - _dbContext.MeshFiles.Add(file); - await _dbContext.SaveChangesAsync(); + var file = SaveMeshFile(testStatus); // Act await _function.Run(null); @@ -129,29 +82,23 @@ public async Task Run_SkipsFreshFiles(MeshFileStatus testStatus) _fileExtractQueueClientMock.Verify(q => q.EnqueueFileExtractAsync(It.IsAny()), Times.Never); _fileTransformQueueClientMock.Verify(q => q.EnqueueFileTransformAsync(It.IsAny()), Times.Never); - var updatedFile = await _dbContext.MeshFiles.FindAsync("file-2"); - Assert.True(updatedFile!.LastUpdatedUtc == lastUpdatedUtc); + AssertFileUnchanged(file.FileId, file.Status, file.LastUpdatedUtc); } - [Fact] - public async Task Run_IgnoresFilesInOtherStatuses() + [Theory] + [InlineData(MeshFileStatus.Transformed)] + [InlineData(MeshFileStatus.FailedExtract)] + [InlineData(MeshFileStatus.FailedTransform)] + public async Task Run_IgnoresFilesInOtherStatuses(MeshFileStatus ignoredStatus) { // Arrange - var file = new MeshFile - { - FileType = MeshFileType.NbssAppointmentEvents, - MailboxId = "test-mailbox", - FileId = "file-5", - Status = MeshFileStatus.Transformed, - LastUpdatedUtc = DateTime.UtcNow.AddHours(-20) - }; - _dbContext.MeshFiles.Add(file); - await _dbContext.SaveChangesAsync(); + SaveMeshFile(ignoredStatus, 20); // Act await _function.Run(null); // Assert + _fileTransformQueueClientMock.Verify(q => q.EnqueueFileTransformAsync(It.IsAny()), Times.Never); _fileExtractQueueClientMock.Verify(q => q.EnqueueFileExtractAsync(It.IsAny()), Times.Never); } @@ -170,48 +117,21 @@ public async Task Run_IfNoFilesFoundDoNothing() public async Task Run_ProcessesMultipleEligibleFiles() { // Arrange - var files = new[] - { - new MeshFile - { - FileType = MeshFileType.NbssAppointmentEvents, - MailboxId = "test-mailbox", - FileId = "file-6", - Status = MeshFileStatus.Discovered, - LastUpdatedUtc = DateTime.UtcNow.AddHours(-13) - }, - new MeshFile - { - FileType = MeshFileType.NbssAppointmentEvents, - MailboxId = "test-mailbox", - FileId = "file-7", - Status = MeshFileStatus.Extracted, - LastUpdatedUtc = DateTime.UtcNow.AddHours(-13) - }, - new MeshFile - { - FileType = MeshFileType.NbssAppointmentEvents, - MailboxId = "test-mailbox", - FileId = "file-8", - Status = MeshFileStatus.Transforming, - LastUpdatedUtc = DateTime.UtcNow.AddHours(-13) - } - }; - - _dbContext.MeshFiles.AddRange(files); - await _dbContext.SaveChangesAsync(); + var file1 = SaveMeshFile(MeshFileStatus.Discovered, 13); + var file2 = SaveMeshFile(MeshFileStatus.Extracted, 13); + var file3 = SaveMeshFile(MeshFileStatus.Transforming, 13); // Act await _function.Run(null); // Assert - _fileExtractQueueClientMock.Verify(q => q.EnqueueFileExtractAsync(It.Is(f => f.FileId == "file-6")), Times.Once); - _fileExtractQueueClientMock.Verify(q => q.EnqueueFileExtractAsync(It.IsAny()), Times.Once); - - foreach (var fileId in new[] { "file-6", "file-7", "file-8" }) - { - var updated = await _dbContext.MeshFiles.FindAsync(fileId); - Assert.True(updated!.LastUpdatedUtc > DateTime.UtcNow.AddMinutes(-1)); - } + _fileExtractQueueClientMock.Verify(q => q.EnqueueFileExtractAsync(It.Is(f => f.FileId == file1.FileId)), Times.Once); + _fileTransformQueueClientMock.Verify(q => q.EnqueueFileTransformAsync(It.Is(f => f.FileId == file2.FileId)), Times.Once); + _fileTransformQueueClientMock.Verify(q => q.EnqueueFileTransformAsync(It.Is(f => f.FileId == file3.FileId)), Times.Once); + + AssertFileUpdated(file1.FileId, MeshFileStatus.Discovered); + AssertFileUpdated(file2.FileId, MeshFileStatus.Extracted); + AssertFileUpdated(file3.FileId, MeshFileStatus.Transforming); } } + diff --git a/tests/ServiceLayer.Mesh.Tests/Functions/FileTransformFunctionTests.cs b/tests/ServiceLayer.Mesh.Tests/Functions/FileTransformFunctionTests.cs index e4bf2a9..d90b925 100644 --- a/tests/ServiceLayer.Mesh.Tests/Functions/FileTransformFunctionTests.cs +++ b/tests/ServiceLayer.Mesh.Tests/Functions/FileTransformFunctionTests.cs @@ -1,43 +1,40 @@ +using System.Text.Json; +using System.Text.Json.Serialization; using Google.Protobuf.WellKnownTypes; -using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Logging; using Moq; -using ServiceLayer.Data; using ServiceLayer.Data.Models; using ServiceLayer.Mesh.Configuration; -using ServiceLayer.Mesh.FileTypes.NbssAppointmentEvents; using ServiceLayer.Mesh.Functions; using ServiceLayer.Mesh.Messaging; using ServiceLayer.Mesh.Storage; +using ServiceLayer.TestUtilities; namespace ServiceLayer.Mesh.Tests.Functions; -public class FileTransformFunctionTests +public class FileTransformFunctionTests : FunctionTestBase { - private readonly Mock> _loggerMock = new(); private readonly Mock _blobStoreMock = new(); - private readonly Mock _configuration = new(); - private readonly Mock _fileParser = new(); - private readonly ServiceLayerDbContext _dbContext; + private readonly Mock _fileTransformQueueClientMock = new(); private readonly FileTransformFunction _function; + private readonly List _fileTransformers = new(); + private readonly Mock _fileTransformerMock = new(); public FileTransformFunctionTests() { - var options = new DbContextOptionsBuilder() - .UseInMemoryDatabase(Guid.NewGuid().ToString()) - .ConfigureWarnings(warnings => - warnings.Ignore(Microsoft.EntityFrameworkCore.Diagnostics.InMemoryEventId.TransactionIgnoredWarning)) - .Options; - _dbContext = new ServiceLayerDbContext(options); + var functionConfigurationMock = new Mock(); + functionConfigurationMock.Setup(c => c.StaleHours).Returns(12); - _configuration.Setup(c => c.StaleHours).Returns(12); + _fileTransformerMock.Setup(c => c.CanHandle(MeshFileType.NbssAppointmentEvents)).Returns(true); + _fileTransformers.Add(_fileTransformerMock.Object); _function = new FileTransformFunction( - _loggerMock.Object, - _dbContext, + LoggerMock.Object, + functionConfigurationMock.Object, + DbContext, + _fileTransformQueueClientMock.Object, _blobStoreMock.Object, - _configuration.Object, - _fileParser.Object + _fileTransformers ); } @@ -51,116 +48,180 @@ public async Task Run_FileNotFound_ExitsSilently() await _function.Run(message); // Assert - _loggerMock.Verify( - x => x.Log( - LogLevel.Warning, - It.IsAny(), - It.Is((v, t) => v.ToString() == $"File with id: {message.FileId} not found in MeshFiles table."), - null, - It.IsAny>() - ), Times.Once); - - Assert.Equal(0, _dbContext.MeshFiles.Count()); + LoggerMock.VerifyLogger(LogLevel.Warning, $"File with id: {message.FileId} not found in MeshFiles table."); + + Assert.Equal(0, DbContext.MeshFiles.Count()); + _blobStoreMock.Verify(x => x.DownloadAsync(It.IsAny()), Times.Never); + } + + [Theory] + [InlineData(MeshFileStatus.Discovered)] + [InlineData(MeshFileStatus.Extracting)] + [InlineData(MeshFileStatus.Transforming)] + [InlineData(MeshFileStatus.Transformed)] + [InlineData(MeshFileStatus.FailedExtract)] + [InlineData(MeshFileStatus.FailedTransform)] + public async Task Run_FileStatusInvalid_ExitsSilently(MeshFileStatus invalidStatus) + { + // Arrange + var file = SaveMeshFile(invalidStatus); + var message = new FileTransformQueueMessage { FileId = file.FileId }; + + // Act + await _function.Run(message); + + // Assert + LoggerMock.VerifyLogger(LogLevel.Warning, + $"File with id: {file.FileId} found in MeshFiles table but is not suitable for transformation. Status: {file.Status}, LastUpdatedUtc: {file.LastUpdatedUtc.ToTimestamp()}."); + _blobStoreMock.Verify(x => x.DownloadAsync(It.IsAny()), Times.Never); + _fileTransformerMock.Verify(c => c.TransformFileAsync(It.IsAny(), It.IsAny()), Times.Never); } [Fact] - public async Task Run_FileStatusInvalid_ExitsSilently() + public async Task Run_FileValidNoTransformersExist_ErrorLoggedAndStatusUpdated() { // Arrange - var file = new MeshFile - { - FileType = MeshFileType.NbssAppointmentEvents, - MailboxId = "test-mailbox", - FileId = "file-1", - Status = MeshFileStatus.FailedExtract, // Not eligible - LastUpdatedUtc = DateTime.UtcNow - }; - _dbContext.MeshFiles.Add(file); - await _dbContext.SaveChangesAsync(); + var file = SaveMeshFile(); - var message = new FileTransformQueueMessage { FileId = "file-1" }; + _fileTransformers.Clear(); + + var message = new FileTransformQueueMessage { FileId = file.FileId }; // Act await _function.Run(message); // Assert - _loggerMock.Verify( - x => x.Log( - LogLevel.Warning, - It.IsAny(), - It.Is((v, t) => v.ToString() == $"File with id: {message.FileId} found in MeshFiles table but is not suitable for transformation. Status: {file.Status}, LastUpdatedUtc: {file.LastUpdatedUtc.ToTimestamp()}."), - null, - It.IsAny>() - ), Times.Once); - var fileFromDb = await _dbContext.MeshFiles.SingleOrDefaultAsync(x => x.FileId == file.FileId); - Assert.Equal(MeshFileStatus.FailedExtract, fileFromDb?.Status); - _blobStoreMock.Verify(x => x.DownloadAsync(It.IsAny()), Times.Never); + LoggerMock.VerifyLogger(LogLevel.Error, + $"An exception occurred during file transformation for fileId: {file.FileId}", + e => e is InvalidOperationException && e.Message.StartsWith("No transformer registered to handle file type: ")); + + _blobStoreMock.Verify(x => x.DownloadAsync(file), Times.Never); + _fileTransformQueueClientMock.Verify(q => q.SendToPoisonQueueAsync(message), Times.Once); + + AssertFileUpdated(file.FileId, MeshFileStatus.FailedTransform); } [Fact] - public async Task Run_FileStatusTransformingButNotTimedOut_ExitsSilently() + public async Task Run_FileValidMultipleTransformersExist_ErrorLoggedAndStatusUpdated() { // Arrange - var file = new MeshFile - { - FileType = MeshFileType.NbssAppointmentEvents, - MailboxId = "test-mailbox", - FileId = "file-1", - Status = MeshFileStatus.Transforming, - LastUpdatedUtc = DateTime.UtcNow.AddHours(-11) // Not timed out - }; - _dbContext.MeshFiles.Add(file); - await _dbContext.SaveChangesAsync(); + var file = SaveMeshFile(); + + var anotherTransformer = new Mock(); + anotherTransformer.Setup(x => x.CanHandle(MeshFileType.NbssAppointmentEvents)).Returns(true); + _fileTransformers.Add(anotherTransformer.Object); - var message = new FileTransformQueueMessage { FileId = "file-1" }; + var message = new FileTransformQueueMessage { FileId = file.FileId }; // Act await _function.Run(message); // Assert - _loggerMock.Verify( - x => x.Log( - LogLevel.Warning, - It.IsAny(), - It.Is((v, t) => v.ToString() == $"File with id: {message.FileId} found in MeshFiles table but is not suitable for transformation. Status: {file.Status}, LastUpdatedUtc: {file.LastUpdatedUtc.ToTimestamp()}."), - null, - It.IsAny>() - ), Times.Once); - var fileFromDb = await _dbContext.MeshFiles.SingleOrDefaultAsync(x => x.FileId == file.FileId); - Assert.Equal(MeshFileStatus.Transforming, fileFromDb?.Status); - _blobStoreMock.Verify(x => x.DownloadAsync(It.IsAny()), Times.Never); + LoggerMock.VerifyLogger(LogLevel.Error, + $"An exception occurred during file transformation for fileId: {file.FileId}", + e => e is InvalidOperationException && e.Message.StartsWith("Multiple transformers found for file type: ")); + + _blobStoreMock.Verify(x => x.DownloadAsync(file), Times.Never); + _fileTransformQueueClientMock.Verify(q => q.SendToPoisonQueueAsync(message), Times.Once); + + AssertFileUpdated(file.FileId, MeshFileStatus.FailedTransform); } [Fact] - public async Task Run_FileValid_DownloadsBlob() + public async Task Run_FileHasValidationErrors_ErrorLoggedAndStatusAndValidationErrorsUpdated() { // Arrange - var file = new MeshFile + var file = SaveMeshFile(); + + var expectedStream = new MemoryStream(); + _blobStoreMock.Setup(m => m.DownloadAsync(file)).ReturnsAsync(expectedStream); + + var validationErrors = new List { - FileType = MeshFileType.NbssAppointmentEvents, - MailboxId = "test-mailbox", - FileId = "file-1", - Status = MeshFileStatus.Extracted, - LastUpdatedUtc = DateTime.UtcNow + new() { Code = "NBSSAPPT001", Error = "error message", Field = "field", RowNumber = 1 }, + new() { Code = "NBSSAPPT002", Error = "error message 2", Field = "field 2" } }; - _dbContext.MeshFiles.Add(file); - await _dbContext.SaveChangesAsync(); - var message = new FileTransformQueueMessage { FileId = "file-1" }; + _fileTransformerMock.Setup(c => c.TransformFileAsync(expectedStream, file)) + .ReturnsAsync(validationErrors); + + var message = new FileTransformQueueMessage { FileId = file.FileId }; // Act await _function.Run(message); // Assert - _loggerMock.Verify( - x => x.Log( - LogLevel.Warning, - It.IsAny(), - It.IsAny(), - null, - It.IsAny>() - ), Times.Never); + LoggerMock.VerifyLogger(LogLevel.Error, + $"An exception occurred during file transformation for fileId: {file.FileId}", + e => e is InvalidOperationException && e.Message.StartsWith("Validation errors encountered")); + _blobStoreMock.Verify(x => x.DownloadAsync(file), Times.Once); + _fileTransformQueueClientMock.Verify(q => q.SendToPoisonQueueAsync(message), Times.Once); + + var updatedFile = AssertFileUpdated(file.FileId, MeshFileStatus.FailedTransform); + var savedValidationErrors = DeserializeValidationErrorsFromMeshFile(updatedFile); + Assert.Equal(validationErrors, savedValidationErrors, new ValidationErrorComparer()); + } + + [Theory] + [InlineData(MeshFileStatus.Extracted, 0)] + [InlineData(MeshFileStatus.Transforming, 13)] + public async Task Run_FileValid_FileTransformedAndStatusUpdated(MeshFileStatus validStatus, int hoursOld) + { + // Arrange + var file = SaveMeshFile(validStatus, hoursOld); + + var expectedStream = new MemoryStream(); + _blobStoreMock.Setup(m => m.DownloadAsync(file)).ReturnsAsync(expectedStream); + + _fileTransformerMock.Setup(c => c.TransformFileAsync(expectedStream, file)) + .ReturnsAsync(new List()); + + var message = new FileTransformQueueMessage { FileId = file.FileId }; + + // Act + await _function.Run(message); + + // Assert + LoggerMock.VerifyNoLogs(LogLevel.Warning); + _blobStoreMock.Verify(x => x.DownloadAsync(file), Times.Once); + _fileTransformerMock.Verify(x => x.TransformFileAsync(expectedStream, file), Times.Once); + AssertFileUpdated(file.FileId, MeshFileStatus.Transformed); + } + + private static readonly JsonSerializerOptions ValidationErrorJsonOptions = new() + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, + WriteIndented = true, + Converters = { new JsonStringEnumConverter(JsonNamingPolicy.CamelCase) } + }; + + private static List DeserializeValidationErrorsFromMeshFile(MeshFile file) + { + return JsonSerializer.Deserialize>( + file.ValidationErrors ?? "[]", + ValidationErrorJsonOptions + ) ?? []; + } + + private class ValidationErrorComparer : IEqualityComparer + { + public bool Equals(ValidationError? x, ValidationError? y) + { + if (ReferenceEquals(x, y)) return true; + if (x is null || y is null) return false; + + return x.Field == y.Field && + x.Error == y.Error && + x.Code == y.Code && + x.RowNumber == y.RowNumber; + } + + public int GetHashCode(ValidationError obj) + { + return HashCode.Combine(obj.Field, obj.Error, obj.Code, obj.RowNumber); + } } } diff --git a/tests/ServiceLayer.Mesh.Tests/Functions/FunctionTestBase.cs b/tests/ServiceLayer.Mesh.Tests/Functions/FunctionTestBase.cs new file mode 100644 index 0000000..dda7bc0 --- /dev/null +++ b/tests/ServiceLayer.Mesh.Tests/Functions/FunctionTestBase.cs @@ -0,0 +1,56 @@ +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Logging; +using Moq; +using ServiceLayer.Data; +using ServiceLayer.Data.Models; + +namespace ServiceLayer.Mesh.Tests.Functions; + +public abstract class FunctionTestBase +{ + protected readonly ServiceLayerDbContext DbContext; + protected readonly Mock> LoggerMock = new(); + + protected FunctionTestBase() + { + var options = new DbContextOptionsBuilder() + .UseInMemoryDatabase(Guid.NewGuid().ToString()) + .ConfigureWarnings(warnings => + warnings.Ignore(Microsoft.EntityFrameworkCore.Diagnostics.InMemoryEventId.TransactionIgnoredWarning)) + .Options; + + DbContext = new ServiceLayerDbContext(options); + } + + protected MeshFile SaveMeshFile(MeshFileStatus status = MeshFileStatus.Extracted, int hoursOld = 1) + { + var file = new MeshFile + { + FileType = MeshFileType.NbssAppointmentEvents, + MailboxId = Guid.NewGuid().ToString(), + FileId = Guid.NewGuid().ToString(), + Status = status, + LastUpdatedUtc = DateTime.UtcNow.AddHours(-hoursOld), + }; + DbContext.MeshFiles.Add(file); + DbContext.SaveChanges(); + return file; + } + + protected MeshFile AssertFileUnchanged(string fileId, MeshFileStatus expectedStatus, + DateTime expectedLastUpdatedUtc) + { + var unchanged = DbContext.MeshFiles.Single(x => x.FileId == fileId); + Assert.Equal(expectedStatus, unchanged.Status); + Assert.Equal(expectedLastUpdatedUtc, unchanged.LastUpdatedUtc); + return unchanged; + } + + protected MeshFile AssertFileUpdated(string fileId, MeshFileStatus expectedStatus) + { + var updated = DbContext.MeshFiles.Single(x => x.FileId == fileId); + Assert.Equal(expectedStatus, updated.Status); + Assert.True(updated.LastUpdatedUtc > DateTime.UtcNow.AddSeconds(-10)); + return updated; + } +} diff --git a/tests/ServiceLayer.Mesh.Tests/Functions/MeshHandshakeFunctionTests.cs b/tests/ServiceLayer.Mesh.Tests/Functions/MeshHandshakeFunctionTests.cs index 6c16e48..efc1ce3 100644 --- a/tests/ServiceLayer.Mesh.Tests/Functions/MeshHandshakeFunctionTests.cs +++ b/tests/ServiceLayer.Mesh.Tests/Functions/MeshHandshakeFunctionTests.cs @@ -5,30 +5,29 @@ using NHS.MESH.Client.Models; using ServiceLayer.Mesh.Configuration; using ServiceLayer.Mesh.Functions; +using ServiceLayer.TestUtilities; namespace ServiceLayer.Mesh.Tests.Functions; -public class MeshHandshakeFunctionTests +public class MeshHandshakeFunctionTests : FunctionTestBase { - private readonly Mock> _loggerMock; private readonly Mock _meshOperationServiceMock; - private readonly Mock _configurationMock; private readonly MeshHandshakeFunction _function; private readonly TimerInfo _timerInfo; private const string TestMailboxId = "test-mailbox-123"; public MeshHandshakeFunctionTests() { - _loggerMock = new Mock>(); _meshOperationServiceMock = new Mock(); - _configurationMock = new Mock(); _timerInfo = new TimerInfo(); - _configurationMock.Setup(c => c.NbssMeshMailboxId).Returns(TestMailboxId); + var functionConfigurationMock = new Mock(); + functionConfigurationMock.Setup(c => c.NbssMeshMailboxId).Returns(TestMailboxId); + _function = new MeshHandshakeFunction( - _loggerMock.Object, + LoggerMock.Object, _meshOperationServiceMock.Object, - _configurationMock.Object + functionConfigurationMock.Object ); } @@ -50,8 +49,8 @@ public async Task Run_SuccessfulHandshake_LogsSuccessAndCompletion() // Assert _meshOperationServiceMock.Verify(s => s.MeshHandshakeAsync(TestMailboxId), Times.Once()); - VerifyLogMessage(LogLevel.Information, "MeshHandshakeFunction started"); - VerifyLogMessage(LogLevel.Information, "Mesh handshake completed successfully for mailbox"); + LoggerMock.VerifyLogger(LogLevel.Information,"MeshHandshakeFunction started."); + LoggerMock.VerifyLogger(LogLevel.Information, $"Mesh handshake completed successfully for mailbox {TestMailboxId}."); } [Fact] @@ -63,7 +62,9 @@ public async Task Run_FailedHandshake_LogsWarningAndCompletion() IsSuccessful = false, Error = new APIErrorResponse { - ErrorDescription = "Authentication failed" + ErrorCode = "code", + ErrorEvent = "event", + ErrorDescription = "desc" } }; _meshOperationServiceMock @@ -75,12 +76,13 @@ public async Task Run_FailedHandshake_LogsWarningAndCompletion() // Assert _meshOperationServiceMock.Verify(s => s.MeshHandshakeAsync(TestMailboxId), Times.Once()); - VerifyLogMessage(LogLevel.Information, "MeshHandshakeFunction started"); - VerifyLogMessage(LogLevel.Warning, "Mesh handshake failed"); + LoggerMock.VerifyLogger(LogLevel.Information,"MeshHandshakeFunction started."); + LoggerMock.VerifyLogger(LogLevel.Warning, + $"Mesh handshake failed for mailbox {TestMailboxId}: [ ErrorEvent: event, ErrorCode: code, ErrorDescription: desc ]"); } [Fact] - public async Task Run_ExceptionThrown_LogsErrorAndCompletion() + public async Task Run_ExceptionThrown_LogsWarningAndCompletion() { // Arrange var expectedException = new InvalidOperationException("Connection failed"); @@ -93,27 +95,8 @@ public async Task Run_ExceptionThrown_LogsErrorAndCompletion() // Assert _meshOperationServiceMock.Verify(s => s.MeshHandshakeAsync(TestMailboxId), Times.Once()); - VerifyLogMessage(LogLevel.Information, "MeshHandshakeFunction started"); - VerifyLogMessage(LogLevel.Error, "An error occurred during mesh handshake"); - _loggerMock.Verify( - x => x.Log( - LogLevel.Error, - It.IsAny(), - It.IsAny(), - expectedException, - It.IsAny>()), - Times.Once); - } - - private void VerifyLogMessage(LogLevel level, string expectedMessage) - { - _loggerMock.Verify( - x => x.Log( - level, - It.IsAny(), - It.Is((v, t) => (v.ToString() ?? string.Empty).Contains(expectedMessage)), - It.IsAny(), - It.IsAny>()), - Times.Once); + LoggerMock.VerifyLogger(LogLevel.Information,"MeshHandshakeFunction started."); + LoggerMock.VerifyLogger(LogLevel.Warning, $"An error occurred during mesh handshake for mailbox {TestMailboxId}.", + e => e is InvalidOperationException && e.Message == "Connection failed"); } } diff --git a/tests/ServiceLayer.Mesh.Tests/ServiceLayer.Mesh.Tests.csproj b/tests/ServiceLayer.Mesh.Tests/ServiceLayer.Mesh.Tests.csproj index 0f95528..12c8eaf 100644 --- a/tests/ServiceLayer.Mesh.Tests/ServiceLayer.Mesh.Tests.csproj +++ b/tests/ServiceLayer.Mesh.Tests/ServiceLayer.Mesh.Tests.csproj @@ -18,6 +18,7 @@ + diff --git a/tests/ServiceLayer.TestUtilities/LoggerAssertions.cs b/tests/ServiceLayer.TestUtilities/LoggerAssertions.cs new file mode 100644 index 0000000..a134f44 --- /dev/null +++ b/tests/ServiceLayer.TestUtilities/LoggerAssertions.cs @@ -0,0 +1,44 @@ +using System.Linq.Expressions; +using Moq; +using Microsoft.Extensions.Logging; + +namespace ServiceLayer.TestUtilities; + +public static class LoggerAssertions +{ + public static void VerifyLogger(this Mock> loggerMock, LogLevel level, string expectedMessage) + { + loggerMock.Verify( + x => x.Log( + level, + It.IsAny(), + It.Is((v, t) => v.ToString() == expectedMessage), + null, + It.IsAny>() + ), Times.Once); + } + + public static void VerifyLogger(this Mock> loggerMock, LogLevel level, string expectedMessage, Expression> exceptionMatch) + { + loggerMock.Verify( + x => x.Log( + level, + It.IsAny(), + It.Is((v, t) => v.ToString() == expectedMessage), + It.Is(exceptionMatch), + It.IsAny>() + ), Times.Once); + } + + public static void VerifyNoLogs(this Mock> loggerMock, LogLevel level) + { + loggerMock.Verify( + x => x.Log( + level, + It.IsAny(), + It.IsAny(), + null, + It.IsAny>() + ), Times.Never); + } +} diff --git a/tests/ServiceLayer.TestUtilities/ServiceLayer.TestUtilities.csproj b/tests/ServiceLayer.TestUtilities/ServiceLayer.TestUtilities.csproj new file mode 100644 index 0000000..88bade2 --- /dev/null +++ b/tests/ServiceLayer.TestUtilities/ServiceLayer.TestUtilities.csproj @@ -0,0 +1,23 @@ + + + + net9.0 + enable + enable + false + + + + + + + + + + + + + + + +