diff --git a/.env.example b/.env.example index 453fbb4..f71dfbc 100644 --- a/.env.example +++ b/.env.example @@ -1,20 +1,21 @@ # Database Configuration DATABASE_USER=SA DATABASE_PASSWORD=YourStrong@Passw0rd -DATABASE_NAME=PathwayCoordinator +DATABASE_NAME=ServiceLayer DATABASE_HOST=db AZURE_WEB_JOBS_STORAGE=UseDevelopmentStorage=true DatabaseConnectionString=Server=${DATABASE_HOST};Database=${DATABASE_NAME};User Id=${DATABASE_USER};Password=${DATABASE_PASSWORD};TrustServerCertificate=True AzureWebJobsStorage=UseDevelopmentStorage=true FUNCTIONS_WORKER_RUNTIME=dotnet-isolated -MailboxId=X26ABC1 MeshSharedKey=TestKey MeshPassword=password -BSSMailBox=X26ABC1 +NbssMailboxId=X26ABC1 MeshApiBaseUrl=http://localhost:8700/messageexchange ASPNETCORE_ENVIRONMENT=Development -DiscoveryTimerExpression=*/5 * * * * +FileDiscoveryTimerExpression=*/5 * * * * QueueUrl=http://127.0.0.1:10001 +FileExtractQueueName=file-extract +FileTransformQueueName=file-transform # API Configuration API_PORT=7071 diff --git a/src/ServiceLayer.Mesh/Configuration/AppConfiguration.cs b/src/ServiceLayer.Mesh/Configuration/AppConfiguration.cs new file mode 100644 index 0000000..3470812 --- /dev/null +++ b/src/ServiceLayer.Mesh/Configuration/AppConfiguration.cs @@ -0,0 +1,26 @@ +namespace ServiceLayer.Mesh.Configuration; + +public class AppConfiguration : + IFileDiscoveryFunctionConfiguration, + IFileExtractFunctionConfiguration, + IFileExtractQueueClientConfiguration, + IFileTransformQueueClientConfiguration +{ + public string NbssMeshMailboxId => GetRequired("NbssMailboxId"); + + public string FileExtractQueueName => GetRequired("FileExtractQueueName"); + + public string FileTransformQueueName => GetRequired("FileTransformQueueName"); + + private static string GetRequired(string key) + { + var value = Environment.GetEnvironmentVariable(key); + + if (string.IsNullOrEmpty(value)) + { + throw new InvalidOperationException($"Environment variable '{key}' is not set or is empty."); + } + + return value; + } +} diff --git a/src/ServiceLayer.Mesh/Configuration/IFileDiscoveryFunctionConfiguration.cs b/src/ServiceLayer.Mesh/Configuration/IFileDiscoveryFunctionConfiguration.cs new file mode 100644 index 0000000..b29d1ac --- /dev/null +++ b/src/ServiceLayer.Mesh/Configuration/IFileDiscoveryFunctionConfiguration.cs @@ -0,0 +1,6 @@ +namespace ServiceLayer.Mesh.Configuration; + +public interface IFileDiscoveryFunctionConfiguration +{ + string NbssMeshMailboxId { get; } +} diff --git a/src/ServiceLayer.Mesh/Configuration/IFileExtractFunctionConfiguration.cs b/src/ServiceLayer.Mesh/Configuration/IFileExtractFunctionConfiguration.cs new file mode 100644 index 0000000..85ed116 --- /dev/null +++ b/src/ServiceLayer.Mesh/Configuration/IFileExtractFunctionConfiguration.cs @@ -0,0 +1,6 @@ +namespace ServiceLayer.Mesh.Configuration; + +public interface IFileExtractFunctionConfiguration +{ + string NbssMeshMailboxId { get; } +} diff --git a/src/ServiceLayer.Mesh/Configuration/IFileExtractQueueClientConfiguration.cs b/src/ServiceLayer.Mesh/Configuration/IFileExtractQueueClientConfiguration.cs new file mode 100644 index 0000000..bc1f4e4 --- /dev/null +++ b/src/ServiceLayer.Mesh/Configuration/IFileExtractQueueClientConfiguration.cs @@ -0,0 +1,6 @@ +namespace ServiceLayer.Mesh.Configuration; + +public interface IFileExtractQueueClientConfiguration +{ + string FileExtractQueueName { get; } +} diff --git a/src/ServiceLayer.Mesh/Configuration/IFileTransformQueueClientConfiguration.cs b/src/ServiceLayer.Mesh/Configuration/IFileTransformQueueClientConfiguration.cs new file mode 100644 index 0000000..e466f63 --- /dev/null +++ b/src/ServiceLayer.Mesh/Configuration/IFileTransformQueueClientConfiguration.cs @@ -0,0 +1,6 @@ +namespace ServiceLayer.Mesh.Configuration; + +public interface IFileTransformQueueClientConfiguration +{ + string FileTransformQueueName { get; } +} diff --git a/src/ServiceLayer.Mesh/Data/DesignTimeDbContextFactory.cs b/src/ServiceLayer.Mesh/Data/DesignTimeDbContextFactory.cs index 569baf9..ec4073e 100644 --- a/src/ServiceLayer.Mesh/Data/DesignTimeDbContextFactory.cs +++ b/src/ServiceLayer.Mesh/Data/DesignTimeDbContextFactory.cs @@ -1,8 +1,7 @@ using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore.Design; -using ServiceLayer.Mesh.Data; -namespace ParticipantManager.API.Data; +namespace ServiceLayer.Mesh.Data; public class DesignTimeDbContextFactory : IDesignTimeDbContextFactory { diff --git a/src/ServiceLayer.Mesh/Data/ServiceLayerDbConext.cs b/src/ServiceLayer.Mesh/Data/ServiceLayerDbContext.cs similarity index 100% rename from src/ServiceLayer.Mesh/Data/ServiceLayerDbConext.cs rename to src/ServiceLayer.Mesh/Data/ServiceLayerDbContext.cs diff --git a/src/ServiceLayer.Mesh/Functions/DiscoveryFunction.cs b/src/ServiceLayer.Mesh/Functions/DiscoveryFunction.cs deleted file mode 100644 index ae06365..0000000 --- a/src/ServiceLayer.Mesh/Functions/DiscoveryFunction.cs +++ /dev/null @@ -1,69 +0,0 @@ -using Azure.Storage.Queues; -using Microsoft.Azure.Functions.Worker; -using Microsoft.EntityFrameworkCore; -using Microsoft.Extensions.Logging; -using NHS.MESH.Client.Contracts.Services; -using ServiceLayer.Mesh.Data; -using ServiceLayer.Mesh.Models; - -namespace ServiceLayer.Mesh.Functions -{ - public class DiscoveryFunction - { - private readonly ILogger _logger; - private readonly IMeshInboxService _meshInboxService; - private readonly ServiceLayerDbContext _serviceLayerDbContext; - private readonly QueueClient _queueClient; - - public DiscoveryFunction(ILogger logger, IMeshInboxService meshInboxService, ServiceLayerDbContext serviceLayerDbContext, QueueClient queueClient) - { - _logger = logger; - _meshInboxService = meshInboxService; - _serviceLayerDbContext = serviceLayerDbContext; - _queueClient = queueClient; - } - - [Function("DiscoveryFunction")] - public async Task Run([TimerTrigger("%DiscoveryTimerExpression%")] TimerInfo myTimer) - { - _logger.LogInformation($"DiscoveryFunction started at: {DateTime.Now}"); - - var mailboxId = Environment.GetEnvironmentVariable("BSSMailBox") - ?? throw new InvalidOperationException($"Environment variable 'BSSMailBox' is not set or is empty."); - - var response = await _meshInboxService.GetMessagesAsync(mailboxId); - - _queueClient.CreateIfNotExists(); - - foreach (var messageId in response.Response.Messages) - { - using var transaction = await _serviceLayerDbContext.Database.BeginTransactionAsync(); - - var existing = await _serviceLayerDbContext.MeshFiles - .AnyAsync(f => f.FileId == messageId); - - if (!existing) - { - _serviceLayerDbContext.MeshFiles.Add(new MeshFile - { - FileId = messageId, - FileType = MeshFileType.NbssAppointmentEvents, - MailboxId = mailboxId, - Status = MeshFileStatus.Discovered, - FirstSeenUtc = DateTime.UtcNow, - LastUpdatedUtc = DateTime.UtcNow - }); - - await _serviceLayerDbContext.SaveChangesAsync(); - await transaction.CommitAsync(); - - _queueClient.SendMessage(messageId); - } - else - { - await transaction.RollbackAsync(); - } - } - } - } -} diff --git a/src/ServiceLayer.Mesh/Functions/FileDiscoveryFunction.cs b/src/ServiceLayer.Mesh/Functions/FileDiscoveryFunction.cs new file mode 100644 index 0000000..423e177 --- /dev/null +++ b/src/ServiceLayer.Mesh/Functions/FileDiscoveryFunction.cs @@ -0,0 +1,59 @@ +using Microsoft.Azure.Functions.Worker; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Logging; +using NHS.MESH.Client.Contracts.Services; +using ServiceLayer.Mesh.Configuration; +using ServiceLayer.Mesh.Data; +using ServiceLayer.Mesh.Messaging; +using ServiceLayer.Mesh.Models; + +namespace ServiceLayer.Mesh.Functions +{ + public class FileDiscoveryFunction( + ILogger logger, + IFileDiscoveryFunctionConfiguration configuration, + IMeshInboxService meshInboxService, + ServiceLayerDbContext serviceLayerDbContext, + IFileExtractQueueClient fileExtractQueueClient) + { + [Function("FileDiscoveryFunction")] + public async Task Run([TimerTrigger("%FileDiscoveryTimerExpression%")] TimerInfo myTimer) + { + logger.LogInformation("{functionName} started at: {time}", nameof(FileDiscoveryFunction), DateTime.UtcNow); + + var response = await meshInboxService.GetMessagesAsync(configuration.NbssMeshMailboxId); + + foreach (var messageId in response.Response.Messages) + { + await using var transaction = await serviceLayerDbContext.Database.BeginTransactionAsync(); + + var existing = await serviceLayerDbContext.MeshFiles + .AnyAsync(f => f.FileId == messageId); + + if (!existing) + { + var file = new MeshFile + { + FileId = messageId, + FileType = MeshFileType.NbssAppointmentEvents, + MailboxId = configuration.NbssMeshMailboxId, + Status = MeshFileStatus.Discovered, + FirstSeenUtc = DateTime.UtcNow, + LastUpdatedUtc = DateTime.UtcNow + }; + + serviceLayerDbContext.MeshFiles.Add(file); + + await serviceLayerDbContext.SaveChangesAsync(); + await transaction.CommitAsync(); + + await fileExtractQueueClient.EnqueueFileExtractAsync(file); + } + else + { + await transaction.RollbackAsync(); + } + } + } + } +} diff --git a/src/ServiceLayer.Mesh/Functions/FileExtractFunction.cs b/src/ServiceLayer.Mesh/Functions/FileExtractFunction.cs new file mode 100644 index 0000000..203c91b --- /dev/null +++ b/src/ServiceLayer.Mesh/Functions/FileExtractFunction.cs @@ -0,0 +1,124 @@ +using Google.Protobuf.WellKnownTypes; +using Microsoft.Azure.Functions.Worker; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Logging; +using NHS.MESH.Client.Contracts.Services; +using ServiceLayer.Mesh.Configuration; +using ServiceLayer.Mesh.Data; +using ServiceLayer.Mesh.Messaging; +using ServiceLayer.Mesh.Models; +using ServiceLayer.Mesh.Storage; + +namespace ServiceLayer.Mesh.Functions; + +public class FileExtractFunction( + ILogger logger, + IFileExtractFunctionConfiguration configuration, + IMeshInboxService meshInboxService, + ServiceLayerDbContext serviceLayerDbContext, + IFileTransformQueueClient fileTransformQueueClient, + IFileExtractQueueClient fileExtractQueueClient, + IMeshFilesBlobStore meshFileBlobStore) +{ + [Function("FileExtractFunction")] + public async Task Run([QueueTrigger("%FileExtractQueueName%")] FileExtractQueueMessage message) + { + logger.LogInformation("{functionName} started at: {time}", nameof(FileDiscoveryFunction), DateTime.UtcNow); + + await using var transaction = await serviceLayerDbContext.Database.BeginTransactionAsync(); + + var file = await GetFileAsync(message.FileId); + if (file == null) + { + return; + } + + if (!IsFileSuitableForExtraction(file)) + { + return; + } + + await UpdateFileStatusForExtraction(file); + await transaction.CommitAsync(); + + try + { + await ProcessFileExtraction(file, message); + } + catch (Exception ex) + { + await HandleExtractionError(file, message, ex); + } + } + + private async Task GetFileAsync(string fileId) + { + var file = await serviceLayerDbContext.MeshFiles + .FirstOrDefaultAsync(f => f.FileId == fileId); + + if (file == null) + { + logger.LogWarning("File with id: {fileId} not found in MeshFiles table.", fileId); + } + + return file; + } + + private bool IsFileSuitableForExtraction(MeshFile file) + { + // We only want to extract files if they are in a Discovered state, + // or are in an Extracting state and were last touched over 12 hours ago. + var expectedStatuses = new[] { MeshFileStatus.Discovered, MeshFileStatus.Extracting }; + if (!expectedStatuses.Contains(file.Status) || + (file.Status == MeshFileStatus.Extracting && file.LastUpdatedUtc > DateTime.UtcNow.AddHours(-12))) + { + logger.LogWarning( + "File with id: {fileId} found in MeshFiles table but is not suitable for extraction. Status: {status}, LastUpdatedUtc: {lastUpdatedUtc}.", + file.FileId, + file.Status, + file.LastUpdatedUtc.ToTimestamp()); + return false; + } + return true; + } + + private async Task UpdateFileStatusForExtraction(MeshFile file) + { + file.Status = MeshFileStatus.Extracting; + file.LastUpdatedUtc = DateTime.UtcNow; + await serviceLayerDbContext.SaveChangesAsync(); + } + + private async Task ProcessFileExtraction(MeshFile file, FileExtractQueueMessage message) + { + var meshResponse = await meshInboxService.GetMessageByIdAsync(configuration.NbssMeshMailboxId, file.FileId); + if (!meshResponse.IsSuccessful) + { + throw new InvalidOperationException($"Mesh extraction failed: {meshResponse.Error}"); + } + + var blobPath = await meshFileBlobStore.UploadAsync(file, meshResponse.Response.FileAttachment.Content); + + var meshAcknowledgementResponse = await meshInboxService.AcknowledgeMessageByIdAsync(configuration.NbssMeshMailboxId, message.FileId); + if (!meshAcknowledgementResponse.IsSuccessful) + { + logger.LogWarning("Mesh acknowledgement failed: {error}.\nThis is not a fatal error so processing will continue.", meshAcknowledgementResponse.Error); + } + + file.BlobPath = blobPath; + file.Status = MeshFileStatus.Extracted; + file.LastUpdatedUtc = DateTime.UtcNow; + await serviceLayerDbContext.SaveChangesAsync(); + + await fileTransformQueueClient.EnqueueFileTransformAsync(file); + } + + private async Task HandleExtractionError(MeshFile file, FileExtractQueueMessage message, Exception ex) + { + logger.LogError(ex, "An exception occurred during file extraction for fileId: {fileId}", message.FileId); + file.Status = MeshFileStatus.FailedExtract; + file.LastUpdatedUtc = DateTime.UtcNow; + await serviceLayerDbContext.SaveChangesAsync(); + await fileExtractQueueClient.SendToPoisonQueueAsync(message); + } +} diff --git a/src/ServiceLayer.Mesh/Functions/FileRetryFunction.cs b/src/ServiceLayer.Mesh/Functions/FileRetryFunction.cs new file mode 100644 index 0000000..71af124 --- /dev/null +++ b/src/ServiceLayer.Mesh/Functions/FileRetryFunction.cs @@ -0,0 +1,6 @@ +namespace ServiceLayer.Mesh.Functions; + +public class FileRetryFunction +{ + +} diff --git a/src/ServiceLayer.Mesh/Functions/FileTransformFunction.cs b/src/ServiceLayer.Mesh/Functions/FileTransformFunction.cs new file mode 100644 index 0000000..7ed7d7e --- /dev/null +++ b/src/ServiceLayer.Mesh/Functions/FileTransformFunction.cs @@ -0,0 +1,6 @@ +namespace ServiceLayer.Mesh.Functions; + +public class FileTransformFunction +{ + +} diff --git a/src/ServiceLayer.Mesh/Functions/MeshHandshakeFunction.cs b/src/ServiceLayer.Mesh/Functions/MeshHandshakeFunction.cs new file mode 100644 index 0000000..2dac340 --- /dev/null +++ b/src/ServiceLayer.Mesh/Functions/MeshHandshakeFunction.cs @@ -0,0 +1,6 @@ +namespace ServiceLayer.Mesh.Functions; + +public class MeshHandshakeFunction +{ + +} diff --git a/src/ServiceLayer.Mesh/Messaging/FileExtractQueueClient.cs b/src/ServiceLayer.Mesh/Messaging/FileExtractQueueClient.cs new file mode 100644 index 0000000..6386783 --- /dev/null +++ b/src/ServiceLayer.Mesh/Messaging/FileExtractQueueClient.cs @@ -0,0 +1,25 @@ +using Azure.Storage.Queues; +using Microsoft.Extensions.Logging; +using ServiceLayer.Mesh.Configuration; +using ServiceLayer.Mesh.Models; + +namespace ServiceLayer.Mesh.Messaging; + +public class FileExtractQueueClient( + ILogger logger, + IFileExtractQueueClientConfiguration configuration, + QueueServiceClient queueServiceClient) + : QueueClientBase(logger, queueServiceClient), IFileExtractQueueClient +{ + public async Task EnqueueFileExtractAsync(MeshFile file) + { + await SendJsonMessageAsync(new FileExtractQueueMessage { FileId = file.FileId }); + } + + public async Task SendToPoisonQueueAsync(FileExtractQueueMessage message) + { + await base.SendToPoisonQueueAsync(message); + } + + protected override string QueueName => configuration.FileExtractQueueName; +} diff --git a/src/ServiceLayer.Mesh/Messaging/FileExtractQueueMessage.cs b/src/ServiceLayer.Mesh/Messaging/FileExtractQueueMessage.cs new file mode 100644 index 0000000..e51032a --- /dev/null +++ b/src/ServiceLayer.Mesh/Messaging/FileExtractQueueMessage.cs @@ -0,0 +1,6 @@ +namespace ServiceLayer.Mesh.Messaging; + +public class FileExtractQueueMessage +{ + public required string FileId { get; set; } +} diff --git a/src/ServiceLayer.Mesh/Messaging/FileTransformQueueClient.cs b/src/ServiceLayer.Mesh/Messaging/FileTransformQueueClient.cs new file mode 100644 index 0000000..5639b91 --- /dev/null +++ b/src/ServiceLayer.Mesh/Messaging/FileTransformQueueClient.cs @@ -0,0 +1,25 @@ +using Azure.Storage.Queues; +using Microsoft.Extensions.Logging; +using ServiceLayer.Mesh.Configuration; +using ServiceLayer.Mesh.Models; + +namespace ServiceLayer.Mesh.Messaging; + +public class FileTransformQueueClient( + ILogger logger, + IFileTransformQueueClientConfiguration configuration, + QueueServiceClient queueServiceClient) + : QueueClientBase(logger, queueServiceClient), IFileTransformQueueClient +{ + public async Task EnqueueFileTransformAsync(MeshFile file) + { + await SendJsonMessageAsync(new FileTransformQueueMessage { FileId = file.FileId }); + } + + public async Task SendToPoisonQueueAsync(FileTransformQueueMessage message) + { + await base.SendToPoisonQueueAsync(message); + } + + protected override string QueueName => configuration.FileTransformQueueName; +} diff --git a/src/ServiceLayer.Mesh/Messaging/FileTransformQueueMessage.cs b/src/ServiceLayer.Mesh/Messaging/FileTransformQueueMessage.cs new file mode 100644 index 0000000..1643858 --- /dev/null +++ b/src/ServiceLayer.Mesh/Messaging/FileTransformQueueMessage.cs @@ -0,0 +1,6 @@ +namespace ServiceLayer.Mesh.Messaging; + +public class FileTransformQueueMessage +{ + public required string FileId { get; set; } +} diff --git a/src/ServiceLayer.Mesh/Messaging/IFileExtractQueueClient.cs b/src/ServiceLayer.Mesh/Messaging/IFileExtractQueueClient.cs new file mode 100644 index 0000000..35f1e30 --- /dev/null +++ b/src/ServiceLayer.Mesh/Messaging/IFileExtractQueueClient.cs @@ -0,0 +1,9 @@ +using ServiceLayer.Mesh.Models; + +namespace ServiceLayer.Mesh.Messaging; + +public interface IFileExtractQueueClient +{ + Task EnqueueFileExtractAsync(MeshFile file); + Task SendToPoisonQueueAsync(FileExtractQueueMessage message); +} diff --git a/src/ServiceLayer.Mesh/Messaging/IFileTransformQueueClient.cs b/src/ServiceLayer.Mesh/Messaging/IFileTransformQueueClient.cs new file mode 100644 index 0000000..13ab5f3 --- /dev/null +++ b/src/ServiceLayer.Mesh/Messaging/IFileTransformQueueClient.cs @@ -0,0 +1,9 @@ +using ServiceLayer.Mesh.Models; + +namespace ServiceLayer.Mesh.Messaging; + +public interface IFileTransformQueueClient +{ + Task EnqueueFileTransformAsync(MeshFile file); + Task SendToPoisonQueueAsync(FileTransformQueueMessage message); +} diff --git a/src/ServiceLayer.Mesh/Messaging/QueueClientBase.cs b/src/ServiceLayer.Mesh/Messaging/QueueClientBase.cs new file mode 100644 index 0000000..b443497 --- /dev/null +++ b/src/ServiceLayer.Mesh/Messaging/QueueClientBase.cs @@ -0,0 +1,66 @@ +using System.Text.Json; +using Azure.Storage.Queues; +using Microsoft.Extensions.Logging; + +namespace ServiceLayer.Mesh.Messaging; + +public abstract class QueueClientBase(ILogger logger, QueueServiceClient queueServiceClient) +{ + private QueueClient? _queueClient; + private QueueClient? _poisonQueueClient; + + private QueueClient Client => _queueClient ??= CreateClient(); + private QueueClient PoisonClient => _poisonQueueClient ??= CreatePoisonClient(); + + protected abstract string QueueName { get; } + + private static readonly JsonSerializerOptions QueueJsonOptions = new() + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + PropertyNameCaseInsensitive = true + }; + + private QueueClient CreateClient() + { + var client = queueServiceClient.GetQueueClient(QueueName); + client.CreateIfNotExists(); // TODO - consider environment gating this + return client; + } + + private QueueClient CreatePoisonClient() + { + var poisonQueueName = $"{QueueName}-poison"; + var client = queueServiceClient.GetQueueClient(poisonQueueName); + client.CreateIfNotExists(); // TODO - consider environment gating this + return client; + } + + protected async Task SendJsonMessageAsync(T message) + { + try + { + var json = JsonSerializer.Serialize(message, QueueJsonOptions); + await Client.SendMessageAsync(json); + } + catch (Exception e) + { + // TODO - consider including file ID or correlation ID in error logs + logger.LogError(e, "Error sending message to queue {QueueName}", QueueName); + throw; + } + } + + protected async Task SendToPoisonQueueAsync(T message) + { + try + { + var json = JsonSerializer.Serialize(message, QueueJsonOptions); + await PoisonClient.SendMessageAsync(json); + } + catch (Exception e) + { + logger.LogError(e, "Error sending message to poison queue {PoisonQueueName}", $"{QueueName}-poison"); + throw; + } + } +} diff --git a/src/ServiceLayer.Mesh/Program.cs b/src/ServiceLayer.Mesh/Program.cs index 4ae7a2d..ce2cb7a 100644 --- a/src/ServiceLayer.Mesh/Program.cs +++ b/src/ServiceLayer.Mesh/Program.cs @@ -6,15 +6,21 @@ using Microsoft.EntityFrameworkCore; using NHS.MESH.Client; using ServiceLayer.Mesh.Data; +using Azure.Storage.Blobs; +using ServiceLayer.Mesh.Configuration; +using ServiceLayer.Mesh.Messaging; var host = new HostBuilder() .ConfigureFunctionsWebApplication() .ConfigureServices(services => { + var environment = Environment.GetEnvironmentVariable("ASPNETCORE_ENVIRONMENT"); + var isLocalEnvironment = environment == "Development"; + // MESH Client config services .AddMeshClient(_ => _.MeshApiBaseUrl = Environment.GetEnvironmentVariable("MeshApiBaseUrl")) - .AddMailbox(Environment.GetEnvironmentVariable("BSSMailBox"), new NHS.MESH.Client.Configuration.MailboxConfiguration + .AddMailbox(Environment.GetEnvironmentVariable("NbssMailboxId"), new NHS.MESH.Client.Configuration.MailboxConfiguration { Password = Environment.GetEnvironmentVariable("MeshPassword"), SharedKey = Environment.GetEnvironmentVariable("MeshSharedKey"), @@ -30,28 +36,33 @@ options.UseSqlServer(connectionString); }); - // Register QueueClient as singleton + // Register QueueClients as singletons services.AddSingleton(provider => { - var environment = Environment.GetEnvironmentVariable("ASPNETCORE_ENVIRONMENT"); - - if (environment == "Development") + if (isLocalEnvironment) { - return new QueueClient("UseDevelopmentStorage=true", "my-local-queue"); + var connectionString = Environment.GetEnvironmentVariable("AzureWebJobsStorage"); + return new QueueServiceClient(connectionString); } - else - { - var queueUrl = Environment.GetEnvironmentVariable("QueueUrl"); - if (string.IsNullOrWhiteSpace(queueUrl)) - { - throw new InvalidOperationException("QueueUrl environment variable is not set."); - } + var meshStorageAccountUrl = Environment.GetEnvironmentVariable("MeshStorageAccountUrl"); + return new QueueServiceClient(new Uri(meshStorageAccountUrl), new DefaultAzureCredential()); + }); - var credential = new ManagedIdentityCredential(); - return new QueueClient(new Uri(queueUrl), credential); - } + services.AddSingleton(); + services.AddSingleton(); + + services.AddSingleton(provider => + { + return new BlobContainerClient( + Environment.GetEnvironmentVariable("AzureWebJobsStorage"), + Environment.GetEnvironmentVariable("BlobContainerName")); }); + + services.AddTransient(); + services.AddTransient(); + services.AddTransient(); + services.AddTransient(); }); diff --git a/src/ServiceLayer.Mesh/ServiceLayer.Mesh.csproj b/src/ServiceLayer.Mesh/ServiceLayer.Mesh.csproj index da05f0e..2743812 100644 --- a/src/ServiceLayer.Mesh/ServiceLayer.Mesh.csproj +++ b/src/ServiceLayer.Mesh/ServiceLayer.Mesh.csproj @@ -9,12 +9,14 @@ + + diff --git a/src/ServiceLayer.Mesh/Storage/IMeshFilesBlobStore.cs b/src/ServiceLayer.Mesh/Storage/IMeshFilesBlobStore.cs new file mode 100644 index 0000000..91a61dc --- /dev/null +++ b/src/ServiceLayer.Mesh/Storage/IMeshFilesBlobStore.cs @@ -0,0 +1,12 @@ +using ServiceLayer.Mesh.Models; + +namespace ServiceLayer.Mesh.Storage; + +public interface IMeshFilesBlobStore +{ + // TODO - return a Stream or a byte array? + public Task DownloadAsync(MeshFile file); + + // Mesh client gives us a byte array, hence this not taking a stream. + public Task UploadAsync(MeshFile file, byte[] data); +} diff --git a/src/ServiceLayer.Mesh/Storage/MeshFilesBlobStore.cs b/src/ServiceLayer.Mesh/Storage/MeshFilesBlobStore.cs new file mode 100644 index 0000000..d0b0c21 --- /dev/null +++ b/src/ServiceLayer.Mesh/Storage/MeshFilesBlobStore.cs @@ -0,0 +1,24 @@ +using Azure.Storage.Blobs; +using ServiceLayer.Mesh.Models; + +namespace ServiceLayer.Mesh.Storage; + +public class MeshFilesBlobStore(BlobContainerClient blobContainerClient) : IMeshFilesBlobStore +{ + public Task DownloadAsync(MeshFile file) + { + throw new NotImplementedException(); + } + + public async Task UploadAsync(MeshFile file, byte[] data) + { + var blobPath = $"{file.FileType}/{file.FileId}"; + var blobClient = blobContainerClient.GetBlobClient(blobPath); + + var dataStream = new MemoryStream(data); + + await blobClient.UploadAsync(dataStream, overwrite: true); + + return blobPath; + } +} diff --git a/src/ServiceLayer.sln b/src/ServiceLayer.sln index 74ff958..76d19f3 100644 --- a/src/ServiceLayer.sln +++ b/src/ServiceLayer.sln @@ -13,73 +13,87 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ServiceLayer.Mesh", "Servic EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ServiceLayer.Mesh.Tests", "..\tests\ServiceLayer.Mesh.Tests\ServiceLayer.Mesh.Tests.csproj", "{E5EF5B92-52DA-4EF3-956B-8AEE3D333428}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NHS.Mesh.Client", "dotnet-mesh-client\application\DotNetMeshClient\NHS.Mesh.Client\NHS.Mesh.Client.csproj", "{F373FEBA-AD0B-4E0B-BEE3-31D22C7AD43D}" +EndProject Global - GlobalSection(SolutionConfigurationPlatforms) = preSolution - Debug|Any CPU = Debug|Any CPU - Debug|x64 = Debug|x64 - Debug|x86 = Debug|x86 - Release|Any CPU = Release|Any CPU - Release|x64 = Release|x64 - Release|x86 = Release|x86 - EndGlobalSection - GlobalSection(ProjectConfigurationPlatforms) = postSolution - {B56B41FF-FA39-0FDE-E266-6EC09B268DFB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {B56B41FF-FA39-0FDE-E266-6EC09B268DFB}.Debug|Any CPU.Build.0 = Debug|Any CPU - {B56B41FF-FA39-0FDE-E266-6EC09B268DFB}.Debug|x64.ActiveCfg = Debug|Any CPU - {B56B41FF-FA39-0FDE-E266-6EC09B268DFB}.Debug|x64.Build.0 = Debug|Any CPU - {B56B41FF-FA39-0FDE-E266-6EC09B268DFB}.Debug|x86.ActiveCfg = Debug|Any CPU - {B56B41FF-FA39-0FDE-E266-6EC09B268DFB}.Debug|x86.Build.0 = Debug|Any CPU - {B56B41FF-FA39-0FDE-E266-6EC09B268DFB}.Release|Any CPU.ActiveCfg = Release|Any CPU - {B56B41FF-FA39-0FDE-E266-6EC09B268DFB}.Release|Any CPU.Build.0 = Release|Any CPU - {B56B41FF-FA39-0FDE-E266-6EC09B268DFB}.Release|x64.ActiveCfg = Release|Any CPU - {B56B41FF-FA39-0FDE-E266-6EC09B268DFB}.Release|x64.Build.0 = Release|Any CPU - {B56B41FF-FA39-0FDE-E266-6EC09B268DFB}.Release|x86.ActiveCfg = Release|Any CPU - {B56B41FF-FA39-0FDE-E266-6EC09B268DFB}.Release|x86.Build.0 = Release|Any CPU - {BA052DAE-6FD1-483A-A0AF-DCBCF9E38C72}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {BA052DAE-6FD1-483A-A0AF-DCBCF9E38C72}.Debug|Any CPU.Build.0 = Debug|Any CPU - {BA052DAE-6FD1-483A-A0AF-DCBCF9E38C72}.Debug|x64.ActiveCfg = Debug|Any CPU - {BA052DAE-6FD1-483A-A0AF-DCBCF9E38C72}.Debug|x64.Build.0 = Debug|Any CPU - {BA052DAE-6FD1-483A-A0AF-DCBCF9E38C72}.Debug|x86.ActiveCfg = Debug|Any CPU - {BA052DAE-6FD1-483A-A0AF-DCBCF9E38C72}.Debug|x86.Build.0 = Debug|Any CPU - {BA052DAE-6FD1-483A-A0AF-DCBCF9E38C72}.Release|Any CPU.ActiveCfg = Release|Any CPU - {BA052DAE-6FD1-483A-A0AF-DCBCF9E38C72}.Release|Any CPU.Build.0 = Release|Any CPU - {BA052DAE-6FD1-483A-A0AF-DCBCF9E38C72}.Release|x64.ActiveCfg = Release|Any CPU - {BA052DAE-6FD1-483A-A0AF-DCBCF9E38C72}.Release|x64.Build.0 = Release|Any CPU - {BA052DAE-6FD1-483A-A0AF-DCBCF9E38C72}.Release|x86.ActiveCfg = Release|Any CPU - {BA052DAE-6FD1-483A-A0AF-DCBCF9E38C72}.Release|x86.Build.0 = Release|Any CPU - {803E8A5E-A180-4799-8BE2-DD5BD3C34ED2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {803E8A5E-A180-4799-8BE2-DD5BD3C34ED2}.Debug|Any CPU.Build.0 = Debug|Any CPU - {803E8A5E-A180-4799-8BE2-DD5BD3C34ED2}.Debug|x64.ActiveCfg = Debug|Any CPU - {803E8A5E-A180-4799-8BE2-DD5BD3C34ED2}.Debug|x64.Build.0 = Debug|Any CPU - {803E8A5E-A180-4799-8BE2-DD5BD3C34ED2}.Debug|x86.ActiveCfg = Debug|Any CPU - {803E8A5E-A180-4799-8BE2-DD5BD3C34ED2}.Debug|x86.Build.0 = Debug|Any CPU - {803E8A5E-A180-4799-8BE2-DD5BD3C34ED2}.Release|Any CPU.ActiveCfg = Release|Any CPU - {803E8A5E-A180-4799-8BE2-DD5BD3C34ED2}.Release|Any CPU.Build.0 = Release|Any CPU - {803E8A5E-A180-4799-8BE2-DD5BD3C34ED2}.Release|x64.ActiveCfg = Release|Any CPU - {803E8A5E-A180-4799-8BE2-DD5BD3C34ED2}.Release|x64.Build.0 = Release|Any CPU - {803E8A5E-A180-4799-8BE2-DD5BD3C34ED2}.Release|x86.ActiveCfg = Release|Any CPU - {803E8A5E-A180-4799-8BE2-DD5BD3C34ED2}.Release|x86.Build.0 = Release|Any CPU - {E5EF5B92-52DA-4EF3-956B-8AEE3D333428}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {E5EF5B92-52DA-4EF3-956B-8AEE3D333428}.Debug|Any CPU.Build.0 = Debug|Any CPU - {E5EF5B92-52DA-4EF3-956B-8AEE3D333428}.Debug|x64.ActiveCfg = Debug|Any CPU - {E5EF5B92-52DA-4EF3-956B-8AEE3D333428}.Debug|x64.Build.0 = Debug|Any CPU - {E5EF5B92-52DA-4EF3-956B-8AEE3D333428}.Debug|x86.ActiveCfg = Debug|Any CPU - {E5EF5B92-52DA-4EF3-956B-8AEE3D333428}.Debug|x86.Build.0 = Debug|Any CPU - {E5EF5B92-52DA-4EF3-956B-8AEE3D333428}.Release|Any CPU.ActiveCfg = Release|Any CPU - {E5EF5B92-52DA-4EF3-956B-8AEE3D333428}.Release|Any CPU.Build.0 = Release|Any CPU - {E5EF5B92-52DA-4EF3-956B-8AEE3D333428}.Release|x64.ActiveCfg = Release|Any CPU - {E5EF5B92-52DA-4EF3-956B-8AEE3D333428}.Release|x64.Build.0 = Release|Any CPU - {E5EF5B92-52DA-4EF3-956B-8AEE3D333428}.Release|x86.ActiveCfg = Release|Any CPU - {E5EF5B92-52DA-4EF3-956B-8AEE3D333428}.Release|x86.Build.0 = Release|Any CPU - EndGlobalSection - GlobalSection(SolutionProperties) = preSolution - HideSolutionNode = FALSE - EndGlobalSection - GlobalSection(NestedProjects) = preSolution - {BA052DAE-6FD1-483A-A0AF-DCBCF9E38C72} = {0AB3BF05-4346-4AA6-1389-037BE0695223} - {E5EF5B92-52DA-4EF3-956B-8AEE3D333428} = {0AB3BF05-4346-4AA6-1389-037BE0695223} - EndGlobalSection - GlobalSection(ExtensibilityGlobals) = postSolution - SolutionGuid = {EEE06B13-019F-4618-A6EB-FD834B6EA7D7} - EndGlobalSection + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Debug|x64 = Debug|x64 + Debug|x86 = Debug|x86 + Release|Any CPU = Release|Any CPU + Release|x64 = Release|x64 + Release|x86 = Release|x86 + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {B56B41FF-FA39-0FDE-E266-6EC09B268DFB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {B56B41FF-FA39-0FDE-E266-6EC09B268DFB}.Debug|Any CPU.Build.0 = Debug|Any CPU + {B56B41FF-FA39-0FDE-E266-6EC09B268DFB}.Debug|x64.ActiveCfg = Debug|Any CPU + {B56B41FF-FA39-0FDE-E266-6EC09B268DFB}.Debug|x64.Build.0 = Debug|Any CPU + {B56B41FF-FA39-0FDE-E266-6EC09B268DFB}.Debug|x86.ActiveCfg = Debug|Any CPU + {B56B41FF-FA39-0FDE-E266-6EC09B268DFB}.Debug|x86.Build.0 = Debug|Any CPU + {B56B41FF-FA39-0FDE-E266-6EC09B268DFB}.Release|Any CPU.ActiveCfg = Release|Any CPU + {B56B41FF-FA39-0FDE-E266-6EC09B268DFB}.Release|Any CPU.Build.0 = Release|Any CPU + {B56B41FF-FA39-0FDE-E266-6EC09B268DFB}.Release|x64.ActiveCfg = Release|Any CPU + {B56B41FF-FA39-0FDE-E266-6EC09B268DFB}.Release|x64.Build.0 = Release|Any CPU + {B56B41FF-FA39-0FDE-E266-6EC09B268DFB}.Release|x86.ActiveCfg = Release|Any CPU + {B56B41FF-FA39-0FDE-E266-6EC09B268DFB}.Release|x86.Build.0 = Release|Any CPU + {BA052DAE-6FD1-483A-A0AF-DCBCF9E38C72}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {BA052DAE-6FD1-483A-A0AF-DCBCF9E38C72}.Debug|Any CPU.Build.0 = Debug|Any CPU + {BA052DAE-6FD1-483A-A0AF-DCBCF9E38C72}.Debug|x64.ActiveCfg = Debug|Any CPU + {BA052DAE-6FD1-483A-A0AF-DCBCF9E38C72}.Debug|x64.Build.0 = Debug|Any CPU + {BA052DAE-6FD1-483A-A0AF-DCBCF9E38C72}.Debug|x86.ActiveCfg = Debug|Any CPU + {BA052DAE-6FD1-483A-A0AF-DCBCF9E38C72}.Debug|x86.Build.0 = Debug|Any CPU + {BA052DAE-6FD1-483A-A0AF-DCBCF9E38C72}.Release|Any CPU.ActiveCfg = Release|Any CPU + {BA052DAE-6FD1-483A-A0AF-DCBCF9E38C72}.Release|Any CPU.Build.0 = Release|Any CPU + {BA052DAE-6FD1-483A-A0AF-DCBCF9E38C72}.Release|x64.ActiveCfg = Release|Any CPU + {BA052DAE-6FD1-483A-A0AF-DCBCF9E38C72}.Release|x64.Build.0 = Release|Any CPU + {BA052DAE-6FD1-483A-A0AF-DCBCF9E38C72}.Release|x86.ActiveCfg = Release|Any CPU + {BA052DAE-6FD1-483A-A0AF-DCBCF9E38C72}.Release|x86.Build.0 = Release|Any CPU + {803E8A5E-A180-4799-8BE2-DD5BD3C34ED2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {803E8A5E-A180-4799-8BE2-DD5BD3C34ED2}.Debug|Any CPU.Build.0 = Debug|Any CPU + {803E8A5E-A180-4799-8BE2-DD5BD3C34ED2}.Debug|x64.ActiveCfg = Debug|Any CPU + {803E8A5E-A180-4799-8BE2-DD5BD3C34ED2}.Debug|x64.Build.0 = Debug|Any CPU + {803E8A5E-A180-4799-8BE2-DD5BD3C34ED2}.Debug|x86.ActiveCfg = Debug|Any CPU + {803E8A5E-A180-4799-8BE2-DD5BD3C34ED2}.Debug|x86.Build.0 = Debug|Any CPU + {803E8A5E-A180-4799-8BE2-DD5BD3C34ED2}.Release|Any CPU.ActiveCfg = Release|Any CPU + {803E8A5E-A180-4799-8BE2-DD5BD3C34ED2}.Release|Any CPU.Build.0 = Release|Any CPU + {803E8A5E-A180-4799-8BE2-DD5BD3C34ED2}.Release|x64.ActiveCfg = Release|Any CPU + {803E8A5E-A180-4799-8BE2-DD5BD3C34ED2}.Release|x64.Build.0 = Release|Any CPU + {803E8A5E-A180-4799-8BE2-DD5BD3C34ED2}.Release|x86.ActiveCfg = Release|Any CPU + {803E8A5E-A180-4799-8BE2-DD5BD3C34ED2}.Release|x86.Build.0 = Release|Any CPU + {E5EF5B92-52DA-4EF3-956B-8AEE3D333428}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {E5EF5B92-52DA-4EF3-956B-8AEE3D333428}.Debug|Any CPU.Build.0 = Debug|Any CPU + {E5EF5B92-52DA-4EF3-956B-8AEE3D333428}.Debug|x64.ActiveCfg = Debug|Any CPU + {E5EF5B92-52DA-4EF3-956B-8AEE3D333428}.Debug|x64.Build.0 = Debug|Any CPU + {E5EF5B92-52DA-4EF3-956B-8AEE3D333428}.Debug|x86.ActiveCfg = Debug|Any CPU + {E5EF5B92-52DA-4EF3-956B-8AEE3D333428}.Debug|x86.Build.0 = Debug|Any CPU + {E5EF5B92-52DA-4EF3-956B-8AEE3D333428}.Release|Any CPU.ActiveCfg = Release|Any CPU + {E5EF5B92-52DA-4EF3-956B-8AEE3D333428}.Release|Any CPU.Build.0 = Release|Any CPU + {E5EF5B92-52DA-4EF3-956B-8AEE3D333428}.Release|x64.ActiveCfg = Release|Any CPU + {E5EF5B92-52DA-4EF3-956B-8AEE3D333428}.Release|x64.Build.0 = Release|Any CPU + {E5EF5B92-52DA-4EF3-956B-8AEE3D333428}.Release|x86.ActiveCfg = Release|Any CPU + {E5EF5B92-52DA-4EF3-956B-8AEE3D333428}.Release|x86.Build.0 = Release|Any CPU + {F373FEBA-AD0B-4E0B-BEE3-31D22C7AD43D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {F373FEBA-AD0B-4E0B-BEE3-31D22C7AD43D}.Debug|Any CPU.Build.0 = Debug|Any CPU + {F373FEBA-AD0B-4E0B-BEE3-31D22C7AD43D}.Debug|x64.ActiveCfg = Debug|Any CPU + {F373FEBA-AD0B-4E0B-BEE3-31D22C7AD43D}.Debug|x64.Build.0 = Debug|Any CPU + {F373FEBA-AD0B-4E0B-BEE3-31D22C7AD43D}.Debug|x86.ActiveCfg = Debug|Any CPU + {F373FEBA-AD0B-4E0B-BEE3-31D22C7AD43D}.Debug|x86.Build.0 = Debug|Any CPU + {F373FEBA-AD0B-4E0B-BEE3-31D22C7AD43D}.Release|Any CPU.ActiveCfg = Release|Any CPU + {F373FEBA-AD0B-4E0B-BEE3-31D22C7AD43D}.Release|Any CPU.Build.0 = Release|Any CPU + {F373FEBA-AD0B-4E0B-BEE3-31D22C7AD43D}.Release|x64.ActiveCfg = Release|Any CPU + {F373FEBA-AD0B-4E0B-BEE3-31D22C7AD43D}.Release|x64.Build.0 = Release|Any CPU + {F373FEBA-AD0B-4E0B-BEE3-31D22C7AD43D}.Release|x86.ActiveCfg = Release|Any CPU + {F373FEBA-AD0B-4E0B-BEE3-31D22C7AD43D}.Release|x86.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(NestedProjects) = preSolution + {BA052DAE-6FD1-483A-A0AF-DCBCF9E38C72} = {0AB3BF05-4346-4AA6-1389-037BE0695223} + {E5EF5B92-52DA-4EF3-956B-8AEE3D333428} = {0AB3BF05-4346-4AA6-1389-037BE0695223} + EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {EEE06B13-019F-4618-A6EB-FD834B6EA7D7} + EndGlobalSection EndGlobal diff --git a/tests/ServiceLayer.Mesh.Tests/Functions/DiscoveryFunctionTests.cs b/tests/ServiceLayer.Mesh.Tests/Functions/FileDiscoveryFunctionTests.cs similarity index 71% rename from tests/ServiceLayer.Mesh.Tests/Functions/DiscoveryFunctionTests.cs rename to tests/ServiceLayer.Mesh.Tests/Functions/FileDiscoveryFunctionTests.cs index 450f0aa..3ce7373 100644 --- a/tests/ServiceLayer.Mesh.Tests/Functions/DiscoveryFunctionTests.cs +++ b/tests/ServiceLayer.Mesh.Tests/Functions/FileDiscoveryFunctionTests.cs @@ -1,34 +1,29 @@ -using System; -using System.Linq; -using System.Threading; -using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Logging; using Moq; -using Xunit; -using ServiceLayer.Mesh.Functions; -using ServiceLayer.Mesh.Models; -using ServiceLayer.Mesh.Data; -using Microsoft.EntityFrameworkCore; using NHS.MESH.Client.Contracts.Services; -using Microsoft.Azure.Functions.Worker; using NHS.MESH.Client.Models; -using Azure.Storage.Queues; -using Azure.Storage.Queues.Models; -using Azure; +using ServiceLayer.Mesh.Configuration; +using ServiceLayer.Mesh.Data; +using ServiceLayer.Mesh.Functions; +using ServiceLayer.Mesh.Messaging; +using ServiceLayer.Mesh.Models; + +namespace ServiceLayer.Mesh.Tests.Functions; -public class DiscoveryFunctionTests +public class FileDiscoveryFunctionTests { - private readonly Mock> _loggerMock; + private readonly Mock> _loggerMock; private readonly Mock _meshInboxServiceMock; private readonly ServiceLayerDbContext _dbContext; - private readonly Mock _queueClientMock; - private readonly DiscoveryFunction _function; + private readonly Mock _queueClientMock; + private readonly FileDiscoveryFunction _function; - public DiscoveryFunctionTests() + public FileDiscoveryFunctionTests() { - _loggerMock = new Mock>(); + _loggerMock = new Mock>(); _meshInboxServiceMock = new Mock(); - _queueClientMock = new Mock(); + _queueClientMock = new Mock(); var options = new DbContextOptionsBuilder() .UseInMemoryDatabase(databaseName: Guid.NewGuid().ToString()) @@ -38,11 +33,12 @@ public DiscoveryFunctionTests() _dbContext = new ServiceLayerDbContext(options); - Environment.SetEnvironmentVariable("MailboxId", "test-mailbox"); - Environment.SetEnvironmentVariable("QueueUrl", "https://fakestorageaccount.queue.core.windows.net/testqueue"); + var functionConfiguration = new Mock(); + functionConfiguration.Setup(c => c.NbssMeshMailboxId).Returns("test-mailbox"); - _function = new DiscoveryFunction( + _function = new FileDiscoveryFunction( _loggerMock.Object, + functionConfiguration.Object, _meshInboxServiceMock.Object, _dbContext, _queueClientMock.Object @@ -70,7 +66,8 @@ public async Task Run_AddsNewMessageToDbAndQueue() Assert.Equal(MeshFileStatus.Discovered, meshFile.Status); Assert.Equal("test-mailbox", meshFile.MailboxId); - _queueClientMock.Verify(q => q.SendMessage(It.IsAny()), Times.Once); + // TODO - replace the It.IsAny with a more specific matcher, or use a callback + _queueClientMock.Verify(q => q.EnqueueFileExtractAsync(It.IsAny()), Times.Once); } [Fact] @@ -102,7 +99,7 @@ public async Task Run_DoesNotAddDuplicateMessageOrQueueIt() var count = _dbContext.MeshFiles.Count(f => f.FileId == duplicateMessageId); Assert.Equal(1, count); - _queueClientMock.Verify(q => q.SendMessage(It.IsAny()), Times.Never); + _queueClientMock.Verify(q => q.EnqueueFileExtractAsync(It.IsAny()), Times.Never); } [Fact] @@ -112,7 +109,7 @@ public async Task Run_NoMessagesInInbox_DoesNothing() _meshInboxServiceMock.Setup(s => s.GetMessagesAsync("test-mailbox")) .ReturnsAsync(new MeshResponse { - Response = new CheckInboxResponse { Messages = Array.Empty() } + Response = new CheckInboxResponse { Messages = [] } }); // Act @@ -120,7 +117,7 @@ public async Task Run_NoMessagesInInbox_DoesNothing() // Assert Assert.Empty(_dbContext.MeshFiles); - _queueClientMock.Verify(q => q.SendMessage(It.IsAny()), Times.Never); + _queueClientMock.Verify(q => q.EnqueueFileExtractAsync(It.IsAny()), Times.Never); } [Fact] @@ -147,6 +144,7 @@ public async Task Run_MultipleMessagesInInbox_AllAreProcessed() Assert.Equal("test-mailbox", meshFile.MailboxId); } - _queueClientMock.Verify(q => q.SendMessage(It.IsAny()), Times.Exactly(messageIds.Length)); + // TODO - replace the It.IsAny with more specific matcher, or use a callback to capture the arguments and check the file IDs + _queueClientMock.Verify(q => q.EnqueueFileExtractAsync(It.IsAny()), Times.Exactly(messageIds.Length)); } } diff --git a/tests/ServiceLayer.Mesh.Tests/Functions/FileExtractFunctionTests.cs b/tests/ServiceLayer.Mesh.Tests/Functions/FileExtractFunctionTests.cs new file mode 100644 index 0000000..1468af8 --- /dev/null +++ b/tests/ServiceLayer.Mesh.Tests/Functions/FileExtractFunctionTests.cs @@ -0,0 +1,315 @@ +using System.Globalization; +using Google.Protobuf.WellKnownTypes; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Logging; +using Moq; +using NHS.MESH.Client.Contracts.Services; +using NHS.MESH.Client.Models; +using ServiceLayer.Mesh.Configuration; +using ServiceLayer.Mesh.Data; +using ServiceLayer.Mesh.Functions; +using ServiceLayer.Mesh.Messaging; +using ServiceLayer.Mesh.Models; +using ServiceLayer.Mesh.Storage; + +namespace ServiceLayer.Mesh.Tests.Functions; + +public class FileExtractFunctionTests +{ + private readonly Mock> _loggerMock; + private readonly Mock _meshInboxServiceMock; + private readonly Mock _fileTransformQueueClientMock; + private readonly Mock _fileExtractQueueClientMock; + private readonly Mock _configurationMock; + private readonly Mock _blobStoreMock; + private readonly ServiceLayerDbContext _dbContext; + private readonly FileExtractFunction _function; + + public FileExtractFunctionTests() + { + _loggerMock = new Mock>(); + _meshInboxServiceMock = new Mock(); + _fileExtractQueueClientMock = new Mock(); + _fileTransformQueueClientMock = new Mock(); + _blobStoreMock = new Mock(); + _configurationMock = new Mock(); + + var options = new DbContextOptionsBuilder() + .UseInMemoryDatabase(Guid.NewGuid().ToString()) + .ConfigureWarnings(warnings => + warnings.Ignore(Microsoft.EntityFrameworkCore.Diagnostics.InMemoryEventId.TransactionIgnoredWarning)) + .Options; + + _dbContext = new ServiceLayerDbContext(options); + + var functionConfiguration = new Mock(); + functionConfiguration.Setup(c => c.NbssMeshMailboxId).Returns("test-mailbox"); + + _function = new FileExtractFunction( + _loggerMock.Object, + functionConfiguration.Object, + _meshInboxServiceMock.Object, + _dbContext, + _fileTransformQueueClientMock.Object, + _fileExtractQueueClientMock.Object, + _blobStoreMock.Object + ); + } + + [Fact] + public async Task Run_FileNotFound_ExitsSilently() + { + // Arrange + var message = new FileExtractQueueMessage { FileId = "nonexistent-file" }; + + // Act + await _function.Run(message); + + // Assert + _loggerMock.Verify( + x => x.Log( + LogLevel.Warning, + It.IsAny(), + It.Is((v, t) => v.ToString() == $"File with id: {message.FileId} not found in MeshFiles table."), + null, + It.IsAny>() + ), Times.Once); + + Assert.Equal(0, _dbContext.MeshFiles.Count()); + _meshInboxServiceMock.Verify(x => x.GetHeadMessageByIdAsync(It.IsAny(), It.IsAny()), Times.Never); + _blobStoreMock.Verify(x => x.UploadAsync(It.IsAny(), It.IsAny()), Times.Never); + _fileTransformQueueClientMock.Verify(x => x.EnqueueFileTransformAsync(It.IsAny()), Times.Never); + _fileTransformQueueClientMock.Verify(x => x.SendToPoisonQueueAsync(It.IsAny()), Times.Never); + } + + [Fact] + public async Task Run_FileStatusInvalid_ExitsSilently() + { + // Arrange + var file = new MeshFile + { + FileType = MeshFileType.NbssAppointmentEvents, + MailboxId = "test-mailbox", + FileId = "file-1", + Status = MeshFileStatus.Transforming, // Not eligible + LastUpdatedUtc = DateTime.UtcNow + }; + _dbContext.MeshFiles.Add(file); + await _dbContext.SaveChangesAsync(); + + var message = new FileExtractQueueMessage { FileId = "file-1" }; + + // Act + await _function.Run(message); + + // Assert + _loggerMock.Verify( + x => x.Log( + LogLevel.Warning, + It.IsAny(), + It.Is((v, t) => v.ToString() == $"File with id: {message.FileId} found in MeshFiles table but is not suitable for extraction. Status: {file.Status}, LastUpdatedUtc: {file.LastUpdatedUtc.ToTimestamp()}."), + null, + It.IsAny>() + ), Times.Once); + + _meshInboxServiceMock.Verify(x => x.GetHeadMessageByIdAsync(It.IsAny(), It.IsAny()), Times.Never); + _blobStoreMock.Verify(x => x.UploadAsync(It.IsAny(), It.IsAny()), Times.Never); + _fileTransformQueueClientMock.Verify(x => x.EnqueueFileTransformAsync(It.IsAny()), Times.Never); + _fileTransformQueueClientMock.Verify(x => x.SendToPoisonQueueAsync(It.IsAny()), Times.Never); + } + + [Fact] + public async Task Run_FileStatusExtractingButNotTimedOut_ExitsSilently() + { + // Arrange + var file = new MeshFile + { + FileType = MeshFileType.NbssAppointmentEvents, + MailboxId = "test-mailbox", + FileId = "file-2", + Status = MeshFileStatus.Extracting, + LastUpdatedUtc = DateTime.UtcNow // Not timed out + }; + _dbContext.MeshFiles.Add(file); + await _dbContext.SaveChangesAsync(); + + var message = new FileExtractQueueMessage { FileId = "file-2" }; + + // Act + await _function.Run(message); + + // Assert + _loggerMock.Verify( + x => x.Log( + LogLevel.Warning, + It.IsAny(), + It.Is((v, t) => v.ToString() == $"File with id: {message.FileId} found in MeshFiles table but is not suitable for extraction. Status: {file.Status}, LastUpdatedUtc: {file.LastUpdatedUtc.ToTimestamp()}."), + null, + It.IsAny>() + ), Times.Once); + + _meshInboxServiceMock.Verify(x => x.GetHeadMessageByIdAsync(It.IsAny(), It.IsAny()), Times.Never); + _blobStoreMock.Verify(x => x.UploadAsync(It.IsAny(), It.IsAny()), Times.Never); + _fileTransformQueueClientMock.Verify(x => x.EnqueueFileTransformAsync(It.IsAny()), Times.Never); + _fileTransformQueueClientMock.Verify(x => x.SendToPoisonQueueAsync(It.IsAny()), Times.Never); + } + + [Fact] + public async Task Run_FileValid_FileUploadedToBlobAndAcknowledgedAndEnqueued() + { + // Arrange + var originalLastUpdatedUtc = DateTime.UtcNow.AddHours(-1); + var file = new MeshFile + { + FileType = MeshFileType.NbssAppointmentEvents, + MailboxId = "test-mailbox", + FileId = "file-3", + Status = MeshFileStatus.Discovered, + LastUpdatedUtc = originalLastUpdatedUtc + }; + _dbContext.MeshFiles.Add(file); + await _dbContext.SaveChangesAsync(); + + var content = new byte[] { 1, 2, 3 }; + const string blobPath = "directory/fileName"; + + _meshInboxServiceMock.Setup(s => s.GetMessageByIdAsync(file.MailboxId, file.FileId)) + .ReturnsAsync(new MeshResponse + { + IsSuccessful = true, + Response = new GetMessageResponse + { + FileAttachment = new FileAttachment { Content = content } + } + }); + _blobStoreMock.Setup(s => s.UploadAsync(file, content)).ReturnsAsync(blobPath); + _meshInboxServiceMock.Setup(s => s.AcknowledgeMessageByIdAsync(file.MailboxId, file.FileId)) + .ReturnsAsync(new MeshResponse + { + IsSuccessful = true + }); + + var message = new FileExtractQueueMessage { FileId = file.FileId }; + + // Act + await _function.Run(message); + + // Assert + _blobStoreMock.Verify(b => b.UploadAsync(It.Is(f => f.FileId == file.FileId), content), Times.Once); + _meshInboxServiceMock.Verify(m => m.AcknowledgeMessageByIdAsync(file.MailboxId, file.FileId), Times.Once); + _fileTransformQueueClientMock.Verify(q => q.EnqueueFileTransformAsync(file), Times.Once); + var updatedFile = _dbContext.MeshFiles.First(); + Assert.Equal(blobPath, updatedFile.BlobPath); + Assert.Equal(MeshFileStatus.Extracted, updatedFile.Status); + Assert.True(updatedFile.LastUpdatedUtc > originalLastUpdatedUtc); + } + + [Fact] + public async Task Run_GetMessageFails_ErrorLoggedAndFileSentToPoisonQueue() + { + // Arrange + var fileId = "file-4"; + var originalLastUpdatedUtc = DateTime.UtcNow.AddHours(-1); + var file = new MeshFile + { + FileType = MeshFileType.NbssAppointmentEvents, + MailboxId = "test-mailbox", + FileId = fileId, + Status = MeshFileStatus.Discovered, + LastUpdatedUtc = originalLastUpdatedUtc + }; + _dbContext.MeshFiles.Add(file); + await _dbContext.SaveChangesAsync(); + + _meshInboxServiceMock.Setup(s => s.GetMessageByIdAsync(file.MailboxId, fileId)) + .ReturnsAsync(new MeshResponse + { + IsSuccessful = false + }); + + var message = new FileExtractQueueMessage { FileId = fileId }; + + // Act + await _function.Run(message); + + // Assert + _loggerMock.Verify( + x => x.Log( + LogLevel.Error, + It.IsAny(), + It.Is((v, t) => v.ToString() == $"An exception occurred during file extraction for fileId: {fileId}"), + It.Is(e => e.Message.StartsWith("Mesh extraction failed: ")), + It.IsAny>() + ), Times.Once); + _blobStoreMock.Verify(b => b.UploadAsync(It.IsAny(), It.IsAny()), Times.Never); + _meshInboxServiceMock.Verify(m => m.AcknowledgeMessageByIdAsync(file.MailboxId, file.FileId), Times.Never); + _fileTransformQueueClientMock.Verify(q => q.EnqueueFileTransformAsync(It.IsAny()), Times.Never); + _fileExtractQueueClientMock.Verify(q => q.SendToPoisonQueueAsync(message), Times.Once); + var updatedFile = _dbContext.MeshFiles.First(); + Assert.Null(updatedFile.BlobPath); + Assert.Equal(MeshFileStatus.FailedExtract, updatedFile.Status); + Assert.True(updatedFile.LastUpdatedUtc > originalLastUpdatedUtc); + } + + [Fact] + public async Task Run_AcknowledgeMessageFails_WarningLoggedAndProcessingContinuesAsNormal() + { + // Arrange + var fileId = "file-4"; + var originalLastUpdatedUtc = DateTime.UtcNow.AddHours(-1); + var file = new MeshFile + { + FileType = MeshFileType.NbssAppointmentEvents, + MailboxId = "test-mailbox", + FileId = fileId, + Status = MeshFileStatus.Discovered, + LastUpdatedUtc = originalLastUpdatedUtc + }; + _dbContext.MeshFiles.Add(file); + await _dbContext.SaveChangesAsync(); + + var content = new byte[] { 1, 2, 3 }; + const string blobPath = "directory/fileName"; + + _meshInboxServiceMock.Setup(s => s.GetMessageByIdAsync(file.MailboxId, fileId)) + .ReturnsAsync(new MeshResponse + { + IsSuccessful = true, + Response = new GetMessageResponse + { + FileAttachment = new FileAttachment { Content = content } + } + }); + _blobStoreMock.Setup(s => s.UploadAsync(file, content)).ReturnsAsync("directory/fileName"); + _meshInboxServiceMock.Setup(s => s.AcknowledgeMessageByIdAsync(file.MailboxId, file.FileId)) + .ReturnsAsync(new MeshResponse + { + IsSuccessful = false + }); + + var message = new FileExtractQueueMessage { FileId = fileId }; + + // Act + await _function.Run(message); + + // Assert + _loggerMock.Verify( + x => x.Log( + LogLevel.Warning, + It.IsAny(), + It.Is((v, t) => + v.ToString().StartsWith("Mesh acknowledgement failed: ") && + v.ToString().EndsWith("This is not a fatal error so processing will continue.")), + null, + It.IsAny>() + ), Times.Once); + _blobStoreMock.Verify(b => b.UploadAsync(file, content), Times.Once); + _meshInboxServiceMock.Verify(m => m.AcknowledgeMessageByIdAsync(file.MailboxId, file.FileId), Times.Once); + _fileTransformQueueClientMock.Verify(q => q.EnqueueFileTransformAsync(file), Times.Once); + _fileExtractQueueClientMock.Verify(q => q.SendToPoisonQueueAsync(message), Times.Never); + var updatedFile = _dbContext.MeshFiles.First(); + Assert.Equal(blobPath, updatedFile.BlobPath); + Assert.Equal(MeshFileStatus.Extracted, updatedFile.Status); + Assert.True(updatedFile.LastUpdatedUtc > originalLastUpdatedUtc); + } +}