feat: DTOSS-8739 Extract function

.env.example

@@ -1,20 +1,21 @@
 # Database Configuration
 DATABASE_USER=SA
 DATABASE_PASSWORD=YourStrong@Passw0rd
-DATABASE_NAME=PathwayCoordinator
+DATABASE_NAME=ServiceLayer
 DATABASE_HOST=db
 AZURE_WEB_JOBS_STORAGE=UseDevelopmentStorage=true
 DatabaseConnectionString=Server=${DATABASE_HOST};Database=${DATABASE_NAME};User Id=${DATABASE_USER};Password=${DATABASE_PASSWORD};TrustServerCertificate=True
 AzureWebJobsStorage=UseDevelopmentStorage=true
 FUNCTIONS_WORKER_RUNTIME=dotnet-isolated
-MailboxId=X26ABC1
 MeshSharedKey=TestKey
 MeshPassword=password
-BSSMailBox=X26ABC1
+NbssMailboxId=X26ABC1
 MeshApiBaseUrl=http://localhost:8700/messageexchange
 ASPNETCORE_ENVIRONMENT=Development
-DiscoveryTimerExpression=*/5 * * * *
+FileDiscoveryTimerExpression=*/5 * * * *
 QueueUrl=http://127.0.0.1:10001
+FileExtractQueueName=file-extract
+FileTransformQueueName=file-transform
 
 # API Configuration
 API_PORT=7071

src/ServiceLayer.Mesh/Configuration/AppConfiguration.cs

@@ -0,0 +1,26 @@
+namespace ServiceLayer.Mesh.Configuration;
+
+public class AppConfiguration :
+    IFileDiscoveryFunctionConfiguration,
+    IFileExtractFunctionConfiguration,
+    IFileExtractQueueClientConfiguration,
+    IFileTransformQueueClientConfiguration
+{
+    public string NbssMeshMailboxId => GetRequired("NbssMailboxId");
+
+    public string FileExtractQueueName => GetRequired("FileExtractQueueName");
+
+    public string FileTransformQueueName => GetRequired("FileTransformQueueName");
+
+    private static string GetRequired(string key)
+    {
+        var value = Environment.GetEnvironmentVariable(key);
+
+        if (string.IsNullOrEmpty(value))
+        {
+            throw new InvalidOperationException($"Environment variable '{key}' is not set or is empty.");
+        }
+
+        return value;
+    }
+}

src/ServiceLayer.Mesh/Configuration/IFileDiscoveryFunctionConfiguration.cs

@@ -0,0 +1,6 @@
+namespace ServiceLayer.Mesh.Configuration;
+
+public interface IFileDiscoveryFunctionConfiguration
+{
+    string NbssMeshMailboxId { get; }
+}

src/ServiceLayer.Mesh/Configuration/IFileExtractFunctionConfiguration.cs

@@ -0,0 +1,6 @@
+namespace ServiceLayer.Mesh.Configuration;
+
+public interface IFileExtractFunctionConfiguration
+{
+    string NbssMeshMailboxId { get; }
+}

src/ServiceLayer.Mesh/Configuration/IFileExtractQueueClientConfiguration.cs

@@ -0,0 +1,6 @@
+namespace ServiceLayer.Mesh.Configuration;
+
+public interface IFileExtractQueueClientConfiguration
+{
+    string FileExtractQueueName { get; }
+}

src/ServiceLayer.Mesh/Configuration/IFileTransformQueueClientConfiguration.cs

@@ -0,0 +1,6 @@
+namespace ServiceLayer.Mesh.Configuration;
+
+public interface IFileTransformQueueClientConfiguration
+{
+    string FileTransformQueueName { get; }
+}

src/ServiceLayer.Mesh/Data/DesignTimeDbContextFactory.cs

@@ -1,8 +1,7 @@
 using Microsoft.EntityFrameworkCore;
 using Microsoft.EntityFrameworkCore.Design;
-using ServiceLayer.Mesh.Data;
 
-namespace ParticipantManager.API.Data;
+namespace ServiceLayer.Mesh.Data;
 
 public class DesignTimeDbContextFactory : IDesignTimeDbContextFactory<ServiceLayerDbContext>
 {

src/ServiceLayer.Mesh/Functions/FileDiscoveryFunction.cs

@@ -0,0 +1,59 @@
+using Microsoft.Azure.Functions.Worker;
+using Microsoft.EntityFrameworkCore;
+using Microsoft.Extensions.Logging;
+using NHS.MESH.Client.Contracts.Services;
+using ServiceLayer.Mesh.Configuration;
+using ServiceLayer.Mesh.Data;
+using ServiceLayer.Mesh.Messaging;
+using ServiceLayer.Mesh.Models;
+
+namespace ServiceLayer.Mesh.Functions
+{
+    public class FileDiscoveryFunction(
+        ILogger<FileDiscoveryFunction> logger,
+        IFileDiscoveryFunctionConfiguration configuration,
+        IMeshInboxService meshInboxService,
+        ServiceLayerDbContext serviceLayerDbContext,
+        IFileExtractQueueClient fileExtractQueueClient)
+    {
+        [Function("FileDiscoveryFunction")]
+        public async Task Run([TimerTrigger("%FileDiscoveryTimerExpression%")] TimerInfo myTimer)
+        {
+            logger.LogInformation("{functionName} started at: {time}", nameof(FileDiscoveryFunction), DateTime.UtcNow);
+
+            var response = await meshInboxService.GetMessagesAsync(configuration.NbssMeshMailboxId);
+
+            foreach (var messageId in response.Response.Messages)
+            {
+                await using var transaction = await serviceLayerDbContext.Database.BeginTransactionAsync();
+
+                var existing = await serviceLayerDbContext.MeshFiles
+                    .AnyAsync(f => f.FileId == messageId);
+
+                if (!existing)
+                {
+                    var file = new MeshFile
+                    {
+                        FileId = messageId,
+                        FileType = MeshFileType.NbssAppointmentEvents,
+                        MailboxId = configuration.NbssMeshMailboxId,
+                        Status = MeshFileStatus.Discovered,
+                        FirstSeenUtc = DateTime.UtcNow,
+                        LastUpdatedUtc = DateTime.UtcNow
+                    };
+
+                    serviceLayerDbContext.MeshFiles.Add(file);
+
+                    await serviceLayerDbContext.SaveChangesAsync();
+                    await transaction.CommitAsync();
+
+                    await fileExtractQueueClient.EnqueueFileExtractAsync(file);
+                }
+                else
+                {
+                    await transaction.RollbackAsync();
+                }
+            }
+        }
+    }
+}

src/ServiceLayer.Mesh/Functions/FileExtractFunction.cs

@@ -0,0 +1,124 @@
+using Google.Protobuf.WellKnownTypes;
+using Microsoft.Azure.Functions.Worker;
+using Microsoft.EntityFrameworkCore;
+using Microsoft.Extensions.Logging;
+using NHS.MESH.Client.Contracts.Services;
+using ServiceLayer.Mesh.Configuration;
+using ServiceLayer.Mesh.Data;
+using ServiceLayer.Mesh.Messaging;
+using ServiceLayer.Mesh.Models;
+using ServiceLayer.Mesh.Storage;
+
+namespace ServiceLayer.Mesh.Functions;
+
+public class FileExtractFunction(
+    ILogger<FileExtractFunction> logger,
+    IFileExtractFunctionConfiguration configuration,
+    IMeshInboxService meshInboxService,
+    ServiceLayerDbContext serviceLayerDbContext,
+    IFileTransformQueueClient fileTransformQueueClient,
+    IFileExtractQueueClient fileExtractQueueClient,
+    IMeshFilesBlobStore meshFileBlobStore)
+{
+    [Function("FileExtractFunction")]
+    public async Task Run([QueueTrigger("%FileExtractQueueName%")] FileExtractQueueMessage message)
+    {
+        logger.LogInformation("{functionName} started at: {time}", nameof(FileDiscoveryFunction), DateTime.UtcNow);
+
+        await using var transaction = await serviceLayerDbContext.Database.BeginTransactionAsync();
+
+        var file = await GetFileAsync(message.FileId);
+        if (file == null)
+        {
+            return;
+        }
+
+        if (!IsFileSuitableForExtraction(file))
+        {
+            return;
+        }
+
+        await UpdateFileStatusForExtraction(file);
+        await transaction.CommitAsync();
+
+        try
+        {
+            await ProcessFileExtraction(file, message);
+        }
+        catch (Exception ex)
+        {
+            await HandleExtractionError(file, message, ex);
+        }
+    }
+
+    private async Task<MeshFile?> GetFileAsync(string fileId)
+    {
+        var file = await serviceLayerDbContext.MeshFiles
+            .FirstOrDefaultAsync(f => f.FileId == fileId);
+
+        if (file == null)
+        {
+            logger.LogWarning("File with id: {fileId} not found in MeshFiles table.", fileId);
+        }
+
+        return file;
+    }
+
+    private bool IsFileSuitableForExtraction(MeshFile file)
+    {
+        // We only want to extract files if they are in a Discovered state,
+        // or are in an Extracting state and were last touched over 12 hours ago.
+        var expectedStatuses = new[] { MeshFileStatus.Discovered, MeshFileStatus.Extracting };
+        if (!expectedStatuses.Contains(file.Status) ||
+            (file.Status == MeshFileStatus.Extracting && file.LastUpdatedUtc > DateTime.UtcNow.AddHours(-12)))
+        {
+            logger.LogWarning(
+                "File with id: {fileId} found in MeshFiles table but is not suitable for extraction. Status: {status}, LastUpdatedUtc: {lastUpdatedUtc}.",
+                file.FileId,
+                file.Status,
+                file.LastUpdatedUtc.ToTimestamp());
+            return false;
+        }
+        return true;
+    }
+
+    private async Task UpdateFileStatusForExtraction(MeshFile file)
+    {
+        file.Status = MeshFileStatus.Extracting;
+        file.LastUpdatedUtc = DateTime.UtcNow;
+        await serviceLayerDbContext.SaveChangesAsync();
+    }
+
+    private async Task ProcessFileExtraction(MeshFile file, FileExtractQueueMessage message)
+    {
+        var meshResponse = await meshInboxService.GetMessageByIdAsync(configuration.NbssMeshMailboxId, file.FileId);
+        if (!meshResponse.IsSuccessful)
+        {
+            throw new InvalidOperationException($"Mesh extraction failed: {meshResponse.Error}");
+        }
+
+        var blobPath = await meshFileBlobStore.UploadAsync(file, meshResponse.Response.FileAttachment.Content);
+
+        var meshAcknowledgementResponse = await meshInboxService.AcknowledgeMessageByIdAsync(configuration.NbssMeshMailboxId, message.FileId);
+        if (!meshAcknowledgementResponse.IsSuccessful)
+        {
+            logger.LogWarning("Mesh acknowledgement failed: {error}.\nThis is not a fatal error so processing will continue.", meshAcknowledgementResponse.Error);
+        }
+
+        file.BlobPath = blobPath;
+        file.Status = MeshFileStatus.Extracted;
+        file.LastUpdatedUtc = DateTime.UtcNow;
+        await serviceLayerDbContext.SaveChangesAsync();
+
+        await fileTransformQueueClient.EnqueueFileTransformAsync(file);
+    }
+
+    private async Task HandleExtractionError(MeshFile file, FileExtractQueueMessage message, Exception ex)
+    {
+        logger.LogError(ex, "An exception occurred during file extraction for fileId: {fileId}", message.FileId);
+        file.Status = MeshFileStatus.FailedExtract;
+        file.LastUpdatedUtc = DateTime.UtcNow;
+        await serviceLayerDbContext.SaveChangesAsync();
+        await fileExtractQueueClient.SendToPoisonQueueAsync(message);
+    }
+}

src/ServiceLayer.Mesh/Functions/FileRetryFunction.cs

@@ -0,0 +1,6 @@
+namespace ServiceLayer.Mesh.Functions;
+
+public class FileRetryFunction
+{
+
+}

src/ServiceLayer.Mesh/Functions/FileTransformFunction.cs

@@ -0,0 +1,6 @@
+namespace ServiceLayer.Mesh.Functions;
+
+public class FileTransformFunction
+{
+
+}

src/ServiceLayer.Mesh/Functions/MeshHandshakeFunction.cs

@@ -0,0 +1,6 @@
+namespace ServiceLayer.Mesh.Functions;
+
+public class MeshHandshakeFunction
+{
+
+}

src/ServiceLayer.Mesh/Messaging/FileExtractQueueClient.cs

@@ -0,0 +1,25 @@
+using Azure.Storage.Queues;
+using Microsoft.Extensions.Logging;
+using ServiceLayer.Mesh.Configuration;
+using ServiceLayer.Mesh.Models;
+
+namespace ServiceLayer.Mesh.Messaging;
+
+public class FileExtractQueueClient(
+    ILogger<FileExtractQueueClient> logger,
+    IFileExtractQueueClientConfiguration configuration,
+    QueueServiceClient queueServiceClient)
+    : QueueClientBase(logger, queueServiceClient), IFileExtractQueueClient
+{
+    public async Task EnqueueFileExtractAsync(MeshFile file)
+    {
+        await SendJsonMessageAsync(new FileExtractQueueMessage { FileId = file.FileId });
+    }
+
+    public async Task SendToPoisonQueueAsync(FileExtractQueueMessage message)
+    {
+        await base.SendToPoisonQueueAsync(message);
+    }
+
+    protected override string QueueName => configuration.FileExtractQueueName;
+}

src/ServiceLayer.Mesh/Messaging/FileExtractQueueMessage.cs

@@ -0,0 +1,6 @@
+namespace ServiceLayer.Mesh.Messaging;
+
+public class FileExtractQueueMessage
+{
+    public required string FileId { get; set; }
+}