feat: DTOSS-9084 - Docker container for service layer mesh project

.env.example

@@ -19,17 +19,20 @@ QueueUrl=http://127.0.0.1:10001
 FileExtractQueueName=file-extract
 FileTransformQueueName=file-transform
 StaleHours=12
+BlobContainerName=incoming-mesh-files
+
 
 # API Configuration
 API_PORT=7071
+MESH_INGEST_PORT=7072
 
 # Event Grid Configuration
 EVENT_GRID_TOPIC_URL=https://localhost:60101/api/events
 EVENT_GRID_TOPIC_KEY=TheLocal+DevelopmentKey=
 
 # Azurite Configuration
 AZURITE_ACCOUNT_KEY=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==
-AZURITE_CONNECTION_STRING=DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;QueueEndpoint=http://127.0.0.1:10001/devstoreaccount1;TableEndpoint=http://127.0.0.1:10002/devstoreaccount1
+AZURITE_CONNECTION_STRING=DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azurite:10000/devstoreaccount1;QueueEndpoint=http://azurite:10001/devstoreaccount1;TableEndpoint=http://azurite/devstoreaccount1
 AZURITE_BLOB_PORT=10000
 AZURITE_QUEUE_PORT=10001
 AZURITE_TABLE_PORT=10002

compose.yaml

@@ -8,15 +8,59 @@ services:
     restart: always
     environment:
       FUNCTIONS_WORKER_RUNTIME: "dotnet-isolated"
-      AzureWebJobsStorage: "${AZURE_WEB_JOBS_STORAGE}"
+      AzureWebJobsStorage: "${AZURITE_CONNECTION_STRING}"
       AzureWebJobsSecretStorageType: "files"
       DatabaseConnectionString: "${DatabaseConnectionString}"
       EVENT_GRID_TOPIC_URL: "${EVENT_GRID_TOPIC_URL}"
       EVENT_GRID_TOPIC_KEY: "${EVENT_GRID_TOPIC_KEY}"
+      ASPNETCORE_URLS: "http://0.0.0.0:8080"
     ports:
-      - "${API_PORT}:80"
+      - "${API_PORT}:8080"
     healthcheck:
-      test: ["CMD-SHELL", "curl -f http://localhost:80/api/health || exit 1"]
+      test: ["CMD-SHELL", "curl -f http://localhost:8080/api/health || exit 1"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
+      start_period: 40s
+    depends_on:
+      azurite:
+        condition: service_healthy
+      db:
+        condition: service_healthy
+    networks:
+      - backend
+
+  mesh-ingest:
+    container_name: "mesh-ingest"
+    build:
+      context: ./Src
+      dockerfile: ServiceLayer.MESH/Dockerfile
+    platform: linux/amd64
+    restart: always
+    environment:
+      FUNCTIONS_WORKER_RUNTIME: "dotnet-isolated"
+      AzureWebJobsStorage: "${AZURITE_CONNECTION_STRING}"
+      AzureWebJobsSecretStorageType: "files"
+      DatabaseConnectionString: "${DatabaseConnectionString}"
+      FileDiscoveryTimerExpression: "${FileDiscoveryTimerExpression}"
+      MeshHandshakeTimerExpression: "${MeshHandshakeTimerExpression}"
+      FileRetryTimerExpression: "${FileRetryTimerExpression}"
+      FileExtractQueueName: "${FileExtractQueueName}"
+      FileTransformQueueName: "${FileTransformQueueName}"
+      StaleHours: "${StaleHours}"
+      MeshApiBaseUrl: "http://mesh_sandbox:80/messageexchange"
+      NbssMailboxId: "${NbssMailboxId}"
+      MeshPassword: "${MeshPassword}"
+      MeshSharedKey: "${MeshSharedKey}"
+      AZURITE_CONNECTION_STRING: "${AZURITE_CONNECTION_STRING}"
+      MeshStorageAccountUrl: "${AZURITE_CONNECTION_STRING}"
+      ASPNETCORE_ENVIRONMENT: "${ASPNETCORE_ENVIRONMENT}"
+      ASPNETCORE_URLS: "http://0.0.0.0:8080"
+      BlobContainerName: "${BlobContainerName}"
+    ports:
+      - "${MESH_INGEST_PORT}:8080"
+    healthcheck:
+      test: ["CMD-SHELL", "curl -f http://localhost:8080/api/health || exit 1"]
       interval: 30s
       timeout: 10s
       retries: 3

src/ServiceLayer.Common/EnvironmentVariables.cs

@@ -0,0 +1,22 @@
+namespace ServiceLayer.Common;
+
+public static class EnvironmentVariables
+{
+    /// <summary>
+    /// Gets an environment variable by name. Throws if not found.
+    /// </summary>
+    /// <param name="key">The name of the environment variable.</param>
+    /// <returns>The value of the environment variable.</returns>
+    /// <exception cref="InvalidOperationException">Thrown when the variable is not found or is empty.</exception>
+    public static string GetRequired(string key)
+    {
+        var value = Environment.GetEnvironmentVariable(key);
+
+        if (string.IsNullOrEmpty(value))
+        {
+            throw new InvalidOperationException($"Environment variable '{key}' is not set or is empty.");
+        }
+
+        return value;
+    }
+}

src/ServiceLayer.Mesh/Configuration/AppConfiguration.cs

@@ -1,3 +1,5 @@
+using ServiceLayer.Common;
+
 namespace ServiceLayer.Mesh.Configuration;
 
 public class AppConfiguration :
@@ -19,12 +21,7 @@ public class AppConfiguration :
 
     private static string GetRequired(string key)
     {
-        var value = Environment.GetEnvironmentVariable(key);
-
-        if (string.IsNullOrEmpty(value))
-        {
-            throw new InvalidOperationException($"Environment variable '{key}' is not set or is empty.");
-        }
+        var value = EnvironmentVariables.GetRequired(key);
 
         return value;
     }

src/ServiceLayer.Mesh/DockerFile

@@ -0,0 +1,23 @@
+FROM mcr.microsoft.com/dotnet/sdk:9.0 AS installer-env
+WORKDIR /src/dotnet-function-app
+
+COPY ./dotnet-mesh-client/application/DotNetMeshClient/NHS.Mesh.Client/NHS.Mesh.Client.csproj ../dotnet-mesh-client/application/DotNetMeshClient/NHS.Mesh.Client/
+COPY ./ServiceLayer.Common/ServiceLayer.Common.csproj ../ServiceLayer.Common/
+COPY ./ServiceLayer.Mesh/ServiceLayer.Mesh.csproj .
+RUN dotnet restore
+
+COPY ./dotnet-mesh-client/ ../dotnet-mesh-client/
+COPY ./ServiceLayer.Common/ ../ServiceLayer.Common/
+COPY ./ServiceLayer.Mesh/ .
+
+RUN dotnet publish -c Release -o /home/site/wwwroot
+
+FROM mcr.microsoft.com/azure-functions/dotnet-isolated:4-dotnet-isolated9.0 AS production
+ENV AzureWebJobsScriptRoot=/home/site/wwwroot \
+    AzureFunctionsJobHost__Logging__Console__IsEnabled=true \
+    ASPNETCORE_ENVIRONMENT=Production
+
+RUN addgroup --system appgroup && adduser --system --ingroup appgroup appuser
+USER appuser
+
+COPY --from=installer-env ["/home/site/wwwroot", "/home/site/wwwroot"]

src/ServiceLayer.Mesh/Functions/FileDiscoveryFunction.cs

@@ -7,52 +7,51 @@
 using ServiceLayer.Mesh.Configuration;
 using ServiceLayer.Mesh.Messaging;
 
-namespace ServiceLayer.Mesh.Functions
+namespace ServiceLayer.Mesh.Functions;
+
+public class FileDiscoveryFunction(
+    ILogger<FileDiscoveryFunction> logger,
+    IFileDiscoveryFunctionConfiguration configuration,
+    IMeshInboxService meshInboxService,
+    ServiceLayerDbContext serviceLayerDbContext,
+    IFileExtractQueueClient fileExtractQueueClient)
 {
-    public class FileDiscoveryFunction(
-        ILogger<FileDiscoveryFunction> logger,
-        IFileDiscoveryFunctionConfiguration configuration,
-        IMeshInboxService meshInboxService,
-        ServiceLayerDbContext serviceLayerDbContext,
-        IFileExtractQueueClient fileExtractQueueClient)
+    [Function("FileDiscoveryFunction")]
+    public async Task Run([TimerTrigger("%FileDiscoveryTimerExpression%")] TimerInfo myTimer)
     {
-        [Function("FileDiscoveryFunction")]
-        public async Task Run([TimerTrigger("%FileDiscoveryTimerExpression%")] TimerInfo myTimer)
+        logger.LogInformation("{functionName} started at: {time}", nameof(FileDiscoveryFunction), DateTime.UtcNow);
+
+        var response = await meshInboxService.GetMessagesAsync(configuration.NbssMeshMailboxId);
+
+        foreach (var messageId in response.Response.Messages)
         {
-            logger.LogInformation("{functionName} started at: {time}", nameof(FileDiscoveryFunction), DateTime.UtcNow);
+            await using var transaction = await serviceLayerDbContext.Database.BeginTransactionAsync();
 
-            var response = await meshInboxService.GetMessagesAsync(configuration.NbssMeshMailboxId);
+            var existing = await serviceLayerDbContext.MeshFiles
+                .AnyAsync(f => f.FileId == messageId);
 
-            foreach (var messageId in response.Response.Messages)
+            if (!existing)
             {
-                await using var transaction = await serviceLayerDbContext.Database.BeginTransactionAsync();
+                var file = new MeshFile
+                {
+                    FileId = messageId,
+                    FileType = MeshFileType.NbssAppointmentEvents,
+                    MailboxId = configuration.NbssMeshMailboxId,
+                    Status = MeshFileStatus.Discovered,
+                    FirstSeenUtc = DateTime.UtcNow,
+                    LastUpdatedUtc = DateTime.UtcNow
+                };
 
-                var existing = await serviceLayerDbContext.MeshFiles
-                    .AnyAsync(f => f.FileId == messageId);
+                serviceLayerDbContext.MeshFiles.Add(file);
 
-                if (!existing)
-                {
-                    var file = new MeshFile
-                    {
-                        FileId = messageId,
-                        FileType = MeshFileType.NbssAppointmentEvents,
-                        MailboxId = configuration.NbssMeshMailboxId,
-                        Status = MeshFileStatus.Discovered,
-                        FirstSeenUtc = DateTime.UtcNow,
-                        LastUpdatedUtc = DateTime.UtcNow
-                    };
-
-                    serviceLayerDbContext.MeshFiles.Add(file);
-
-                    await serviceLayerDbContext.SaveChangesAsync();
-                    await transaction.CommitAsync();
-
-                    await fileExtractQueueClient.EnqueueFileExtractAsync(file);
-                }
-                else
-                {
-                    await transaction.RollbackAsync();
-                }
+                await serviceLayerDbContext.SaveChangesAsync();
+                await transaction.CommitAsync();
+
+                await fileExtractQueueClient.EnqueueFileExtractAsync(file);
+            }
+            else
+            {
+                await transaction.RollbackAsync();
             }
         }
     }

src/ServiceLayer.Mesh/Functions/FileExtractFunction.cs

@@ -23,17 +23,12 @@ public class FileExtractFunction(
     [Function("FileExtractFunction")]
     public async Task Run([QueueTrigger("%FileExtractQueueName%")] FileExtractQueueMessage message)
     {
-        logger.LogInformation("{functionName} started at: {time}", nameof(FileDiscoveryFunction), DateTime.UtcNow);
+        logger.LogInformation("{functionName} started.", nameof(FileExtractFunction));
 
         await using var transaction = await serviceLayerDbContext.Database.BeginTransactionAsync();
 
         var file = await GetFileAsync(message.FileId);
-        if (file == null)
-        {
-            return;
-        }
-
-        if (!IsFileSuitableForExtraction(file))
+        if (file == null || !IsFileSuitableForExtraction(file))
         {
             return;
         }

src/ServiceLayer.Mesh/Functions/FileRetryFunction.cs

@@ -22,9 +22,8 @@ public async Task Run([TimerTrigger("%FileRetryTimerExpression%")] TimerInfo myT
 
         var staleDateTimeUtc = DateTime.UtcNow.AddHours(-configuration.StaleHours);
 
-        await Task.WhenAll(
-            RetryStaleExtractions(staleDateTimeUtc),
-            RetryStaleTransformations(staleDateTimeUtc));
+        await RetryStaleExtractions(staleDateTimeUtc);
+        await RetryStaleTransformations(staleDateTimeUtc);
     }
 
     private async Task RetryStaleExtractions(DateTime staleDateTimeUtc)

src/ServiceLayer.Mesh/Functions/FileTransformFunction.cs

@@ -44,7 +44,8 @@ public async Task Run([QueueTrigger("%FileTransformQueueName%")] FileTransformQu
         var parsedfile = fileParser.Parse(fileContent);
 
         // TODO - take dependency on IEnumerable<IFileTransformer>.
-        // After initial common checks against database, find the appropriate implementation of IFileTransformer to handle the functionality that differs between file type.
+        // After initial common checks against database, find the appropriate implementation of IFileTransformer
+        // to handle the functionality that differs between file types.
     }
 
     private async Task UpdateFileStatusForTransformation(MeshFile file)

src/ServiceLayer.Mesh/Program.cs

@@ -8,45 +8,52 @@
 using ServiceLayer.Mesh.Configuration;
 using ServiceLayer.Mesh.Messaging;
 using ServiceLayer.Data;
+using ServiceLayer.Mesh.Storage;
+using ServiceLayer.Common;
 using ServiceLayer.Mesh.FileTypes.NbssAppointmentEvents;
 
 var host = new HostBuilder()
     .ConfigureFunctionsWebApplication()
     .ConfigureServices(services =>
     {
-        var environment = Environment.GetEnvironmentVariable("ASPNETCORE_ENVIRONMENT");
+        var environment = EnvironmentVariables.GetRequired("ASPNETCORE_ENVIRONMENT");
         var isLocalEnvironment = environment == "Development";
 
         // MESH Client config
         services
-            .AddMeshClient(_ => _.MeshApiBaseUrl = Environment.GetEnvironmentVariable("MeshApiBaseUrl"))
-            .AddMailbox(Environment.GetEnvironmentVariable("NbssMailboxId"), new NHS.MESH.Client.Configuration.MailboxConfiguration
+            .AddMeshClient(_ => _.MeshApiBaseUrl = EnvironmentVariables.GetRequired("MeshApiBaseUrl"))
+            .AddMailbox(EnvironmentVariables.GetRequired("NbssMailboxId"), new NHS.MESH.Client.Configuration.MailboxConfiguration
             {
-                Password = Environment.GetEnvironmentVariable("MeshPassword"),
-                SharedKey = Environment.GetEnvironmentVariable("MeshSharedKey"),
+                Password = EnvironmentVariables.GetRequired("MeshPassword"),
+                SharedKey = EnvironmentVariables.GetRequired("MeshSharedKey"),
             }).Build();
 
         // EF Core DbContext
         services.AddDbContext<ServiceLayerDbContext>(options =>
         {
-            var connectionString = Environment.GetEnvironmentVariable("DatabaseConnectionString");
+            var connectionString = EnvironmentVariables.GetRequired("DatabaseConnectionString");
             if (string.IsNullOrEmpty(connectionString))
                 throw new InvalidOperationException("The connection string has not been initialized.");
 
             options.UseSqlServer(connectionString);
         });
 
+        var queueClientOptions = new QueueClientOptions
+        {
+            MessageEncoding = QueueMessageEncoding.Base64
+        };
+
         // Register QueueClients as singletons
         services.AddSingleton(provider =>
         {
             if (isLocalEnvironment)
             {
-                var connectionString = Environment.GetEnvironmentVariable("AzureWebJobsStorage");
-                return new QueueServiceClient(connectionString);
+                var connectionString = EnvironmentVariables.GetRequired("AzureWebJobsStorage");
+                return new QueueServiceClient(connectionString, queueClientOptions);
             }
 
-            var meshStorageAccountUrl = Environment.GetEnvironmentVariable("MeshStorageAccountUrl");
-            return new QueueServiceClient(new Uri(meshStorageAccountUrl), new DefaultAzureCredential());
+            var meshStorageAccountUrl = EnvironmentVariables.GetRequired("MeshStorageAccountUrl");
+            return new QueueServiceClient(new Uri(meshStorageAccountUrl), new DefaultAzureCredential(), queueClientOptions);
         });
 
         services.AddSingleton<IFileExtractQueueClient, FileExtractQueueClient>();
@@ -56,10 +63,12 @@
         services.AddSingleton(provider =>
         {
             return new BlobContainerClient(
-                Environment.GetEnvironmentVariable("AzureWebJobsStorage"),
-                Environment.GetEnvironmentVariable("BlobContainerName"));
+                EnvironmentVariables.GetRequired("AzureWebJobsStorage"),
+                EnvironmentVariables.GetRequired("BlobContainerName"));
         });
 
+        services.AddSingleton<IMeshFilesBlobStore, MeshFilesBlobStore>();
+
         services.AddTransient<IFileDiscoveryFunctionConfiguration, AppConfiguration>();
         services.AddTransient<IFileExtractFunctionConfiguration, AppConfiguration>();
         services.AddTransient<IFileExtractQueueClientConfiguration, AppConfiguration>();