feat: DTOSS-8747 Integration test

.gitmodules

@@ -1,5 +1,8 @@
 [submodule "src/dotnet-mesh-client"]
-	# path = src/Shared/dotnet-mesh-client
 	path = src/dotnet-mesh-client
 	url = https://github.com/NHSDigital/dotnet-mesh-client.git
 	branch = main
+[submodule "tests/mesh-sandbox"]
+	path = tests/mesh-sandbox
+	url = https://github.com/NHSDigital/mesh-sandbox
+	branch = main

compose.yaml

@@ -48,7 +48,7 @@ services:
       FileExtractQueueName: "${FileExtractQueueName}"
       FileTransformQueueName: "${FileTransformQueueName}"
       StaleHours: "${StaleHours}"
-      MeshApiBaseUrl: "http://mesh_sandbox:80/messageexchange"
+      MeshApiBaseUrl: "http://mesh-sandbox/messageexchange"
       NbssMailboxId: "${NbssMailboxId}"
       MeshPassword: "${MeshPassword}"
       MeshSharedKey: "${MeshSharedKey}"
@@ -71,6 +71,8 @@ services:
         condition: service_healthy
       db:
         condition: service_healthy
+      mesh-sandbox:
+        condition: service_started
     volumes:
       - mesh-config-data:/azure-functions-host/Secrets/
     networks:
@@ -106,10 +108,12 @@ services:
     ports:
       - "1433:1433"
     user: "root"
-    volumes:
-      - db-data:/var/opt/mssql
     healthcheck:
-      test: ["CMD-SHELL", "pgrep -f sqlservr || exit 1"]
+      test:
+        [
+          "CMD-SHELL",
+          "grep -q 'SQL Server is now ready for client connections' /var/opt/mssql/log/errorlog || exit 1",
+        ]
       interval: 20s
       timeout: 10s
       retries: 6
@@ -134,16 +138,34 @@ services:
     networks:
       - backend
 
+  mesh-sandbox:
+    container_name: mesh-sandbox
+    build: tests/mesh-sandbox/
+    ports:
+      - "8700:80"
+    deploy:
+      restart_policy:
+        condition: on-failure
+        max_attempts: 3
+    environment:
+      - SHARED_KEY=TestKey
+      - SSL=no
+    volumes:
+      # mount a different mailboxes.jsonl to pre created mailboxes
+      - ../mesh-sandbox/src/mesh_sandbox/store/data/mailboxes.jsonl:/app/mesh_sandbox/store/data/mailboxes.jsonl:ro
+      - ../mesh-sandbox/src/mesh_sandbox/test_plugin:/app/mesh_sandbox/plugins:ro
+      # you can mount a directory if you want access the stored messages
+      - ../mesh-sandbox/messages:/tmp/mesh_store
+    networks:
+      - backend
+
 networks:
   backend:
     name: backend-network
     driver: bridge
 volumes:
   azurite-data:
     name: azurite-data
-  db-data:
-    name: db-data
-    driver: local
   mesh-config-data:
     name: mesh-config-data
     driver: local

scripts/tests/unit.sh

@@ -7,7 +7,7 @@ if [[ "${1:-}" == "--no-build" ]]; then
 fi
 
 COVERAGE_DIR="coverage"
-TEST_PROJECTS=$(find tests -name '*.csproj')
+TEST_PROJECTS=$(find tests -name '*.csproj' -not -name "ServiceLayer.IntegrationTests.csproj")
 
 rm -rf "$COVERAGE_DIR"
 mkdir -p "$COVERAGE_DIR"

src/ServiceLayer.Mesh/Functions/HealthCheckFunction.cs

@@ -0,0 +1,14 @@
+using Microsoft.Azure.Functions.Worker;
+using Microsoft.Azure.Functions.Worker.Http;
+using System.Net;
+
+namespace ServiceLayer.Mesh.Functions;
+
+public static class HealthCheckFunction
+{
+    [Function("HealthCheckFunction")]
+    public static HttpResponseData Run([HttpTrigger(AuthorizationLevel.Anonymous, "get", Route = "health")] HttpRequestData req)
+    {
+        return req.CreateResponse(HttpStatusCode.OK);
+    }
+}

tests/ServiceLayer.IntegrationTests/.env.tests

@@ -0,0 +1,35 @@
+# Database
+DATABASE_USER=SA
+DATABASE_PASSWORD=YourStrong@Passw0rd
+DATABASE_NAME=ServiceLayer
+DATABASE_HOST=db
+DatabaseConnectionString=Server=${DATABASE_HOST};Database=${DATABASE_NAME};User Id=${DATABASE_USER};Password=${DATABASE_PASSWORD};TrustServerCertificate=True
+
+# MESH
+MeshSharedKey=TestKey
+MeshPassword=password
+NbssMailboxId=X26ABC1
+MeshApiBaseUrl=http://localhost:8700/messageexchange
+
+# Other
+ASPNETCORE_ENVIRONMENT=Development
+FileDiscoveryTimerExpression=*/5 * * * * * # Every 5 seconds so that the test doesn't have to wait too long
+MeshHandshakeTimerExpression=0 0 0 * * * # Midnight
+FileRetryTimerExpression=0 0 * * * *
+FileExtractQueueName=file-extract
+FileTransformQueueName=file-transform
+StaleHours=12
+MeshBlobContainerName=incoming-mesh-files
+API_PORT=7071
+MESH_INGEST_PORT=7072
+
+# Event Grid
+EVENT_GRID_TOPIC_URL=https://localhost:60101/api/events
+EVENT_GRID_TOPIC_KEY=TheLocal+DevelopmentKey=
+
+# Azurite
+AZURITE_ACCOUNT_KEY=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw== # Standard default Azurite key
+AZURITE_CONNECTION_STRING=DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=${AZURITE_ACCOUNT_KEY};BlobEndpoint=http://azurite:10000/devstoreaccount1;QueueEndpoint=http://azurite:10001/devstoreaccount1;TableEndpoint=http://azurite/devstoreaccount1
+AZURITE_BLOB_PORT=10000
+AZURITE_QUEUE_PORT=10001
+AZURITE_TABLE_PORT=10002

tests/ServiceLayer.IntegrationTests/IntegrationTests.cs

@@ -0,0 +1,207 @@
+using System.Diagnostics;
+using System.Net.Http.Headers;
+using System.Text.Json;
+using System.Text.Json.Serialization;
+using Azure.Storage.Blobs;
+using Azure.Storage.Blobs.Models;
+using Microsoft.EntityFrameworkCore;
+using ServiceLayer.Data;
+using ServiceLayer.TestUtilities;
+
+namespace ServiceLayer.IntegrationTests;
+
+[CollectionDefinition("DockerComposeCollection")]
+public class DockerComposeCollection : ICollectionFixture<DockerComposeFixture>
+{
+}
+
+[Collection("DockerComposeCollection")]
+public class IntegrationTests
+{
+    private readonly string _azuriteAccountKey = Environment.GetEnvironmentVariable("AZURITE_ACCOUNT_KEY")
+        ?? throw new InvalidOperationException($"Environment variable 'AZURITE_ACCOUNT_KEY' is not set.");
+    private readonly string _azuriteAccountName = Environment.GetEnvironmentVariable("AZURITE_ACCOUNT_NAME")
+        ?? throw new InvalidOperationException($"Environment variable 'AZURITE_ACCOUNT_NAME' is not set.");
+    private readonly string _azuriteBlobPort = Environment.GetEnvironmentVariable("AZURITE_BLOB_PORT")
+        ?? throw new InvalidOperationException($"Environment variable 'AZURITE_BLOB_PORT' is not set.");
+    private readonly string _meshIngestPort = Environment.GetEnvironmentVariable("MESH_INGEST_PORT")
+        ?? throw new InvalidOperationException($"Environment variable 'MESH_INGEST_PORT' is not set.");
+    private readonly string _meshSandboxPort = Environment.GetEnvironmentVariable("MESH_SANDBOX_PORT")
+        ?? throw new InvalidOperationException($"Environment variable 'MESH_SANDBOX_PORT' is not set.");
+    private readonly string _meshBlobContainerName = Environment.GetEnvironmentVariable("MESH_BLOB_CONTAINER_NAME")
+        ?? throw new InvalidOperationException($"Environment variable 'MESH_BLOB_CONTAINER_NAME' is not set.");
+    private readonly string _databaseConnectionString = Environment.GetEnvironmentVariable("DATABASE_CONNECTION_STRING")
+        ?? throw new InvalidOperationException($"Environment variable 'DATABASE_CONNECTION_STRING' is not set.");
+
+    [Fact]
+    public async Task FileSentToMeshInbox_FileIsUploadedToBlobContainerAndInsertedIntoDb()
+    {
+        // Arrange
+        await WaitForHealthyService();
+
+        // Act
+        var fileId = await SendFileToMeshInbox("KMK_20250212095121_APPT_87.dat");
+
+        // Wait to allow functions to ingest the file
+        await Task.Delay(45000);
+
+        // Assert
+        Assert.NotNull(fileId);
+        Assert.True(await WasFileUploadedToBlobContainer(fileId));
+        Assert.True(await WasFileInsertedIntoDatabase(fileId));
+    }
+
+    private async Task WaitForHealthyService()
+    {
+        Console.WriteLine("Waiting for Mesh Ingest Service health check to pass...");
+
+        int attemptCounter = 0;
+
+        while (attemptCounter < 10)
+        {
+            var response = await HttpHelper.SendHttpRequestAsync(HttpMethod.Get, $"http://localhost:{_meshIngestPort}/api/health");
+
+            if (response.IsSuccessStatusCode)
+            {
+                Console.WriteLine("Mesh Ingest Service is healthy and ready to start ingesting files.");
+                return;
+            }
+
+            Console.WriteLine("Mesh Ingest Service is unhealthy");
+            attemptCounter++;
+            await Task.Delay(5000);
+        }
+
+        Console.WriteLine("Max attempts reached. Mesh Ingest Service is still unhealthy.");
+        throw new TimeoutException("Timed out waiting on Mesh Ingest Service health check");
+    }
+
+    private async Task<string?> SendFileToMeshInbox(string fileName)
+    {
+        byte[] binaryData = await File.ReadAllBytesAsync($"TestData/{fileName}");
+        var content = new ByteArrayContent(binaryData);
+        content.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream");
+
+        var response = await HttpHelper.SendHttpRequestAsync(
+            HttpMethod.Post,
+            $"http://localhost:{_meshSandboxPort}/messageexchange/X26ABC1/outbox",
+            content,
+            headers =>
+            {
+                headers.Add("Authorization", "NHSMESH X26ABC1:a42f77b9-58de-4b45-b599-2d5bf320b44d:0:202407291437:e3005627136e01706efabcfe72269bc8da3192e90a840ab344ab7f82a39bb5c6");
+                headers.Add("Mex-Filename", fileName);
+                headers.Add("Mex-From", "X26ABC1");
+                headers.Add("Mex-To", "X26ABC1");
+                headers.Add("Mex-Workflowid", "API-DOCS-TEST");
+            }
+        );
+
+        string responseBody = await response.Content.ReadAsStringAsync();
+
+        var responseObject = JsonSerializer.Deserialize<MeshResponse>(responseBody);
+
+        return responseObject?.MessageID;
+    }
+
+    private async Task<bool> WasFileUploadedToBlobContainer(string fileId)
+    {
+        var blobConnectionString = $"DefaultEndpointsProtocol=http;AccountName={_azuriteAccountName};AccountKey={_azuriteAccountKey};BlobEndpoint=http://localhost:{_azuriteBlobPort}/{_azuriteAccountName}";
+
+        var containerClient = new BlobContainerClient(blobConnectionString, _meshBlobContainerName);
+
+        try
+        {
+            var blobClient = containerClient.GetBlobClient($"NbssAppointmentEvents/{fileId}");
+
+            BlobProperties properties = await blobClient.GetPropertiesAsync();
+            return true; // If we get properties, the blob exists
+        }
+        catch (Exception ex)
+        {
+            Console.WriteLine($"An error occurred: {ex.Message}");
+            return false;
+        }
+    }
+
+    private async Task<bool> WasFileInsertedIntoDatabase(string fileId)
+    {
+        var options = new DbContextOptionsBuilder<ServiceLayerDbContext>()
+            .UseSqlServer(_databaseConnectionString)
+            .Options;
+
+        var context = new ServiceLayerDbContext(options);
+
+        return await context.MeshFiles.AnyAsync(x => x.FileId == fileId);
+    }
+
+    public class MeshResponse
+    {
+        [JsonPropertyName("messageID")]
+        public required string MessageID { get; set; }
+    }
+}
+
+public class DockerComposeFixture : IAsyncLifetime
+{
+    public async Task InitializeAsync()
+    {
+        Console.WriteLine("Starting up docker containers...");
+
+        var startInfo = new ProcessStartInfo
+        {
+            FileName = "docker",
+            Arguments = "compose --env-file .env.tests up -d svclyr-mesh-ingest mesh-sandbox azurite db db-migrations",
+            RedirectStandardOutput = true,
+            RedirectStandardError = true,
+            UseShellExecute = false,
+            CreateNoWindow = true
+        };
+
+        using var process = Process.Start(startInfo);
+
+        if (process == null)
+        {
+            throw new Exception("Failed to start the Docker process.");
+        }
+
+        await process.WaitForExitAsync();
+
+        if (process.ExitCode != 0)
+        {
+            throw new Exception($"Docker process started but failed, error: {process.StandardError.ReadToEnd()}");
+        }
+
+        Console.WriteLine("Docker containers successfully started");
+    }
+
+    public async Task DisposeAsync()
+    {
+        Console.WriteLine("Stopping docker containers...");
+
+        var stopInfo = new ProcessStartInfo
+        {
+            FileName = "docker",
+            Arguments = "compose down",
+            RedirectStandardOutput = true,
+            RedirectStandardError = true,
+            UseShellExecute = false,
+            CreateNoWindow = true
+        };
+
+        using var process = Process.Start(stopInfo);
+
+        if (process == null)
+        {
+            throw new Exception("Failed to start the Docker process.");
+        }
+
+        await process.WaitForExitAsync();
+
+        if (process.ExitCode != 0)
+        {
+            throw new Exception($"Docker process started but failed, error: {process.StandardError.ReadToEnd()}");
+        }
+
+        Console.WriteLine("Docker containers stopped");
+    }
+}

tests/ServiceLayer.IntegrationTests/README.md

@@ -0,0 +1,28 @@
+# Integration Tests
+
+The integration tests start up docker containers using [compose.yaml](../../compose.yaml) and provide them with the environment variables defined in [.env.tests](.env.tests). Once the containers are up, the integration tests are executed, after the tests have finished, the containers are stopped.
+
+## How to run the tests
+
+To run the integration tests, you also need to pass the following environment variables as arguments when executing `dotnet test`:
+
+- AZURITE_ACCOUNT_KEY
+- AZURITE_ACCOUNT_NAME
+- AZURITE_BLOB_PORT
+- MESH_INGEST_PORT
+- MESH_SANDBOX_PORT
+- MESH_BLOB_CONTAINER_NAME
+- DATABASE_CONNECTION_STRING
+
+E.g.
+
+```sh
+dotnet test \
+    -e AZURITE_ACCOUNT_KEY="Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==" \
+    -e AZURITE_ACCOUNT_NAME=devstoreaccount1 \
+    -e AZURITE_BLOB_PORT=10000 \
+    -e MESH_INGEST_PORT=7072 \
+    -e MESH_SANDBOX_PORT=8700 \
+    -e MESH_BLOB_CONTAINER_NAME=incoming-mesh-files \
+    -e DATABASE_CONNECTION_STRING="Server=localhost;Database=ServiceLayer;User Id=SA;Password=YourStrong@Passw0rd;TrustServerCertificate=True"
+```