diff --git a/.env.example b/.env.example index e3478f6..07bec7a 100644 --- a/.env.example +++ b/.env.example @@ -19,9 +19,12 @@ QueueUrl=http://127.0.0.1:10001 FileExtractQueueName=file-extract FileTransformQueueName=file-transform StaleHours=12 +BlobContainerName=incoming-mesh-files + # API Configuration API_PORT=7071 +MESH_INGEST_PORT=7072 # Event Grid Configuration EVENT_GRID_TOPIC_URL=https://localhost:60101/api/events @@ -29,7 +32,7 @@ EVENT_GRID_TOPIC_KEY=TheLocal+DevelopmentKey= # Azurite Configuration AZURITE_ACCOUNT_KEY=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw== -AZURITE_CONNECTION_STRING=DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;QueueEndpoint=http://127.0.0.1:10001/devstoreaccount1;TableEndpoint=http://127.0.0.1:10002/devstoreaccount1 +AZURITE_CONNECTION_STRING=DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azurite:10000/devstoreaccount1;QueueEndpoint=http://azurite:10001/devstoreaccount1;TableEndpoint=http://azurite/devstoreaccount1 AZURITE_BLOB_PORT=10000 AZURITE_QUEUE_PORT=10001 AZURITE_TABLE_PORT=10002 diff --git a/compose.yaml b/compose.yaml index 4663f36..ae785a8 100644 --- a/compose.yaml +++ b/compose.yaml @@ -8,15 +8,59 @@ services: restart: always environment: FUNCTIONS_WORKER_RUNTIME: "dotnet-isolated" - AzureWebJobsStorage: "${AZURE_WEB_JOBS_STORAGE}" + AzureWebJobsStorage: "${AZURITE_CONNECTION_STRING}" AzureWebJobsSecretStorageType: "files" DatabaseConnectionString: "${DatabaseConnectionString}" EVENT_GRID_TOPIC_URL: "${EVENT_GRID_TOPIC_URL}" EVENT_GRID_TOPIC_KEY: "${EVENT_GRID_TOPIC_KEY}" + ASPNETCORE_URLS: "http://0.0.0.0:8080" ports: - - "${API_PORT}:80" + - "${API_PORT}:8080" healthcheck: - test: ["CMD-SHELL", "curl -f http://localhost:80/api/health || exit 1"] + test: ["CMD-SHELL", "curl -f http://localhost:8080/api/health || exit 1"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 40s + depends_on: + azurite: + condition: service_healthy + db: + condition: service_healthy + networks: + - backend + + mesh-ingest: + container_name: "mesh-ingest" + build: + context: ./Src + dockerfile: ServiceLayer.MESH/Dockerfile + platform: linux/amd64 + restart: always + environment: + FUNCTIONS_WORKER_RUNTIME: "dotnet-isolated" + AzureWebJobsStorage: "${AZURITE_CONNECTION_STRING}" + AzureWebJobsSecretStorageType: "files" + DatabaseConnectionString: "${DatabaseConnectionString}" + FileDiscoveryTimerExpression: "${FileDiscoveryTimerExpression}" + MeshHandshakeTimerExpression: "${MeshHandshakeTimerExpression}" + FileRetryTimerExpression: "${FileRetryTimerExpression}" + FileExtractQueueName: "${FileExtractQueueName}" + FileTransformQueueName: "${FileTransformQueueName}" + StaleHours: "${StaleHours}" + MeshApiBaseUrl: "http://mesh_sandbox:80/messageexchange" + NbssMailboxId: "${NbssMailboxId}" + MeshPassword: "${MeshPassword}" + MeshSharedKey: "${MeshSharedKey}" + AZURITE_CONNECTION_STRING: "${AZURITE_CONNECTION_STRING}" + MeshStorageAccountUrl: "${AZURITE_CONNECTION_STRING}" + ASPNETCORE_ENVIRONMENT: "${ASPNETCORE_ENVIRONMENT}" + ASPNETCORE_URLS: "http://0.0.0.0:8080" + BlobContainerName: "${BlobContainerName}" + ports: + - "${MESH_INGEST_PORT}:8080" + healthcheck: + test: ["CMD-SHELL", "curl -f http://localhost:8080/api/health || exit 1"] interval: 30s timeout: 10s retries: 3 diff --git a/src/ServiceLayer.Common/EnvironmentVariables.cs b/src/ServiceLayer.Common/EnvironmentVariables.cs new file mode 100644 index 0000000..52a26c1 --- /dev/null +++ b/src/ServiceLayer.Common/EnvironmentVariables.cs @@ -0,0 +1,22 @@ +namespace ServiceLayer.Common; + +public static class EnvironmentVariables +{ + /// + /// Gets an environment variable by name. Throws if not found. + /// + /// The name of the environment variable. + /// The value of the environment variable. + /// Thrown when the variable is not found or is empty. + public static string GetRequired(string key) + { + var value = Environment.GetEnvironmentVariable(key); + + if (string.IsNullOrEmpty(value)) + { + throw new InvalidOperationException($"Environment variable '{key}' is not set or is empty."); + } + + return value; + } +} diff --git a/src/ServiceLayer.Mesh/Configuration/AppConfiguration.cs b/src/ServiceLayer.Mesh/Configuration/AppConfiguration.cs index 0b2d0d0..08799cd 100644 --- a/src/ServiceLayer.Mesh/Configuration/AppConfiguration.cs +++ b/src/ServiceLayer.Mesh/Configuration/AppConfiguration.cs @@ -1,3 +1,5 @@ +using ServiceLayer.Common; + namespace ServiceLayer.Mesh.Configuration; public class AppConfiguration : @@ -19,12 +21,7 @@ public class AppConfiguration : private static string GetRequired(string key) { - var value = Environment.GetEnvironmentVariable(key); - - if (string.IsNullOrEmpty(value)) - { - throw new InvalidOperationException($"Environment variable '{key}' is not set or is empty."); - } + var value = EnvironmentVariables.GetRequired(key); return value; } diff --git a/src/ServiceLayer.Mesh/DockerFile b/src/ServiceLayer.Mesh/DockerFile new file mode 100644 index 0000000..2257ba7 --- /dev/null +++ b/src/ServiceLayer.Mesh/DockerFile @@ -0,0 +1,23 @@ +FROM mcr.microsoft.com/dotnet/sdk:9.0 AS installer-env +WORKDIR /src/dotnet-function-app + +COPY ./dotnet-mesh-client/application/DotNetMeshClient/NHS.Mesh.Client/NHS.Mesh.Client.csproj ../dotnet-mesh-client/application/DotNetMeshClient/NHS.Mesh.Client/ +COPY ./ServiceLayer.Common/ServiceLayer.Common.csproj ../ServiceLayer.Common/ +COPY ./ServiceLayer.Mesh/ServiceLayer.Mesh.csproj . +RUN dotnet restore + +COPY ./dotnet-mesh-client/ ../dotnet-mesh-client/ +COPY ./ServiceLayer.Common/ ../ServiceLayer.Common/ +COPY ./ServiceLayer.Mesh/ . + +RUN dotnet publish -c Release -o /home/site/wwwroot + +FROM mcr.microsoft.com/azure-functions/dotnet-isolated:4-dotnet-isolated9.0 AS production +ENV AzureWebJobsScriptRoot=/home/site/wwwroot \ + AzureFunctionsJobHost__Logging__Console__IsEnabled=true \ + ASPNETCORE_ENVIRONMENT=Production + +RUN addgroup --system appgroup && adduser --system --ingroup appgroup appuser +USER appuser + +COPY --from=installer-env ["/home/site/wwwroot", "/home/site/wwwroot"] diff --git a/src/ServiceLayer.Mesh/Functions/FileDiscoveryFunction.cs b/src/ServiceLayer.Mesh/Functions/FileDiscoveryFunction.cs index 704186b..ffddeac 100644 --- a/src/ServiceLayer.Mesh/Functions/FileDiscoveryFunction.cs +++ b/src/ServiceLayer.Mesh/Functions/FileDiscoveryFunction.cs @@ -7,52 +7,51 @@ using ServiceLayer.Mesh.Configuration; using ServiceLayer.Mesh.Messaging; -namespace ServiceLayer.Mesh.Functions +namespace ServiceLayer.Mesh.Functions; + +public class FileDiscoveryFunction( + ILogger logger, + IFileDiscoveryFunctionConfiguration configuration, + IMeshInboxService meshInboxService, + ServiceLayerDbContext serviceLayerDbContext, + IFileExtractQueueClient fileExtractQueueClient) { - public class FileDiscoveryFunction( - ILogger logger, - IFileDiscoveryFunctionConfiguration configuration, - IMeshInboxService meshInboxService, - ServiceLayerDbContext serviceLayerDbContext, - IFileExtractQueueClient fileExtractQueueClient) + [Function("FileDiscoveryFunction")] + public async Task Run([TimerTrigger("%FileDiscoveryTimerExpression%")] TimerInfo myTimer) { - [Function("FileDiscoveryFunction")] - public async Task Run([TimerTrigger("%FileDiscoveryTimerExpression%")] TimerInfo myTimer) + logger.LogInformation("{functionName} started at: {time}", nameof(FileDiscoveryFunction), DateTime.UtcNow); + + var response = await meshInboxService.GetMessagesAsync(configuration.NbssMeshMailboxId); + + foreach (var messageId in response.Response.Messages) { - logger.LogInformation("{functionName} started at: {time}", nameof(FileDiscoveryFunction), DateTime.UtcNow); + await using var transaction = await serviceLayerDbContext.Database.BeginTransactionAsync(); - var response = await meshInboxService.GetMessagesAsync(configuration.NbssMeshMailboxId); + var existing = await serviceLayerDbContext.MeshFiles + .AnyAsync(f => f.FileId == messageId); - foreach (var messageId in response.Response.Messages) + if (!existing) { - await using var transaction = await serviceLayerDbContext.Database.BeginTransactionAsync(); + var file = new MeshFile + { + FileId = messageId, + FileType = MeshFileType.NbssAppointmentEvents, + MailboxId = configuration.NbssMeshMailboxId, + Status = MeshFileStatus.Discovered, + FirstSeenUtc = DateTime.UtcNow, + LastUpdatedUtc = DateTime.UtcNow + }; - var existing = await serviceLayerDbContext.MeshFiles - .AnyAsync(f => f.FileId == messageId); + serviceLayerDbContext.MeshFiles.Add(file); - if (!existing) - { - var file = new MeshFile - { - FileId = messageId, - FileType = MeshFileType.NbssAppointmentEvents, - MailboxId = configuration.NbssMeshMailboxId, - Status = MeshFileStatus.Discovered, - FirstSeenUtc = DateTime.UtcNow, - LastUpdatedUtc = DateTime.UtcNow - }; - - serviceLayerDbContext.MeshFiles.Add(file); - - await serviceLayerDbContext.SaveChangesAsync(); - await transaction.CommitAsync(); - - await fileExtractQueueClient.EnqueueFileExtractAsync(file); - } - else - { - await transaction.RollbackAsync(); - } + await serviceLayerDbContext.SaveChangesAsync(); + await transaction.CommitAsync(); + + await fileExtractQueueClient.EnqueueFileExtractAsync(file); + } + else + { + await transaction.RollbackAsync(); } } } diff --git a/src/ServiceLayer.Mesh/Functions/FileExtractFunction.cs b/src/ServiceLayer.Mesh/Functions/FileExtractFunction.cs index 75505e9..e0d7ed4 100644 --- a/src/ServiceLayer.Mesh/Functions/FileExtractFunction.cs +++ b/src/ServiceLayer.Mesh/Functions/FileExtractFunction.cs @@ -23,17 +23,12 @@ public class FileExtractFunction( [Function("FileExtractFunction")] public async Task Run([QueueTrigger("%FileExtractQueueName%")] FileExtractQueueMessage message) { - logger.LogInformation("{functionName} started at: {time}", nameof(FileDiscoveryFunction), DateTime.UtcNow); + logger.LogInformation("{functionName} started.", nameof(FileExtractFunction)); await using var transaction = await serviceLayerDbContext.Database.BeginTransactionAsync(); var file = await GetFileAsync(message.FileId); - if (file == null) - { - return; - } - - if (!IsFileSuitableForExtraction(file)) + if (file == null || !IsFileSuitableForExtraction(file)) { return; } diff --git a/src/ServiceLayer.Mesh/Functions/FileRetryFunction.cs b/src/ServiceLayer.Mesh/Functions/FileRetryFunction.cs index 7595af6..439bd4a 100644 --- a/src/ServiceLayer.Mesh/Functions/FileRetryFunction.cs +++ b/src/ServiceLayer.Mesh/Functions/FileRetryFunction.cs @@ -22,9 +22,8 @@ public async Task Run([TimerTrigger("%FileRetryTimerExpression%")] TimerInfo myT var staleDateTimeUtc = DateTime.UtcNow.AddHours(-configuration.StaleHours); - await Task.WhenAll( - RetryStaleExtractions(staleDateTimeUtc), - RetryStaleTransformations(staleDateTimeUtc)); + await RetryStaleExtractions(staleDateTimeUtc); + await RetryStaleTransformations(staleDateTimeUtc); } private async Task RetryStaleExtractions(DateTime staleDateTimeUtc) diff --git a/src/ServiceLayer.Mesh/Functions/FileTransformFunction.cs b/src/ServiceLayer.Mesh/Functions/FileTransformFunction.cs index 7484a2e..cc1a955 100644 --- a/src/ServiceLayer.Mesh/Functions/FileTransformFunction.cs +++ b/src/ServiceLayer.Mesh/Functions/FileTransformFunction.cs @@ -44,7 +44,8 @@ public async Task Run([QueueTrigger("%FileTransformQueueName%")] FileTransformQu var parsedfile = fileParser.Parse(fileContent); // TODO - take dependency on IEnumerable. - // After initial common checks against database, find the appropriate implementation of IFileTransformer to handle the functionality that differs between file type. + // After initial common checks against database, find the appropriate implementation of IFileTransformer + // to handle the functionality that differs between file types. } private async Task UpdateFileStatusForTransformation(MeshFile file) diff --git a/src/ServiceLayer.Mesh/Program.cs b/src/ServiceLayer.Mesh/Program.cs index b8246be..110688d 100644 --- a/src/ServiceLayer.Mesh/Program.cs +++ b/src/ServiceLayer.Mesh/Program.cs @@ -8,45 +8,52 @@ using ServiceLayer.Mesh.Configuration; using ServiceLayer.Mesh.Messaging; using ServiceLayer.Data; +using ServiceLayer.Mesh.Storage; +using ServiceLayer.Common; using ServiceLayer.Mesh.FileTypes.NbssAppointmentEvents; var host = new HostBuilder() .ConfigureFunctionsWebApplication() .ConfigureServices(services => { - var environment = Environment.GetEnvironmentVariable("ASPNETCORE_ENVIRONMENT"); + var environment = EnvironmentVariables.GetRequired("ASPNETCORE_ENVIRONMENT"); var isLocalEnvironment = environment == "Development"; // MESH Client config services - .AddMeshClient(_ => _.MeshApiBaseUrl = Environment.GetEnvironmentVariable("MeshApiBaseUrl")) - .AddMailbox(Environment.GetEnvironmentVariable("NbssMailboxId"), new NHS.MESH.Client.Configuration.MailboxConfiguration + .AddMeshClient(_ => _.MeshApiBaseUrl = EnvironmentVariables.GetRequired("MeshApiBaseUrl")) + .AddMailbox(EnvironmentVariables.GetRequired("NbssMailboxId"), new NHS.MESH.Client.Configuration.MailboxConfiguration { - Password = Environment.GetEnvironmentVariable("MeshPassword"), - SharedKey = Environment.GetEnvironmentVariable("MeshSharedKey"), + Password = EnvironmentVariables.GetRequired("MeshPassword"), + SharedKey = EnvironmentVariables.GetRequired("MeshSharedKey"), }).Build(); // EF Core DbContext services.AddDbContext(options => { - var connectionString = Environment.GetEnvironmentVariable("DatabaseConnectionString"); + var connectionString = EnvironmentVariables.GetRequired("DatabaseConnectionString"); if (string.IsNullOrEmpty(connectionString)) throw new InvalidOperationException("The connection string has not been initialized."); options.UseSqlServer(connectionString); }); + var queueClientOptions = new QueueClientOptions + { + MessageEncoding = QueueMessageEncoding.Base64 + }; + // Register QueueClients as singletons services.AddSingleton(provider => { if (isLocalEnvironment) { - var connectionString = Environment.GetEnvironmentVariable("AzureWebJobsStorage"); - return new QueueServiceClient(connectionString); + var connectionString = EnvironmentVariables.GetRequired("AzureWebJobsStorage"); + return new QueueServiceClient(connectionString, queueClientOptions); } - var meshStorageAccountUrl = Environment.GetEnvironmentVariable("MeshStorageAccountUrl"); - return new QueueServiceClient(new Uri(meshStorageAccountUrl), new DefaultAzureCredential()); + var meshStorageAccountUrl = EnvironmentVariables.GetRequired("MeshStorageAccountUrl"); + return new QueueServiceClient(new Uri(meshStorageAccountUrl), new DefaultAzureCredential(), queueClientOptions); }); services.AddSingleton(); @@ -56,10 +63,12 @@ services.AddSingleton(provider => { return new BlobContainerClient( - Environment.GetEnvironmentVariable("AzureWebJobsStorage"), - Environment.GetEnvironmentVariable("BlobContainerName")); + EnvironmentVariables.GetRequired("AzureWebJobsStorage"), + EnvironmentVariables.GetRequired("BlobContainerName")); }); + services.AddSingleton(); + services.AddTransient(); services.AddTransient(); services.AddTransient(); diff --git a/src/ServiceLayer.Mesh/Storage/MeshFilesBlobStore.cs b/src/ServiceLayer.Mesh/Storage/MeshFilesBlobStore.cs index 8a18e8f..ee707a2 100644 --- a/src/ServiceLayer.Mesh/Storage/MeshFilesBlobStore.cs +++ b/src/ServiceLayer.Mesh/Storage/MeshFilesBlobStore.cs @@ -3,18 +3,26 @@ namespace ServiceLayer.Mesh.Storage; -public class MeshFilesBlobStore(BlobContainerClient blobContainerClient) : IMeshFilesBlobStore +public class MeshFilesBlobStore : IMeshFilesBlobStore { + private BlobContainerClient _blobContainerClient; + + public MeshFilesBlobStore(BlobContainerClient blobContainerClient) + { + _blobContainerClient = blobContainerClient; + EnsureContainerExists(); + } + public async Task DownloadAsync(MeshFile file) { - var blobClient = blobContainerClient.GetBlobClient(file.BlobPath); + var blobClient = _blobContainerClient.GetBlobClient(file.BlobPath); return (await blobClient.DownloadAsync()).Value.Content; } public async Task UploadAsync(MeshFile file, byte[] data) { var blobPath = $"{file.FileType}/{file.FileId}"; - var blobClient = blobContainerClient.GetBlobClient(blobPath); + var blobClient = _blobContainerClient.GetBlobClient(blobPath); var dataStream = new MemoryStream(data); @@ -22,4 +30,9 @@ public async Task UploadAsync(MeshFile file, byte[] data) return blobPath; } + + private void EnsureContainerExists() + { + _blobContainerClient.CreateIfNotExists(); + } }