feat: transform function skeleton and blob download functionality (#14)

alex-clayton-1 · ianfnelson · web-flow · commit 3488158931fb · 2025-05-19T09:31:16.000Z
Co-authored-by: Ian Nelson &lt;ian@iannelson.systems&gt;
diff --git a/src/ServiceLayer.Mesh/Configuration/AppConfiguration.cs b/src/ServiceLayer.Mesh/Configuration/AppConfiguration.cs
@@ -5,6 +5,7 @@ public class AppConfiguration :
     IFileExtractFunctionConfiguration,
     IFileExtractQueueClientConfiguration,
     IFileTransformQueueClientConfiguration,
+    IFileTransformFunctionConfiguration,
     IFileRetryFunctionConfiguration,
     IMeshHandshakeFunctionConfiguration
 {
diff --git a/src/ServiceLayer.Mesh/Configuration/IFileTransformFunctionConfiguration.cs b/src/ServiceLayer.Mesh/Configuration/IFileTransformFunctionConfiguration.cs
@@ -0,0 +1,6 @@
+namespace ServiceLayer.Mesh.Configuration;
+
+public interface IFileTransformFunctionConfiguration
+{
+    int StaleHours { get; }
+}
diff --git a/src/ServiceLayer.Mesh/FileTypes/NbssAppointmentEvents/FileParser.cs b/src/ServiceLayer.Mesh/FileTypes/NbssAppointmentEvents/FileParser.cs
@@ -0,0 +1,12 @@
+using ServiceLayer.Mesh.FileTypes.NbssAppointmentEvents.Models;
+
+namespace ServiceLayer.Mesh.FileTypes.NbssAppointmentEvents;
+
+public class FileParser : IFileParser
+{
+    public ParsedFile Parse(Stream stream)
+    {
+        // TODO - implement this
+        throw new NotImplementedException();
+    }
+}
diff --git a/src/ServiceLayer.Mesh/FileTypes/NbssAppointmentEvents/FileTransformer.cs b/src/ServiceLayer.Mesh/FileTypes/NbssAppointmentEvents/FileTransformer.cs
@@ -0,0 +1,35 @@
+using ServiceLayer.Data.Models;
+using ServiceLayer.Mesh.FileTypes.NbssAppointmentEvents.Validation;
+
+namespace ServiceLayer.Mesh.FileTypes.NbssAppointmentEvents;
+
+// TODO - NBSS appointment file specific implementation of IFileTransformer. To orchestrate parsing, validation and staging of data (delegated to separate classes)
+public class FileTransformer : IFileTransformer
+{
+    private readonly IFileParser _fileParser;
+    private readonly IValidationRunner _validationRunner;
+    private readonly IStagingPersister _stagingPersister;
+
+    public FileTransformer(IFileParser fileParser, IValidationRunner validationRunner, IStagingPersister stagingPersister)
+    {
+        _fileParser = fileParser;
+        _validationRunner = validationRunner;
+        _stagingPersister = stagingPersister;
+    }
+
+    public MeshFileType HandlesFileType => MeshFileType.NbssAppointmentEvents;
+
+    public async Task<IList<ValidationError>> TransformFileAsync(Stream stream, MeshFile metaData)
+    {
+        // TODO - wrap this parsing in a try-catch and return a List<ValidationError> in case of any unforeseen parsing issues (file is totally unlike anything we expect)
+        var parsed = _fileParser.Parse(stream);
+
+        var validationErrors = _validationRunner.Validate(parsed);
+        if (!validationErrors.Any())
+        {
+            await _stagingPersister.WriteStagedData(parsed);
+        }
+
+        return validationErrors;
+    }
+}
diff --git a/src/ServiceLayer.Mesh/FileTypes/NbssAppointmentEvents/IFileParser.cs b/src/ServiceLayer.Mesh/FileTypes/NbssAppointmentEvents/IFileParser.cs
@@ -0,0 +1,8 @@
+using ServiceLayer.Mesh.FileTypes.NbssAppointmentEvents.Models;
+
+namespace ServiceLayer.Mesh.FileTypes.NbssAppointmentEvents;
+
+public interface IFileParser
+{
+    ParsedFile Parse(Stream stream);
+}
diff --git a/src/ServiceLayer.Mesh/FileTypes/NbssAppointmentEvents/IStagingPersister.cs b/src/ServiceLayer.Mesh/FileTypes/NbssAppointmentEvents/IStagingPersister.cs
@@ -0,0 +1,9 @@
+using ServiceLayer.Mesh.FileTypes.NbssAppointmentEvents.Models;
+
+namespace ServiceLayer.Mesh.FileTypes.NbssAppointmentEvents;
+
+// TODO - interface for class to take validated AppointmentEventsFile and save the records to NbssAppointmentEvents table
+public interface IStagingPersister
+{
+    Task WriteStagedData(ParsedFile parsedFile);
+}
diff --git a/src/ServiceLayer.Mesh/FileTypes/NbssAppointmentEvents/Models/FileControlRecord.cs b/src/ServiceLayer.Mesh/FileTypes/NbssAppointmentEvents/Models/FileControlRecord.cs
@@ -0,0 +1,14 @@
+namespace ServiceLayer.Mesh.FileTypes.NbssAppointmentEvents.Models;
+
+public class FileControlRecord
+{
+    public string? RecordTypeIdentifier { get; set; }
+
+    public string? ExtractId { get; set; }
+
+    public string? TransferStartDate { get; set; }
+
+    public string? TransferStartTime { get; set; }
+
+    public string? RecordCount { get; set; }
+}
diff --git a/src/ServiceLayer.Mesh/FileTypes/NbssAppointmentEvents/Models/FileDataRecord.cs b/src/ServiceLayer.Mesh/FileTypes/NbssAppointmentEvents/Models/FileDataRecord.cs
@@ -0,0 +1,9 @@
+namespace ServiceLayer.Mesh.FileTypes.NbssAppointmentEvents.Models;
+
+public class FileDataRecord
+{
+    public int RowNumber { get; set; }
+    public Dictionary<string, string> Fields { get; } = [];
+
+    public string? this[string fieldName] => Fields.GetValueOrDefault(fieldName);
+}
diff --git a/src/ServiceLayer.Mesh/FileTypes/NbssAppointmentEvents/Models/ParsedFile.cs b/src/ServiceLayer.Mesh/FileTypes/NbssAppointmentEvents/Models/ParsedFile.cs
@@ -0,0 +1,9 @@
+namespace ServiceLayer.Mesh.FileTypes.NbssAppointmentEvents.Models;
+
+public class ParsedFile
+{
+    public FileControlRecord? FileHeader { get; set; }
+    public FileControlRecord? FileTrailer { get; set; }
+    public required List<string> ColumnHeadings { get; set; } = [];
+    public required List<FileDataRecord> DataRecords { get; set; } = [];
+}
diff --git a/src/ServiceLayer.Mesh/FileTypes/NbssAppointmentEvents/StagingPersister.cs b/src/ServiceLayer.Mesh/FileTypes/NbssAppointmentEvents/StagingPersister.cs
@@ -0,0 +1,13 @@
+using ServiceLayer.Mesh.FileTypes.NbssAppointmentEvents.Models;
+
+namespace ServiceLayer.Mesh.FileTypes.NbssAppointmentEvents;
+
+// TODO - class to take validated AppointmentEventsFile and save the records to NbssAppointmentEvents table
+public class StagingPersister : IStagingPersister
+{
+    public Task WriteStagedData(ParsedFile parsedFile)
+    {
+        // TODO - implement this
+        throw new NotImplementedException();
+    }
+}
diff --git a/src/ServiceLayer.Mesh/FileTypes/NbssAppointmentEvents/Validation/IFileValidator.cs b/src/ServiceLayer.Mesh/FileTypes/NbssAppointmentEvents/Validation/IFileValidator.cs
@@ -0,0 +1,9 @@
+using ServiceLayer.Mesh.FileTypes.NbssAppointmentEvents.Models;
+
+namespace ServiceLayer.Mesh.FileTypes.NbssAppointmentEvents.Validation;
+
+// TODO - create a whole bunch of implementations of this to perform the validation against NBSS Appointment events files
+public interface IFileValidator
+{
+    IEnumerable<ValidationError> Validate(ParsedFile file);
+}
diff --git a/src/ServiceLayer.Mesh/FileTypes/NbssAppointmentEvents/Validation/IRecordValidator.cs b/src/ServiceLayer.Mesh/FileTypes/NbssAppointmentEvents/Validation/IRecordValidator.cs
@@ -0,0 +1,9 @@
+using ServiceLayer.Mesh.FileTypes.NbssAppointmentEvents.Models;
+
+namespace ServiceLayer.Mesh.FileTypes.NbssAppointmentEvents.Validation;
+
+// TODO - create a whole bunch of implementations of this to perform the validation against NBSS Appointment events records
+public interface IRecordValidator
+{
+    IEnumerable<ValidationError> Validate(FileDataRecord fileDataRecord);
+}
diff --git a/src/ServiceLayer.Mesh/FileTypes/NbssAppointmentEvents/Validation/IValidationRunner.cs b/src/ServiceLayer.Mesh/FileTypes/NbssAppointmentEvents/Validation/IValidationRunner.cs
@@ -0,0 +1,10 @@
+using ServiceLayer.Mesh.FileTypes.NbssAppointmentEvents.Models;
+
+namespace ServiceLayer.Mesh.FileTypes.NbssAppointmentEvents.Validation;
+
+public interface IValidationRunner
+{
+    IList<ValidationError> Validate(ParsedFile file);
+}
+
+
diff --git a/src/ServiceLayer.Mesh/FileTypes/NbssAppointmentEvents/Validation/ValidationRunner.cs b/src/ServiceLayer.Mesh/FileTypes/NbssAppointmentEvents/Validation/ValidationRunner.cs
@@ -0,0 +1,29 @@
+using ServiceLayer.Mesh.FileTypes.NbssAppointmentEvents.Models;
+
+namespace ServiceLayer.Mesh.FileTypes.NbssAppointmentEvents.Validation;
+
+public class ValidationRunner(
+    IEnumerable<IFileValidator> fileValidators,
+    IEnumerable<IRecordValidator> recordValidators)
+    : IValidationRunner
+{
+    public IList<ValidationError> Validate(ParsedFile file)
+    {
+        var errors = new List<ValidationError>();
+
+        foreach (var dataRecord in file.DataRecords)
+        {
+            foreach (var recordValidator in recordValidators)
+            {
+                errors.AddRange(recordValidator.Validate(dataRecord));
+            }
+        }
+
+        foreach (var validator in fileValidators)
+        {
+            errors.AddRange(validator.Validate(file));
+        }
+
+        return errors;
+    }
+}
diff --git a/src/ServiceLayer.Mesh/Functions/FileTransformFunction.cs b/src/ServiceLayer.Mesh/Functions/FileTransformFunction.cs
@@ -1,6 +1,70 @@
+using Google.Protobuf.WellKnownTypes;
+using Microsoft.Azure.Functions.Worker;
+using Microsoft.EntityFrameworkCore;
+using Microsoft.Extensions.Logging;
+using ServiceLayer.Data;
+using ServiceLayer.Data.Models;
+using ServiceLayer.Mesh.Configuration;
+using ServiceLayer.Mesh.Messaging;
+using ServiceLayer.Mesh.Storage;
+
 namespace ServiceLayer.Mesh.Functions;
 
-public class FileTransformFunction
+public class FileTransformFunction(
+    ILogger<FileTransformFunction> logger,
+    ServiceLayerDbContext serviceLayerDbContext,
+    IMeshFilesBlobStore meshFileBlobStore,
+    IFileTransformFunctionConfiguration configuration)
 {
+    [Function("FileTransformFunction")]
+    public async Task Run([QueueTrigger("%FileTransformQueueName%")] FileTransformQueueMessage message)
+    {
+        await using var transaction = await serviceLayerDbContext.Database.BeginTransactionAsync();
+
+        var file = await serviceLayerDbContext.MeshFiles.FirstOrDefaultAsync(f => f.FileId == message.FileId);
+
+        if (file == null)
+        {
+            logger.LogWarning("File with id: {fileId} not found in MeshFiles table.", message.FileId);
+            return;
+        }
+
+        if (!IsFileSuitableForTransformation(file))
+        {
+            return;
+        }
+
+        await UpdateFileStatusForTransformation(file);
+        await transaction.CommitAsync();
+
+        var fileContent = await meshFileBlobStore.DownloadAsync(file);
+
+        // TODO - take dependency on IEnumerable<IFileTransformer>.
+        // After initial common checks against database, find the appropriate implementation of IFileTransformer to handle the functionality that differs between file type.
+    }
+
+    private async Task UpdateFileStatusForTransformation(MeshFile file)
+    {
+        file.Status = MeshFileStatus.Transforming;
+        file.LastUpdatedUtc = DateTime.UtcNow;
+        await serviceLayerDbContext.SaveChangesAsync();
+    }
 
+    private bool IsFileSuitableForTransformation(MeshFile file)
+    {
+        // We only want to transform files if they are in a Extracted state,
+        // or are in a Transforming state and were last touched over 12 hours ago.
+        var expectedStatuses = new[] { MeshFileStatus.Extracted, MeshFileStatus.Transforming };
+        if (!expectedStatuses.Contains(file.Status) ||
+            (file.Status == MeshFileStatus.Transforming && file.LastUpdatedUtc > DateTime.UtcNow.AddHours(-configuration.StaleHours)))
+        {
+            logger.LogWarning(
+                "File with id: {fileId} found in MeshFiles table but is not suitable for transformation. Status: {status}, LastUpdatedUtc: {lastUpdatedUtc}.",
+                file.FileId,
+                file.Status,
+                file.LastUpdatedUtc.ToTimestamp());
+            return false;
+        }
+        return true;
+    }
 }
diff --git a/src/ServiceLayer.Mesh/IFileTransformer.cs b/src/ServiceLayer.Mesh/IFileTransformer.cs
@@ -0,0 +1,9 @@
+using ServiceLayer.Data.Models;
+
+namespace ServiceLayer.Mesh;
+
+public interface IFileTransformer
+{
+    MeshFileType HandlesFileType { get; }
+    Task<IList<ValidationError>> TransformFileAsync(Stream stream, MeshFile metaData);
+}
diff --git a/src/ServiceLayer.Mesh/Program.cs b/src/ServiceLayer.Mesh/Program.cs
@@ -64,6 +64,7 @@
         services.AddTransient<IFileTransformQueueClientConfiguration, AppConfiguration>();
         services.AddTransient<IMeshHandshakeFunctionConfiguration, AppConfiguration>();
         services.AddTransient<IFileRetryFunctionConfiguration, AppConfiguration>();
+        services.AddTransient<IFileTransformFunctionConfiguration, AppConfiguration>();
     });
 
 
diff --git a/src/ServiceLayer.Mesh/Storage/MeshFilesBlobStore.cs b/src/ServiceLayer.Mesh/Storage/MeshFilesBlobStore.cs
@@ -5,9 +5,10 @@ namespace ServiceLayer.Mesh.Storage;
 
 public class MeshFilesBlobStore(BlobContainerClient blobContainerClient) : IMeshFilesBlobStore
 {
-    public Task<Stream> DownloadAsync(MeshFile file)
+    public async Task<Stream> DownloadAsync(MeshFile file)
     {
-        throw new NotImplementedException();
+        var blobClient = blobContainerClient.GetBlobClient(file.BlobPath);
+        return (await blobClient.DownloadAsync()).Value.Content;
     }
 
     public async Task<string> UploadAsync(MeshFile file, byte[] data)
diff --git a/src/ServiceLayer.Mesh/ValidationError.cs b/src/ServiceLayer.Mesh/ValidationError.cs
@@ -0,0 +1,12 @@
+namespace ServiceLayer.Mesh;
+
+public class ValidationError
+{
+    public int? RowNumber { get; set; }
+
+    public required string Field { get; set; }
+
+    public required string Code { get; set; }
+
+    public required string Error { get; set; }
+}
diff --git a/tests/ServiceLayer.Mesh.Tests/Functions/FileTransformFunctionTests.cs b/tests/ServiceLayer.Mesh.Tests/Functions/FileTransformFunctionTests.cs

Original file line number	Diff line number	Diff line change
`@@ -5,6 +5,7 @@ public class AppConfiguration :`
`5`	`5`	`IFileExtractFunctionConfiguration,`
`6`	`6`	`IFileExtractQueueClientConfiguration,`
`7`	`7`	`IFileTransformQueueClientConfiguration,`
	`8`	`+ IFileTransformFunctionConfiguration,`
`8`	`9`	`IFileRetryFunctionConfiguration,`
`9`	`10`	`IMeshHandshakeFunctionConfiguration`
`10`	`11`	`{`