feat: FileParser

src/ServiceLayer.Mesh/FileTypes/NbssAppointmentEvents/FileParser.cs

@@ -1,12 +1,143 @@
+using CsvHelper;
+using CsvHelper.Configuration;
 using ServiceLayer.Mesh.FileTypes.NbssAppointmentEvents.Models;
+using System.Globalization;
+using System.Text;
 
 namespace ServiceLayer.Mesh.FileTypes.NbssAppointmentEvents;
 
 public class FileParser : IFileParser
 {
+    private const string HeaderIdentifier = "NBSSAPPT_HDR";
+    private const string FieldsIdentifier = "NBSSAPPT_FLDS";
+    private const string DataIdentifier = "NBSSAPPT_DATA";
+    private const string TrailerIdentifier = "NBSSAPPT_END";
+    private const int RecordTypeIdentifier = 0;
+
+    /// <summary>
+    /// Parse a stream of appointment data
+    /// </summary>
     public ParsedFile Parse(Stream stream)
     {
-        // TODO - implement this
-        throw new NotImplementedException();
+        if (stream == null)
+        {
+            throw new ArgumentNullException(nameof(stream), "Stream cannot be null");
+        }
+
+        var result = new ParsedFile();
+
+        using var reader = CreateStreamReader(stream);
+        using var csv = CreateCsvReader(reader);
+
+        var rowNumber = 0;
+        var fields = new List<string>();
+
+        while (csv.Read())
+        {
+            var recordIdentifier = GetFieldValue(csv, RecordTypeIdentifier);
+
+            switch (recordIdentifier)
+            {
+                case HeaderIdentifier:
+                    result.FileHeader = ParseHeader(csv);
+                    break;
+
+                case FieldsIdentifier:
+                    fields = ParseFields(csv);
+                    break;
+
+                case DataIdentifier:
+                    rowNumber++;
+                    result.DataRecords.Add(ParseDataRecord(csv, fields, rowNumber));
+                    break;
+
+                case TrailerIdentifier:
+                    result.FileTrailer = ParseTrailer(csv);
+                    break;
+
+                default:
+                    throw new InvalidOperationException($"Unknown record identifier: {recordIdentifier}");
+            }
+        }
+
+        return result;
+    }
+
+    private static List<string> ParseFields(CsvReader csv)
+    {
+        return Enumerable.Range(1, csv.Parser.Count - 1)
+        .Select(i => GetFieldValue(csv, i))
+        .Where(x => !string.IsNullOrEmpty(x))
+        .ToList()!;
+    }
+
+    private static string? GetFieldValue(CsvReader csv, int index) => index < csv.Parser.Count ? csv.GetField(index) : null;
+    private static StreamReader CreateStreamReader(Stream stream) => new(stream, Encoding.UTF8, detectEncodingFromByteOrderMarks: true, bufferSize: 1024, leaveOpen: true);
+    private static FileHeaderRecord ParseHeader(CsvReader csv) => csv.GetRecord<FileHeaderRecord>();
+    private static FileTrailerRecord ParseTrailer(CsvReader csv) => csv.GetRecord<FileTrailerRecord>();
+    private static CsvReader CreateCsvReader(StreamReader reader)
+    {
+        var config = new CsvConfiguration(CultureInfo.InvariantCulture)
+        {
+            Delimiter = "|",
+            Quote = '"',
+            Escape = '\\',
+            HasHeaderRecord = false,
+            Mode = CsvMode.RFC4180,
+            BadDataFound = null
+        };
+
+        var csv = new CsvReader(reader, config);
+        csv.Context.RegisterClassMap<FileHeaderRecordMap>();
+        csv.Context.RegisterClassMap<FileTrailerRecordMap>();
+
+        return csv;
+    }
+
+    private static FileDataRecord ParseDataRecord(CsvReader csv, List<string> columnHeadings, int rowNumber)
+    {
+        if (columnHeadings.Count == 0)
+        {
+            throw new InvalidOperationException("Field headers (NBSSAPPT_FLDS) must appear before data records.");
+        }
+
+        const int dataFieldStartIndex = 1;
+
+        var record = new FileDataRecord { RowNumber = rowNumber };
+
+        foreach (var (heading, index) in columnHeadings.Select((header, index) => (header, index + dataFieldStartIndex)))
+        {
+            if (index < csv.Parser.Count)
+            {
+                record.Fields[heading] = GetFieldValue(csv, index) ?? string.Empty;
+            }
+        }
+
+        return record;
+    }
+
+    public sealed class FileTrailerRecordMap : ClassMap<FileTrailerRecord>
+    {
+        public FileTrailerRecordMap()
+        {
+            Map(m => m.RecordTypeIdentifier).Index(0);
+            Map(m => m.ExtractId).Index(1);
+            Map(m => m.TransferEndDate).Index(2);
+            Map(m => m.TransferEndTime).Index(3);
+            Map(m => m.RecordCount).Index(4);
+        }
+    }
+
+    public sealed class FileHeaderRecordMap : ClassMap<FileHeaderRecord>
+    {
+        public FileHeaderRecordMap()
+        {
+            Map(m => m.RecordTypeIdentifier).Index(0);
+            Map(m => m.ExtractId).Index(1);
+            Map(m => m.TransferStartDate).Index(2);
+            Map(m => m.TransferStartTime).Index(3);
+            Map(m => m.RecordCount).Index(4);
+        }
     }
 }
+

src/ServiceLayer.Mesh/FileTypes/NbssAppointmentEvents/Models/ParsedFile.cs

@@ -4,5 +4,5 @@ public class ParsedFile
 {
     public FileHeaderRecord? FileHeader { get; set; }
     public FileTrailerRecord? FileTrailer { get; set; }
-    public required List<FileDataRecord> DataRecords { get; set; } = [];
+    public List<FileDataRecord> DataRecords { get; set; } = [];
 }

src/ServiceLayer.Mesh/Functions/FileTransformFunction.cs

@@ -5,6 +5,7 @@
 using ServiceLayer.Data;
 using ServiceLayer.Data.Models;
 using ServiceLayer.Mesh.Configuration;
+using ServiceLayer.Mesh.FileTypes.NbssAppointmentEvents;
 using ServiceLayer.Mesh.Messaging;
 using ServiceLayer.Mesh.Storage;
 
@@ -14,7 +15,8 @@ public class FileTransformFunction(
     ILogger<FileTransformFunction> logger,
     ServiceLayerDbContext serviceLayerDbContext,
     IMeshFilesBlobStore meshFileBlobStore,
-    IFileTransformFunctionConfiguration configuration)
+    IFileTransformFunctionConfiguration configuration,
+    IFileParser fileParser)
 {
     [Function("FileTransformFunction")]
     public async Task Run([QueueTrigger("%FileTransformQueueName%")] FileTransformQueueMessage message)
@@ -25,7 +27,7 @@ public async Task Run([QueueTrigger("%FileTransformQueueName%")] FileTransformQu
 
         if (file == null)
         {
-            logger.LogWarning("File with id: {fileId} not found in MeshFiles table.", message.FileId);
+            logger.LogWarning("File with id: {FileId} not found in MeshFiles table.", message.FileId);
             return;
         }
 
@@ -39,6 +41,8 @@ public async Task Run([QueueTrigger("%FileTransformQueueName%")] FileTransformQu
 
         var fileContent = await meshFileBlobStore.DownloadAsync(file);
 
+        var parsedfile = fileParser.Parse(fileContent);
+
         // TODO - take dependency on IEnumerable<IFileTransformer>.
         // After initial common checks against database, find the appropriate implementation of IFileTransformer to handle the functionality that differs between file type.
     }
@@ -59,7 +63,7 @@ private bool IsFileSuitableForTransformation(MeshFile file)
             (file.Status == MeshFileStatus.Transforming && file.LastUpdatedUtc > DateTime.UtcNow.AddHours(-configuration.StaleHours)))
         {
             logger.LogWarning(
-                "File with id: {fileId} found in MeshFiles table but is not suitable for transformation. Status: {status}, LastUpdatedUtc: {lastUpdatedUtc}.",
+                "File with id: {FileId} found in MeshFiles table but is not suitable for transformation. Status: {Status}, LastUpdatedUtc: {LastUpdatedUtc}.",
                 file.FileId,
                 file.Status,
                 file.LastUpdatedUtc.ToTimestamp());

src/ServiceLayer.Mesh/Program.cs

@@ -8,6 +8,7 @@
 using ServiceLayer.Mesh.Configuration;
 using ServiceLayer.Mesh.Messaging;
 using ServiceLayer.Data;
+using ServiceLayer.Mesh.FileTypes.NbssAppointmentEvents;
 
 var host = new HostBuilder()
     .ConfigureFunctionsWebApplication()
@@ -50,6 +51,7 @@
 
         services.AddSingleton<IFileExtractQueueClient, FileExtractQueueClient>();
         services.AddSingleton<IFileTransformQueueClient, FileTransformQueueClient>();
+        services.AddSingleton<IFileParser, FileParser>();
 
         services.AddSingleton(provider =>
         {

src/ServiceLayer.Mesh/ServiceLayer.Mesh.csproj

@@ -11,6 +11,7 @@
     <PackageReference Include="Azure.Identity" Version="1.13.2" />
     <PackageReference Include="Azure.Storage.Blobs" Version="12.24.0" />
     <PackageReference Include="Azure.Storage.Queues" Version="12.22.0" />
+    <PackageReference Include="CsvHelper" Version="33.0.1" />
     <!-- Application Insights isn't enabled by default. See https://aka.ms/AAt8mw4. -->
     <!-- <PackageReference Include="Microsoft.ApplicationInsights.WorkerService" Version="2.22.0" /> -->
     <!-- <PackageReference Include="Microsoft.Azure.Functions.Worker.ApplicationInsights" Version="2.0.0" /> -->