Skip to content
This repository was archived by the owner on Jul 28, 2025. It is now read-only.

Commit 83951fd

Browse files
Warren-Pittersonianfnelsonalex-clayton-1
authored
feat: FileParser (#19)
Co-authored-by: Ian Nelson <[email protected]> Co-authored-by: alex-clayton-1 <[email protected]>
1 parent 4ec365b commit 83951fd

File tree

19 files changed

+582
-7
lines changed

19 files changed

+582
-7
lines changed

‎src/ServiceLayer.Mesh/FileTypes/NbssAppointmentEvents/FileParser.cs‎

Lines changed: 133 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,143 @@
1+
using CsvHelper;
2+
using CsvHelper.Configuration;
13
using ServiceLayer.Mesh.FileTypes.NbssAppointmentEvents.Models;
4+
using System.Globalization;
5+
using System.Text;
26

37
namespace ServiceLayer.Mesh.FileTypes.NbssAppointmentEvents;
48

59
public class FileParser : IFileParser
610
{
11+
private const string HeaderIdentifier = "NBSSAPPT_HDR";
12+
private const string FieldsIdentifier = "NBSSAPPT_FLDS";
13+
private const string DataIdentifier = "NBSSAPPT_DATA";
14+
private const string TrailerIdentifier = "NBSSAPPT_END";
15+
private const int RecordTypeIdentifier = 0;
16+
17+
/// <summary>
18+
/// Parse a stream of appointment data
19+
/// </summary>
720
public ParsedFile Parse(Stream stream)
821
{
9-
// TODO - implement this
10-
throw new NotImplementedException();
22+
if (stream == null)
23+
{
24+
throw new ArgumentNullException(nameof(stream), "Stream cannot be null");
25+
}
26+
27+
var result = new ParsedFile();
28+
29+
using var reader = CreateStreamReader(stream);
30+
using var csv = CreateCsvReader(reader);
31+
32+
var rowNumber = 0;
33+
var fields = new List<string>();
34+
35+
while (csv.Read())
36+
{
37+
var recordIdentifier = GetFieldValue(csv, RecordTypeIdentifier);
38+
39+
switch (recordIdentifier)
40+
{
41+
case HeaderIdentifier:
42+
result.FileHeader = ParseHeader(csv);
43+
break;
44+
45+
case FieldsIdentifier:
46+
fields = ParseFields(csv);
47+
break;
48+
49+
case DataIdentifier:
50+
rowNumber++;
51+
result.DataRecords.Add(ParseDataRecord(csv, fields, rowNumber));
52+
break;
53+
54+
case TrailerIdentifier:
55+
result.FileTrailer = ParseTrailer(csv);
56+
break;
57+
58+
default:
59+
throw new InvalidOperationException($"Unknown record identifier: {recordIdentifier}");
60+
}
61+
}
62+
63+
return result;
64+
}
65+
66+
private static List<string> ParseFields(CsvReader csv)
67+
{
68+
return Enumerable.Range(1, csv.Parser.Count - 1)
69+
.Select(i => GetFieldValue(csv, i))
70+
.Where(x => !string.IsNullOrEmpty(x))
71+
.ToList()!;
72+
}
73+
74+
private static string? GetFieldValue(CsvReader csv, int index) => index < csv.Parser.Count ? csv.GetField(index) : null;
75+
private static StreamReader CreateStreamReader(Stream stream) => new(stream, Encoding.UTF8, detectEncodingFromByteOrderMarks: true, bufferSize: 1024, leaveOpen: true);
76+
private static FileHeaderRecord ParseHeader(CsvReader csv) => csv.GetRecord<FileHeaderRecord>();
77+
private static FileTrailerRecord ParseTrailer(CsvReader csv) => csv.GetRecord<FileTrailerRecord>();
78+
private static CsvReader CreateCsvReader(StreamReader reader)
79+
{
80+
var config = new CsvConfiguration(CultureInfo.InvariantCulture)
81+
{
82+
Delimiter = "|",
83+
Quote = '"',
84+
Escape = '\\',
85+
HasHeaderRecord = false,
86+
Mode = CsvMode.RFC4180,
87+
BadDataFound = null
88+
};
89+
90+
var csv = new CsvReader(reader, config);
91+
csv.Context.RegisterClassMap<FileHeaderRecordMap>();
92+
csv.Context.RegisterClassMap<FileTrailerRecordMap>();
93+
94+
return csv;
95+
}
96+
97+
private static FileDataRecord ParseDataRecord(CsvReader csv, List<string> columnHeadings, int rowNumber)
98+
{
99+
if (columnHeadings.Count == 0)
100+
{
101+
throw new InvalidOperationException("Field headers (NBSSAPPT_FLDS) must appear before data records.");
102+
}
103+
104+
const int dataFieldStartIndex = 1;
105+
106+
var record = new FileDataRecord { RowNumber = rowNumber };
107+
108+
foreach (var (heading, index) in columnHeadings.Select((header, index) => (header, index + dataFieldStartIndex)))
109+
{
110+
if (index < csv.Parser.Count)
111+
{
112+
record.Fields[heading] = GetFieldValue(csv, index) ?? string.Empty;
113+
}
114+
}
115+
116+
return record;
117+
}
118+
119+
public sealed class FileTrailerRecordMap : ClassMap<FileTrailerRecord>
120+
{
121+
public FileTrailerRecordMap()
122+
{
123+
Map(m => m.RecordTypeIdentifier).Index(0);
124+
Map(m => m.ExtractId).Index(1);
125+
Map(m => m.TransferEndDate).Index(2);
126+
Map(m => m.TransferEndTime).Index(3);
127+
Map(m => m.RecordCount).Index(4);
128+
}
129+
}
130+
131+
public sealed class FileHeaderRecordMap : ClassMap<FileHeaderRecord>
132+
{
133+
public FileHeaderRecordMap()
134+
{
135+
Map(m => m.RecordTypeIdentifier).Index(0);
136+
Map(m => m.ExtractId).Index(1);
137+
Map(m => m.TransferStartDate).Index(2);
138+
Map(m => m.TransferStartTime).Index(3);
139+
Map(m => m.RecordCount).Index(4);
140+
}
11141
}
12142
}
143+

‎src/ServiceLayer.Mesh/FileTypes/NbssAppointmentEvents/Models/ParsedFile.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,5 +4,5 @@ public class ParsedFile
44
{
55
public FileHeaderRecord? FileHeader { get; set; }
66
public FileTrailerRecord? FileTrailer { get; set; }
7-
public required List<FileDataRecord> DataRecords { get; set; } = [];
7+
public List<FileDataRecord> DataRecords { get; set; } = [];
88
}

‎src/ServiceLayer.Mesh/Functions/FileTransformFunction.cs‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
using ServiceLayer.Data;
66
using ServiceLayer.Data.Models;
77
using ServiceLayer.Mesh.Configuration;
8+
using ServiceLayer.Mesh.FileTypes.NbssAppointmentEvents;
89
using ServiceLayer.Mesh.Messaging;
910
using ServiceLayer.Mesh.Storage;
1011

@@ -14,7 +15,8 @@ public class FileTransformFunction(
1415
ILogger<FileTransformFunction> logger,
1516
ServiceLayerDbContext serviceLayerDbContext,
1617
IMeshFilesBlobStore meshFileBlobStore,
17-
IFileTransformFunctionConfiguration configuration)
18+
IFileTransformFunctionConfiguration configuration,
19+
IFileParser fileParser)
1820
{
1921
[Function("FileTransformFunction")]
2022
public async Task Run([QueueTrigger("%FileTransformQueueName%")] FileTransformQueueMessage message)
@@ -25,7 +27,7 @@ public async Task Run([QueueTrigger("%FileTransformQueueName%")] FileTransformQu
2527

2628
if (file == null)
2729
{
28-
logger.LogWarning("File with id: {fileId} not found in MeshFiles table.", message.FileId);
30+
logger.LogWarning("File with id: {FileId} not found in MeshFiles table.", message.FileId);
2931
return;
3032
}
3133

@@ -39,6 +41,8 @@ public async Task Run([QueueTrigger("%FileTransformQueueName%")] FileTransformQu
3941

4042
var fileContent = await meshFileBlobStore.DownloadAsync(file);
4143

44+
var parsedfile = fileParser.Parse(fileContent);
45+
4246
// TODO - take dependency on IEnumerable<IFileTransformer>.
4347
// After initial common checks against database, find the appropriate implementation of IFileTransformer to handle the functionality that differs between file type.
4448
}
@@ -59,7 +63,7 @@ private bool IsFileSuitableForTransformation(MeshFile file)
5963
(file.Status == MeshFileStatus.Transforming && file.LastUpdatedUtc > DateTime.UtcNow.AddHours(-configuration.StaleHours)))
6064
{
6165
logger.LogWarning(
62-
"File with id: {fileId} found in MeshFiles table but is not suitable for transformation. Status: {status}, LastUpdatedUtc: {lastUpdatedUtc}.",
66+
"File with id: {FileId} found in MeshFiles table but is not suitable for transformation. Status: {Status}, LastUpdatedUtc: {LastUpdatedUtc}.",
6367
file.FileId,
6468
file.Status,
6569
file.LastUpdatedUtc.ToTimestamp());

‎src/ServiceLayer.Mesh/Program.cs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
using ServiceLayer.Mesh.Configuration;
99
using ServiceLayer.Mesh.Messaging;
1010
using ServiceLayer.Data;
11+
using ServiceLayer.Mesh.FileTypes.NbssAppointmentEvents;
1112

1213
var host = new HostBuilder()
1314
.ConfigureFunctionsWebApplication()
@@ -50,6 +51,7 @@
5051

5152
services.AddSingleton<IFileExtractQueueClient, FileExtractQueueClient>();
5253
services.AddSingleton<IFileTransformQueueClient, FileTransformQueueClient>();
54+
services.AddSingleton<IFileParser, FileParser>();
5355

5456
services.AddSingleton(provider =>
5557
{

‎src/ServiceLayer.Mesh/ServiceLayer.Mesh.csproj‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
<PackageReference Include="Azure.Identity" Version="1.13.2" />
1212
<PackageReference Include="Azure.Storage.Blobs" Version="12.24.0" />
1313
<PackageReference Include="Azure.Storage.Queues" Version="12.22.0" />
14+
<PackageReference Include="CsvHelper" Version="33.0.1" />
1415
<!-- Application Insights isn't enabled by default. See https://aka.ms/AAt8mw4. -->
1516
<!-- <PackageReference Include="Microsoft.ApplicationInsights.WorkerService" Version="2.22.0" /> -->
1617
<!-- <PackageReference Include="Microsoft.Azure.Functions.Worker.ApplicationInsights" Version="2.0.0" /> -->

0 commit comments

Comments
 (0)