Skip to content
This repository was archived by the owner on Jul 28, 2025. It is now read-only.

Commit d72b604

Browse files
ianfnelsonianfnelson
authored
feat: DTOSS-9346 Audit trail (#51)
1 parent df2ee5a commit d72b604

16 files changed

+547
-76
lines changed

‎src/ServiceLayer.Mesh/Functions/FileDiscoveryFunction.cs‎

Lines changed: 28 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ public class FileDiscoveryFunction(
1515
IMeshInboxService meshInboxService,
1616
ServiceLayerDbContext serviceLayerDbContext,
1717
IFileExtractQueueClient fileExtractQueueClient)
18+
: MeshFileFunctionBase(serviceLayerDbContext)
1819
{
1920
[Function("FileDiscoveryFunction")]
2021
public async Task Run([TimerTrigger("%FileDiscoveryTimerExpression%")] TimerInfo myTimer)
@@ -26,28 +27,16 @@ public async Task Run([TimerTrigger("%FileDiscoveryTimerExpression%")] TimerInfo
2627
// TODO - check if response.IsSuccessful before proceeding to dereference the Response.Messages
2728
foreach (var messageId in response.Response.Messages)
2829
{
29-
await using var transaction = await serviceLayerDbContext.Database.BeginTransactionAsync();
30+
await using var transaction = await ServiceLayerDbContext.Database.BeginTransactionAsync();
3031

31-
var existing = await serviceLayerDbContext.MeshFiles
32+
var existing = await ServiceLayerDbContext.MeshFiles
3233
.AnyAsync(f => f.FileId == messageId);
3334

3435
if (!existing)
3536
{
36-
var file = new MeshFile
37-
{
38-
FileId = messageId,
39-
FileType = MeshFileType.NbssAppointmentEvents,
40-
MailboxId = configuration.NbssMeshMailboxId,
41-
Status = MeshFileStatus.Discovered,
42-
FirstSeenUtc = DateTime.UtcNow,
43-
LastUpdatedUtc = DateTime.UtcNow
44-
};
45-
46-
serviceLayerDbContext.MeshFiles.Add(file);
47-
48-
await serviceLayerDbContext.SaveChangesAsync();
49-
await transaction.CommitAsync();
37+
var file = await CreateMeshFile(messageId);
5038

39+
await transaction.CommitAsync();
5140
await fileExtractQueueClient.EnqueueFileExtractAsync(file);
5241
}
5342
else
@@ -56,4 +45,27 @@ public async Task Run([TimerTrigger("%FileDiscoveryTimerExpression%")] TimerInfo
5645
}
5746
}
5847
}
48+
49+
private async Task<MeshFile> CreateMeshFile(string messageId)
50+
{
51+
var now = DateTime.UtcNow;
52+
53+
var file = new MeshFile
54+
{
55+
FileId = messageId,
56+
FileType = MeshFileType.NbssAppointmentEvents,
57+
MailboxId = configuration.NbssMeshMailboxId,
58+
Status = MeshFileStatus.Discovered,
59+
FirstSeenUtc = now,
60+
LastUpdatedUtc = now
61+
};
62+
63+
ServiceLayerDbContext.MeshFiles.Add(file);
64+
65+
await UpdateMeshFile(file, MeshFileStatus.Discovered);
66+
67+
return file;
68+
}
69+
70+
protected override FileEventSource Source => FileEventSource.DiscoveryFunction;
5971
}

‎src/ServiceLayer.Mesh/Functions/FileExtractFunction.cs‎

Lines changed: 8 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -17,22 +17,22 @@ public class FileExtractFunction(
1717
ServiceLayerDbContext serviceLayerDbContext,
1818
IFileTransformQueueClient fileTransformQueueClient,
1919
IFileExtractQueueClient fileExtractQueueClient,
20-
IMeshFilesBlobStore meshFileBlobStore)
20+
IMeshFilesBlobStore meshFileBlobStore) : MeshFileFunctionBase(serviceLayerDbContext)
2121
{
2222
[Function("FileExtractFunction")]
2323
public async Task Run([QueueTrigger("%FileExtractQueueName%")] FileExtractQueueMessage message)
2424
{
2525
logger.LogInformation("{FunctionName} started. Processing fileId: {FileId}", nameof(FileExtractFunction), message.FileId);
2626

27-
await using var transaction = await serviceLayerDbContext.Database.BeginTransactionAsync();
27+
await using var transaction = await ServiceLayerDbContext.Database.BeginTransactionAsync();
2828

2929
var file = await GetFileAsync(message.FileId);
3030
if (file == null || !IsFileSuitableForExtraction(file))
3131
{
3232
return;
3333
}
3434

35-
await UpdateFileStatusForExtraction(file);
35+
await UpdateMeshFile(file, MeshFileStatus.Extracting);
3636
await transaction.CommitAsync();
3737

3838
try
@@ -47,7 +47,7 @@ public async Task Run([QueueTrigger("%FileExtractQueueName%")] FileExtractQueueM
4747

4848
private async Task<MeshFile?> GetFileAsync(string fileId)
4949
{
50-
var file = await serviceLayerDbContext.MeshFiles
50+
var file = await ServiceLayerDbContext.MeshFiles
5151
.FirstOrDefaultAsync(f => f.FileId == fileId);
5252

5353
if (file == null)
@@ -76,13 +76,6 @@ private bool IsFileSuitableForExtraction(MeshFile file)
7676
return true;
7777
}
7878

79-
private async Task UpdateFileStatusForExtraction(MeshFile file)
80-
{
81-
file.Status = MeshFileStatus.Extracting;
82-
file.LastUpdatedUtc = DateTime.UtcNow;
83-
await serviceLayerDbContext.SaveChangesAsync();
84-
}
85-
8679
private async Task ProcessFileExtraction(MeshFile file)
8780
{
8881
var meshResponse = await meshInboxService.GetMessageByIdAsync(file.MailboxId, file.FileId);
@@ -100,19 +93,17 @@ private async Task ProcessFileExtraction(MeshFile file)
10093
}
10194

10295
file.BlobPath = blobPath;
103-
file.Status = MeshFileStatus.Extracted;
104-
file.LastUpdatedUtc = DateTime.UtcNow;
105-
await serviceLayerDbContext.SaveChangesAsync();
96+
await UpdateMeshFile(file, MeshFileStatus.Extracted);
10697

10798
await fileTransformQueueClient.EnqueueFileTransformAsync(file);
10899
}
109100

110101
private async Task HandleExtractionError(MeshFile file, FileExtractQueueMessage message, Exception ex)
111102
{
112103
logger.LogError(ex, "An exception occurred during file extraction for fileId: {FileId}", message.FileId);
113-
file.Status = MeshFileStatus.FailedExtract;
114-
file.LastUpdatedUtc = DateTime.UtcNow;
115-
await serviceLayerDbContext.SaveChangesAsync();
104+
await UpdateMeshFile(file, MeshFileStatus.FailedExtract);
116105
await fileExtractQueueClient.SendToPoisonQueueAsync(message);
117106
}
107+
108+
protected override FileEventSource Source => FileEventSource.ExtractFunction;
118109
}

‎src/ServiceLayer.Mesh/Functions/FileRetryFunction.cs‎

Lines changed: 9 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ public class FileRetryFunction(
1313
ServiceLayerDbContext serviceLayerDbContext,
1414
IFileExtractQueueClient fileExtractQueueClient,
1515
IFileTransformQueueClient fileTransformQueueClient,
16-
IFileRetryFunctionConfiguration configuration)
16+
IFileRetryFunctionConfiguration configuration) : MeshFileFunctionBase(serviceLayerDbContext)
1717
{
1818
[Function("FileRetryFunction")]
1919
public async Task Run([TimerTrigger("%FileRetryTimerExpression%")] TimerInfo myTimer)
@@ -28,7 +28,7 @@ public async Task Run([TimerTrigger("%FileRetryTimerExpression%")] TimerInfo myT
2828

2929
private async Task RetryStaleExtractions(DateTime staleDateTimeUtc)
3030
{
31-
var staleFiles = await serviceLayerDbContext.MeshFiles
31+
var staleFiles = await ServiceLayerDbContext.MeshFiles
3232
.Where(f =>
3333
(f.Status == MeshFileStatus.Discovered || f.Status == MeshFileStatus.Extracting)
3434
&& f.LastUpdatedUtc <= staleDateTimeUtc)
@@ -39,15 +39,15 @@ private async Task RetryStaleExtractions(DateTime staleDateTimeUtc)
3939
foreach (var file in staleFiles)
4040
{
4141
await fileExtractQueueClient.EnqueueFileExtractAsync(file);
42-
file.LastUpdatedUtc = DateTime.UtcNow;
43-
await serviceLayerDbContext.SaveChangesAsync();
42+
await UpdateMeshFile(file, file.Status);
43+
await ServiceLayerDbContext.SaveChangesAsync();
4444
logger.LogInformation("FileRetryFunction: File {FileFileId} enqueued to Extract queue", file.FileId);
4545
}
4646
}
4747

4848
private async Task RetryStaleTransformations(DateTime staleDateTimeUtc)
4949
{
50-
var staleFiles = await serviceLayerDbContext.MeshFiles
50+
var staleFiles = await ServiceLayerDbContext.MeshFiles
5151
.Where(f =>
5252
(f.Status == MeshFileStatus.Extracted || f.Status == MeshFileStatus.Transforming)
5353
&& f.LastUpdatedUtc <= staleDateTimeUtc)
@@ -58,9 +58,11 @@ private async Task RetryStaleTransformations(DateTime staleDateTimeUtc)
5858
foreach (var file in staleFiles)
5959
{
6060
await fileTransformQueueClient.EnqueueFileTransformAsync(file);
61-
file.LastUpdatedUtc = DateTime.UtcNow;
62-
await serviceLayerDbContext.SaveChangesAsync();
61+
await UpdateMeshFile(file, file.Status);
62+
await ServiceLayerDbContext.SaveChangesAsync();
6363
logger.LogInformation("FileRetryFunction: File {FileFileId} enqueued to Transform queue", file.FileId);
6464
}
6565
}
66+
67+
protected override FileEventSource Source => FileEventSource.RetryFunction;
6668
}

‎src/ServiceLayer.Mesh/Functions/FileTransformFunction.cs‎

Lines changed: 8 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ public class FileTransformFunction(
1818
ServiceLayerDbContext serviceLayerDbContext,
1919
IFileTransformQueueClient fileTransformQueueClient,
2020
IMeshFilesBlobStore meshFileBlobStore,
21-
IEnumerable<IFileTransformer> fileTransformers)
21+
IEnumerable<IFileTransformer> fileTransformers) : MeshFileFunctionBase(serviceLayerDbContext)
2222
{
2323
private static readonly JsonSerializerOptions ValidationErrorJsonOptions = new()
2424
{
@@ -34,15 +34,15 @@ public async Task Run([QueueTrigger("%FileTransformQueueName%")] FileTransformQu
3434
logger.LogInformation("{FunctionName} started. Processing fileId: {FileId}", nameof(FileTransformFunction),
3535
message.FileId);
3636

37-
await using var transaction = await serviceLayerDbContext.Database.BeginTransactionAsync();
37+
await using var transaction = await ServiceLayerDbContext.Database.BeginTransactionAsync();
3838

3939
var file = await GetFileAsync(message.FileId);
4040
if (file == null || !IsFileSuitableForTransformation(file))
4141
{
4242
return;
4343
}
4444

45-
await UpdateFileStatusForTransformation(file);
45+
await UpdateMeshFile(file, MeshFileStatus.Transforming);
4646
await transaction.CommitAsync();
4747

4848
try
@@ -57,7 +57,7 @@ public async Task Run([QueueTrigger("%FileTransformQueueName%")] FileTransformQu
5757

5858
private async Task<MeshFile?> GetFileAsync(string fileId)
5959
{
60-
var file = await serviceLayerDbContext.MeshFiles
60+
var file = await ServiceLayerDbContext.MeshFiles
6161
.FirstOrDefaultAsync(f => f.FileId == fileId);
6262

6363
if (file == null)
@@ -87,13 +87,6 @@ private bool IsFileSuitableForTransformation(MeshFile file)
8787
return true;
8888
}
8989

90-
private async Task UpdateFileStatusForTransformation(MeshFile file)
91-
{
92-
file.Status = MeshFileStatus.Transforming;
93-
file.LastUpdatedUtc = DateTime.UtcNow;
94-
await serviceLayerDbContext.SaveChangesAsync();
95-
}
96-
9790
private async Task ProcessFileTransformation(MeshFile file)
9891
{
9992
var transformer = GetTransformerFor(file.FileType);
@@ -107,9 +100,7 @@ private async Task ProcessFileTransformation(MeshFile file)
107100
throw new InvalidOperationException("Validation errors encountered");
108101
}
109102

110-
file.Status = MeshFileStatus.Transformed;
111-
file.LastUpdatedUtc = DateTime.UtcNow;
112-
await serviceLayerDbContext.SaveChangesAsync();
103+
await UpdateMeshFile(file, MeshFileStatus.Transformed);
113104
}
114105

115106
private IFileTransformer GetTransformerFor(MeshFileType type)
@@ -129,9 +120,9 @@ private IFileTransformer GetTransformerFor(MeshFileType type)
129120
private async Task HandleTransformationError(MeshFile file, FileTransformQueueMessage message, Exception ex)
130121
{
131122
logger.LogError(ex, "An exception occurred during file transformation for fileId: {FileId}", message.FileId);
132-
file.Status = MeshFileStatus.FailedTransform;
133-
file.LastUpdatedUtc = DateTime.UtcNow;
134-
await serviceLayerDbContext.SaveChangesAsync();
123+
await UpdateMeshFile(file, MeshFileStatus.FailedTransform);
135124
await fileTransformQueueClient.SendToPoisonQueueAsync(message);
136125
}
126+
127+
protected override FileEventSource Source => FileEventSource.TransformFunction;
137128
}

‎src/ServiceLayer.Mesh/Functions/MeshFileFunctionBase.cs‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
using ServiceLayer.Data;
2+
using ServiceLayer.Data.Models;
3+
4+
namespace ServiceLayer.Mesh.Functions;
5+
6+
public abstract class MeshFileFunctionBase(ServiceLayerDbContext serviceLayerDbContext)
7+
{
8+
protected ServiceLayerDbContext ServiceLayerDbContext { get; } = serviceLayerDbContext;
9+
10+
protected abstract FileEventSource Source { get; }
11+
12+
protected async Task UpdateMeshFile(MeshFile file, MeshFileStatus status)
13+
{
14+
var now = DateTime.UtcNow;
15+
16+
file.Status = status;
17+
file.LastUpdatedUtc = now;
18+
19+
var fileEvent = new MeshFileEvent
20+
{
21+
FileId = file.FileId,
22+
Status = status,
23+
TimestampUtc = now,
24+
Source = Source
25+
};
26+
27+
ServiceLayerDbContext.MeshFileEvents.Add(fileEvent);
28+
29+
await ServiceLayerDbContext.SaveChangesAsync();
30+
}
31+
}

0 commit comments

Comments
 (0)