Skip to content
This repository was archived by the owner on Jul 28, 2025. It is now read-only.

Commit 111f22d

Browse files
ianfnelsonianfnelson
committed
WIP
1 parent b07546d commit 111f22d

File tree

10 files changed

+89
-73
lines changed

10 files changed

+89
-73
lines changed

‎src/ServiceLayer.Mesh/Functions/FileDiscoveryFunction.cs‎

Lines changed: 28 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ public class FileDiscoveryFunction(
1515
IMeshInboxService meshInboxService,
1616
ServiceLayerDbContext serviceLayerDbContext,
1717
IFileExtractQueueClient fileExtractQueueClient)
18+
: MeshFileFunctionBase(serviceLayerDbContext)
1819
{
1920
[Function("FileDiscoveryFunction")]
2021
public async Task Run([TimerTrigger("%FileDiscoveryTimerExpression%")] TimerInfo myTimer)
@@ -26,28 +27,16 @@ public async Task Run([TimerTrigger("%FileDiscoveryTimerExpression%")] TimerInfo
2627
// TODO - check if response.IsSuccessful before proceeding to dereference the Response.Messages
2728
foreach (var messageId in response.Response.Messages)
2829
{
29-
await using var transaction = await serviceLayerDbContext.Database.BeginTransactionAsync();
30+
await using var transaction = await ServiceLayerDbContext.Database.BeginTransactionAsync();
3031

31-
var existing = await serviceLayerDbContext.MeshFiles
32+
var existing = await ServiceLayerDbContext.MeshFiles
3233
.AnyAsync(f => f.FileId == messageId);
3334

3435
if (!existing)
3536
{
36-
var file = new MeshFile
37-
{
38-
FileId = messageId,
39-
FileType = MeshFileType.NbssAppointmentEvents,
40-
MailboxId = configuration.NbssMeshMailboxId,
41-
Status = MeshFileStatus.Discovered,
42-
FirstSeenUtc = DateTime.UtcNow,
43-
LastUpdatedUtc = DateTime.UtcNow
44-
};
45-
46-
serviceLayerDbContext.MeshFiles.Add(file);
47-
48-
await serviceLayerDbContext.SaveChangesAsync();
49-
await transaction.CommitAsync();
37+
var file = await CreateMeshFile(messageId);
5038

39+
await transaction.CommitAsync();
5140
await fileExtractQueueClient.EnqueueFileExtractAsync(file);
5241
}
5342
else
@@ -56,4 +45,27 @@ public async Task Run([TimerTrigger("%FileDiscoveryTimerExpression%")] TimerInfo
5645
}
5746
}
5847
}
48+
49+
private async Task<MeshFile> CreateMeshFile(string messageId)
50+
{
51+
var now = DateTime.UtcNow;
52+
53+
var file = new MeshFile
54+
{
55+
FileId = messageId,
56+
FileType = MeshFileType.NbssAppointmentEvents,
57+
MailboxId = configuration.NbssMeshMailboxId,
58+
Status = MeshFileStatus.Discovered,
59+
FirstSeenUtc = now,
60+
LastUpdatedUtc = now
61+
};
62+
63+
ServiceLayerDbContext.MeshFiles.Add(file);
64+
65+
await UpdateFileStatus(file, MeshFileStatus.Discovered);
66+
67+
return file;
68+
}
69+
70+
protected override FileEventSource Source => FileEventSource.DiscoveryFunction;
5971
}

‎src/ServiceLayer.Mesh/Functions/FileExtractFunction.cs‎

Lines changed: 8 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -17,22 +17,22 @@ public class FileExtractFunction(
1717
ServiceLayerDbContext serviceLayerDbContext,
1818
IFileTransformQueueClient fileTransformQueueClient,
1919
IFileExtractQueueClient fileExtractQueueClient,
20-
IMeshFilesBlobStore meshFileBlobStore)
20+
IMeshFilesBlobStore meshFileBlobStore) : MeshFileFunctionBase(serviceLayerDbContext)
2121
{
2222
[Function("FileExtractFunction")]
2323
public async Task Run([QueueTrigger("%FileExtractQueueName%")] FileExtractQueueMessage message)
2424
{
2525
logger.LogInformation("{FunctionName} started. Processing fileId: {FileId}", nameof(FileExtractFunction), message.FileId);
2626

27-
await using var transaction = await serviceLayerDbContext.Database.BeginTransactionAsync();
27+
await using var transaction = await ServiceLayerDbContext.Database.BeginTransactionAsync();
2828

2929
var file = await GetFileAsync(message.FileId);
3030
if (file == null || !IsFileSuitableForExtraction(file))
3131
{
3232
return;
3333
}
3434

35-
await UpdateFileStatusForExtraction(file);
35+
await UpdateFileStatus(file, MeshFileStatus.Extracting);
3636
await transaction.CommitAsync();
3737

3838
try
@@ -47,7 +47,7 @@ public async Task Run([QueueTrigger("%FileExtractQueueName%")] FileExtractQueueM
4747

4848
private async Task<MeshFile?> GetFileAsync(string fileId)
4949
{
50-
var file = await serviceLayerDbContext.MeshFiles
50+
var file = await ServiceLayerDbContext.MeshFiles
5151
.FirstOrDefaultAsync(f => f.FileId == fileId);
5252

5353
if (file == null)
@@ -76,13 +76,6 @@ private bool IsFileSuitableForExtraction(MeshFile file)
7676
return true;
7777
}
7878

79-
private async Task UpdateFileStatusForExtraction(MeshFile file)
80-
{
81-
file.Status = MeshFileStatus.Extracting;
82-
file.LastUpdatedUtc = DateTime.UtcNow;
83-
await serviceLayerDbContext.SaveChangesAsync();
84-
}
85-
8679
private async Task ProcessFileExtraction(MeshFile file)
8780
{
8881
var meshResponse = await meshInboxService.GetMessageByIdAsync(file.MailboxId, file.FileId);
@@ -100,19 +93,17 @@ private async Task ProcessFileExtraction(MeshFile file)
10093
}
10194

10295
file.BlobPath = blobPath;
103-
file.Status = MeshFileStatus.Extracted;
104-
file.LastUpdatedUtc = DateTime.UtcNow;
105-
await serviceLayerDbContext.SaveChangesAsync();
96+
await UpdateFileStatus(file, MeshFileStatus.Extracted);
10697

10798
await fileTransformQueueClient.EnqueueFileTransformAsync(file);
10899
}
109100

110101
private async Task HandleExtractionError(MeshFile file, FileExtractQueueMessage message, Exception ex)
111102
{
112103
logger.LogError(ex, "An exception occurred during file extraction for fileId: {FileId}", message.FileId);
113-
file.Status = MeshFileStatus.FailedExtract;
114-
file.LastUpdatedUtc = DateTime.UtcNow;
115-
await serviceLayerDbContext.SaveChangesAsync();
104+
await UpdateFileStatus(file, MeshFileStatus.FailedExtract);
116105
await fileExtractQueueClient.SendToPoisonQueueAsync(message);
117106
}
107+
108+
protected override FileEventSource Source => FileEventSource.ExtractFunction;
118109
}

‎src/ServiceLayer.Mesh/Functions/FileRetryFunction.cs‎

Lines changed: 9 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ public class FileRetryFunction(
1313
ServiceLayerDbContext serviceLayerDbContext,
1414
IFileExtractQueueClient fileExtractQueueClient,
1515
IFileTransformQueueClient fileTransformQueueClient,
16-
IFileRetryFunctionConfiguration configuration)
16+
IFileRetryFunctionConfiguration configuration) : MeshFileFunctionBase(serviceLayerDbContext)
1717
{
1818
[Function("FileRetryFunction")]
1919
public async Task Run([TimerTrigger("%FileRetryTimerExpression%")] TimerInfo myTimer)
@@ -28,7 +28,7 @@ public async Task Run([TimerTrigger("%FileRetryTimerExpression%")] TimerInfo myT
2828

2929
private async Task RetryStaleExtractions(DateTime staleDateTimeUtc)
3030
{
31-
var staleFiles = await serviceLayerDbContext.MeshFiles
31+
var staleFiles = await ServiceLayerDbContext.MeshFiles
3232
.Where(f =>
3333
(f.Status == MeshFileStatus.Discovered || f.Status == MeshFileStatus.Extracting)
3434
&& f.LastUpdatedUtc <= staleDateTimeUtc)
@@ -39,15 +39,15 @@ private async Task RetryStaleExtractions(DateTime staleDateTimeUtc)
3939
foreach (var file in staleFiles)
4040
{
4141
await fileExtractQueueClient.EnqueueFileExtractAsync(file);
42-
file.LastUpdatedUtc = DateTime.UtcNow;
43-
await serviceLayerDbContext.SaveChangesAsync();
42+
await UpdateFileStatus(file, file.Status);
43+
await ServiceLayerDbContext.SaveChangesAsync();
4444
logger.LogInformation("FileRetryFunction: File {FileFileId} enqueued to Extract queue", file.FileId);
4545
}
4646
}
4747

4848
private async Task RetryStaleTransformations(DateTime staleDateTimeUtc)
4949
{
50-
var staleFiles = await serviceLayerDbContext.MeshFiles
50+
var staleFiles = await ServiceLayerDbContext.MeshFiles
5151
.Where(f =>
5252
(f.Status == MeshFileStatus.Extracted || f.Status == MeshFileStatus.Transforming)
5353
&& f.LastUpdatedUtc <= staleDateTimeUtc)
@@ -58,9 +58,11 @@ private async Task RetryStaleTransformations(DateTime staleDateTimeUtc)
5858
foreach (var file in staleFiles)
5959
{
6060
await fileTransformQueueClient.EnqueueFileTransformAsync(file);
61-
file.LastUpdatedUtc = DateTime.UtcNow;
62-
await serviceLayerDbContext.SaveChangesAsync();
61+
await UpdateFileStatus(file, file.Status);
62+
await ServiceLayerDbContext.SaveChangesAsync();
6363
logger.LogInformation("FileRetryFunction: File {FileFileId} enqueued to Transform queue", file.FileId);
6464
}
6565
}
66+
67+
protected override FileEventSource Source => FileEventSource.RetryFunction;
6668
}

‎src/ServiceLayer.Mesh/Functions/FileTransformFunction.cs‎

Lines changed: 8 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ public class FileTransformFunction(
1818
ServiceLayerDbContext serviceLayerDbContext,
1919
IFileTransformQueueClient fileTransformQueueClient,
2020
IMeshFilesBlobStore meshFileBlobStore,
21-
IEnumerable<IFileTransformer> fileTransformers)
21+
IEnumerable<IFileTransformer> fileTransformers) : MeshFileFunctionBase(serviceLayerDbContext)
2222
{
2323
private static readonly JsonSerializerOptions ValidationErrorJsonOptions = new()
2424
{
@@ -34,15 +34,15 @@ public async Task Run([QueueTrigger("%FileTransformQueueName%")] FileTransformQu
3434
logger.LogInformation("{FunctionName} started. Processing fileId: {FileId}", nameof(FileTransformFunction),
3535
message.FileId);
3636

37-
await using var transaction = await serviceLayerDbContext.Database.BeginTransactionAsync();
37+
await using var transaction = await ServiceLayerDbContext.Database.BeginTransactionAsync();
3838

3939
var file = await GetFileAsync(message.FileId);
4040
if (file == null || !IsFileSuitableForTransformation(file))
4141
{
4242
return;
4343
}
4444

45-
await UpdateFileStatusForTransformation(file);
45+
await UpdateFileStatus(file, MeshFileStatus.Transforming);
4646
await transaction.CommitAsync();
4747

4848
try
@@ -57,7 +57,7 @@ public async Task Run([QueueTrigger("%FileTransformQueueName%")] FileTransformQu
5757

5858
private async Task<MeshFile?> GetFileAsync(string fileId)
5959
{
60-
var file = await serviceLayerDbContext.MeshFiles
60+
var file = await ServiceLayerDbContext.MeshFiles
6161
.FirstOrDefaultAsync(f => f.FileId == fileId);
6262

6363
if (file == null)
@@ -87,13 +87,6 @@ private bool IsFileSuitableForTransformation(MeshFile file)
8787
return true;
8888
}
8989

90-
private async Task UpdateFileStatusForTransformation(MeshFile file)
91-
{
92-
file.Status = MeshFileStatus.Transforming;
93-
file.LastUpdatedUtc = DateTime.UtcNow;
94-
await serviceLayerDbContext.SaveChangesAsync();
95-
}
96-
9790
private async Task ProcessFileTransformation(MeshFile file)
9891
{
9992
var transformer = GetTransformerFor(file.FileType);
@@ -107,9 +100,7 @@ private async Task ProcessFileTransformation(MeshFile file)
107100
throw new InvalidOperationException("Validation errors encountered");
108101
}
109102

110-
file.Status = MeshFileStatus.Transformed;
111-
file.LastUpdatedUtc = DateTime.UtcNow;
112-
await serviceLayerDbContext.SaveChangesAsync();
103+
await UpdateFileStatus(file, MeshFileStatus.Transformed);
113104
}
114105

115106
private IFileTransformer GetTransformerFor(MeshFileType type)
@@ -129,9 +120,9 @@ private IFileTransformer GetTransformerFor(MeshFileType type)
129120
private async Task HandleTransformationError(MeshFile file, FileTransformQueueMessage message, Exception ex)
130121
{
131122
logger.LogError(ex, "An exception occurred during file transformation for fileId: {FileId}", message.FileId);
132-
file.Status = MeshFileStatus.FailedTransform;
133-
file.LastUpdatedUtc = DateTime.UtcNow;
134-
await serviceLayerDbContext.SaveChangesAsync();
123+
await UpdateFileStatus(file, MeshFileStatus.FailedTransform);
135124
await fileTransformQueueClient.SendToPoisonQueueAsync(message);
136125
}
126+
127+
protected override FileEventSource Source => FileEventSource.TransformFunction;
137128
}

‎src/ServiceLayer.Mesh/Functions/MeshFileFunctionBase.cs‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
using ServiceLayer.Data;
2+
using ServiceLayer.Data.Models;
3+
4+
namespace ServiceLayer.Mesh.Functions;
5+
6+
public abstract class MeshFileFunctionBase(ServiceLayerDbContext serviceLayerDbContext)
7+
{
8+
protected ServiceLayerDbContext ServiceLayerDbContext { get; } = serviceLayerDbContext;
9+
10+
protected abstract FileEventSource Source { get; }
11+
12+
protected async Task UpdateFileStatus(MeshFile file, MeshFileStatus status)
13+
{
14+
var now = DateTime.UtcNow;
15+
16+
file.Status = status;
17+
file.LastUpdatedUtc = now;
18+
19+
var fileEvent = new MeshFileEvent
20+
{
21+
FileId = file.FileId,
22+
Status = status,
23+
TimestampUtc = now,
24+
Source = Source
25+
};
26+
27+
ServiceLayerDbContext.MeshFileEvents.Add(fileEvent);
28+
29+
await ServiceLayerDbContext.SaveChangesAsync();
30+
}
31+
}

‎src/ServiceLayer.Mesh/Program.cs‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,6 @@
66
using ServiceLayer.Mesh.Messaging;
77
using ServiceLayer.Mesh.Storage;
88
using ServiceLayer.Mesh.FileTypes.NbssAppointmentEvents;
9-
using ServiceLayer.Mesh.Functions;
109

1110
var host = new HostBuilder()
1211
.ConfigureFunctionsWebApplication()

‎src/ServiceLayer.Shared/Data/Migrations/ServiceLayerDbContextModelSnapshot.cs‎

Lines changed: 1 addition & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -72,12 +72,7 @@ protected override void BuildModel(ModelBuilder modelBuilder)
7272
.HasMaxLength(255)
7373
.HasColumnType("nvarchar(255)");
7474

75-
b.Property<string>("NewStatus")
76-
.IsRequired()
77-
.HasMaxLength(20)
78-
.HasColumnType("nvarchar(20)");
79-
80-
b.Property<string>("OldStatus")
75+
b.Property<string>("Status")
8176
.IsRequired()
8277
.HasMaxLength(20)
8378
.HasColumnType("nvarchar(20)");

‎src/ServiceLayer.Shared/Data/Models/MeshFileEvent.cs‎

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -14,10 +14,7 @@ public class MeshFileEvent
1414
public required DateTime TimestampUtc { get; set; }
1515

1616
[MaxLength(20)]
17-
public MeshFileStatus OldStatus { get; set; }
18-
19-
[MaxLength(20)]
20-
public required MeshFileStatus NewStatus { get; set; }
17+
public required MeshFileStatus Status { get; set; }
2118

2219
[MaxLength(20)]
2320
public required FileEventSource Source { get; set; }

‎src/ServiceLayer.Shared/Data/ServiceLayerDbContext.cs‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,8 +26,7 @@ private static void ConfigureMeshFiles(ModelBuilder modelBuilder)
2626
private static void ConfigureMeshFileEvents(ModelBuilder modelBuilder)
2727
{
2828
modelBuilder.Entity<MeshFileEvent>().HasKey(e => e.EventId);
29-
modelBuilder.Entity<MeshFileEvent>().Property(e => e.NewStatus).HasConversion<string>();
30-
modelBuilder.Entity<MeshFileEvent>().Property(e => e.OldStatus).HasConversion<string>();
29+
modelBuilder.Entity<MeshFileEvent>().Property(e => e.Status).HasConversion<string>();
3130
modelBuilder.Entity<MeshFileEvent>().Property(e => e.Source).HasConversion<string>();
3231
modelBuilder.Entity<MeshFileEvent>()
3332
.HasOne<MeshFile>()

‎tests/ServiceLayer.Mesh.Tests/Functions/FunctionTestBase.cs‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,4 @@
11
using Microsoft.EntityFrameworkCore;
2-
using Microsoft.EntityFrameworkCore.Update;
32
using Microsoft.Extensions.Logging;
43
using Moq;
54
using ServiceLayer.Data;
@@ -40,7 +39,7 @@ protected MeshFile SaveMeshFile(MeshFileStatus status = MeshFileStatus.Extracted
4039
var fileEvent = new MeshFileEvent
4140
{
4241
FileId = fileId,
43-
NewStatus = status,
42+
Status = status,
4443
TimestampUtc = lastUpdated,
4544
Source = FileEventSource.DiscoveryFunction
4645
};
@@ -72,7 +71,7 @@ protected MeshFile AssertFileUpdated(string fileId, MeshFileStatus expectedStatu
7271
.OrderByDescending(x => x.TimestampUtc)
7372
.First();
7473

75-
Assert.Equal(expectedStatus, lastEvent.NewStatus);
74+
Assert.Equal(expectedStatus, lastEvent.Status);
7675
Assert.True(lastEvent.TimestampUtc > DateTime.UtcNow.AddSeconds(-10));
7776
Assert.Equal(expectedSource, lastEvent.Source);
7877

0 commit comments

Comments
 (0)