feat: retry function

.env.example

@@ -12,11 +12,13 @@ MeshPassword=password
 NbssMailboxId=X26ABC1
 MeshApiBaseUrl=http://localhost:8700/messageexchange
 ASPNETCORE_ENVIRONMENT=Development
-FileDiscoveryTimerExpression=*/5 * * * *
+FileDiscoveryTimerExpression=0 */5 * * * *
 MeshHandshakeTimerExpression=0 0 0 * * * # Midnight
+FileRetryTimerExpression=0 0 * * * *
 QueueUrl=http://127.0.0.1:10001
 FileExtractQueueName=file-extract
 FileTransformQueueName=file-transform
+StaleHours=12
 
 # API Configuration
 API_PORT=7071

src/ServiceLayer.Mesh/Configuration/AppConfiguration.cs

@@ -5,6 +5,7 @@ public class AppConfiguration :
     IFileExtractFunctionConfiguration,
     IFileExtractQueueClientConfiguration,
     IFileTransformQueueClientConfiguration,
+    IFileRetryFunctionConfiguration,
     IMeshHandshakeFunctionConfiguration
 {
     public string NbssMeshMailboxId => GetRequired("NbssMailboxId");
@@ -13,6 +14,8 @@ public class AppConfiguration :
 
     public string FileTransformQueueName => GetRequired("FileTransformQueueName");
 
+    public int StaleHours => GetRequiredInt("StaleHours");
+
     private static string GetRequired(string key)
     {
         var value = Environment.GetEnvironmentVariable(key);
@@ -24,4 +27,16 @@ private static string GetRequired(string key)
 
         return value;
     }
+
+    private static int GetRequiredInt(string key)
+    {
+        var value = GetRequired(key);
+
+        if (!int.TryParse(value, out var intValue))
+        {
+            throw new InvalidOperationException($"Environment variable '{key}' is not a valid integer");
+        }
+
+        return intValue;
+    }
 }

src/ServiceLayer.Mesh/Configuration/IFileRetryFunctionConfiguration.cs

@@ -0,0 +1,6 @@
+namespace ServiceLayer.Mesh.Configuration;
+
+public interface IFileRetryFunctionConfiguration
+{
+    int StaleHours { get; }
+}

src/ServiceLayer.Mesh/Functions/FileRetryFunction.cs

@@ -1,6 +1,67 @@
+using Microsoft.Azure.Functions.Worker;
+using Microsoft.EntityFrameworkCore;
+using Microsoft.Extensions.Logging;
+using ServiceLayer.Mesh.Data;
+using ServiceLayer.Mesh.Messaging;
+using ServiceLayer.Mesh.Models;
+using ServiceLayer.Mesh.Configuration;
+
 namespace ServiceLayer.Mesh.Functions;
 
-public class FileRetryFunction
+public class FileRetryFunction(
+    ILogger<FileRetryFunction> logger,
+    ServiceLayerDbContext serviceLayerDbContext,
+    IFileExtractQueueClient fileExtractQueueClient,
+    IFileTransformQueueClient fileTransformQueueClient,
+    IFileRetryFunctionConfiguration configuration)
 {
+    [Function("FileRetryFunction")]
+    public async Task Run([TimerTrigger("%FileRetryTimerExpression%")] TimerInfo myTimer)
+    {
+        logger.LogInformation("FileRetryFunction started");
+
+        var staleDateTimeUtc = DateTime.UtcNow.AddHours(-configuration.StaleHours);
+
+        await Task.WhenAll(
+            RetryStaleExtractions(staleDateTimeUtc),
+            RetryStaleTransformations(staleDateTimeUtc));
+    }
+
+    private async Task RetryStaleExtractions(DateTime staleDateTimeUtc)
+    {
+        var staleFiles = await serviceLayerDbContext.MeshFiles
+            .Where(f =>
+                (f.Status == MeshFileStatus.Discovered || f.Status == MeshFileStatus.Extracting)
+                && f.LastUpdatedUtc <= staleDateTimeUtc)
+            .ToListAsync();
+
+        logger.LogInformation($"FileRetryFunction: {staleFiles.Count} stale files found for extraction retry");
+
+        foreach (var file in staleFiles)
+        {
+            await fileExtractQueueClient.EnqueueFileExtractAsync(file);
+            file.LastUpdatedUtc = DateTime.UtcNow;
+            await serviceLayerDbContext.SaveChangesAsync();
+            logger.LogInformation($"FileRetryFunction: File {file.FileId} enqueued to Extract queue");
+        }
+    }
+
+    private async Task RetryStaleTransformations(DateTime staleDateTimeUtc)
+    {
+        var staleFiles = await serviceLayerDbContext.MeshFiles
+            .Where(f =>
+                (f.Status == MeshFileStatus.Extracted || f.Status == MeshFileStatus.Transforming)
+                && f.LastUpdatedUtc <= staleDateTimeUtc)
+            .ToListAsync();
+
+        logger.LogInformation($"FileRetryFunction: {staleFiles.Count} stale files found for transforming retry");
 
+        foreach (var file in staleFiles)
+        {
+            await fileTransformQueueClient.EnqueueFileTransformAsync(file);
+            file.LastUpdatedUtc = DateTime.UtcNow;
+            await serviceLayerDbContext.SaveChangesAsync();
+            logger.LogInformation($"FileRetryFunction: File {file.FileId} enqueued to Transform queue");
+        }
+    }
 }

src/ServiceLayer.Mesh/Functions/host.json

@@ -0,0 +1,7 @@
+{
+  "version": "2.0",
+  "extensionBundle": {
+    "id": "Microsoft.Azure.Functions.ExtensionBundle",
+    "version": "[4.*, 5.0.0)"
+  }
+}

src/ServiceLayer.Mesh/Program.cs

@@ -66,6 +66,7 @@
         services.AddTransient<IFileExtractQueueClientConfiguration, AppConfiguration>();
         services.AddTransient<IFileTransformQueueClientConfiguration, AppConfiguration>();
         services.AddTransient<IMeshHandshakeFunctionConfiguration, AppConfiguration>();
+        services.AddTransient<IFileRetryFunctionConfiguration, AppConfiguration>();
     });
 
 

tests/ServiceLayer.Mesh.Tests/Functions/FileRetryFunctionTests.cs

@@ -0,0 +1,218 @@
+using Microsoft.EntityFrameworkCore;
+using Microsoft.Extensions.Logging;
+using Moq;
+using NHS.MESH.Client.Contracts.Services;
+using ServiceLayer.Mesh.Data;
+using ServiceLayer.Mesh.Functions;
+using ServiceLayer.Mesh.Messaging;
+using ServiceLayer.Mesh.Models;
+using ServiceLayer.Mesh.Configuration;
+
+namespace ServiceLayer.Mesh.Tests.Functions;
+
+public class FileRetryFunctionTests
+{
+    private readonly Mock<ILogger<FileRetryFunction>> _loggerMock;
+    private readonly Mock<IFileExtractQueueClient> _fileExtractQueueClientMock;
+    private readonly Mock<IFileTransformQueueClient> _fileTransformQueueClientMock;
+    private readonly Mock<IFileRetryFunctionConfiguration> _configuration;
+    private readonly ServiceLayerDbContext _dbContext;
+    private readonly FileRetryFunction _function;
+
+    public FileRetryFunctionTests()
+    {
+        _loggerMock = new Mock<ILogger<FileRetryFunction>>();
+        _fileExtractQueueClientMock = new Mock<IFileExtractQueueClient>();
+        _fileTransformQueueClientMock = new Mock<IFileTransformQueueClient>();
+        _configuration = new Mock<IFileRetryFunctionConfiguration>();
+
+        var options = new DbContextOptionsBuilder<ServiceLayerDbContext>()
+            .UseInMemoryDatabase(databaseName: Guid.NewGuid().ToString())
+            .ConfigureWarnings(warnings =>
+                warnings.Ignore(Microsoft.EntityFrameworkCore.Diagnostics.InMemoryEventId.TransactionIgnoredWarning))
+            .Options;
+
+        _dbContext = new ServiceLayerDbContext(options);
+
+        _configuration.Setup(c => c.StaleHours).Returns(12);
+
+        _function = new FileRetryFunction(
+            _loggerMock.Object,
+            _dbContext,
+            _fileExtractQueueClientMock.Object,
+            _fileTransformQueueClientMock.Object,
+            _configuration.Object
+        );
+    }
+
+    [Theory]
+    [InlineData(MeshFileStatus.Discovered)]
+    [InlineData(MeshFileStatus.Extracting)]
+    public async Task Run_EnqueuesDiscoveredOrExtractingFilesOlderThan12Hours(MeshFileStatus testStatus)
+    {
+        var file = new MeshFile
+        {
+            FileType = MeshFileType.NbssAppointmentEvents,
+            MailboxId = "test-mailbox",
+            FileId = "file-1",
+            Status = testStatus,
+            LastUpdatedUtc = DateTime.UtcNow.AddHours(-13)
+        };
+
+        _dbContext.MeshFiles.Add(file);
+        await _dbContext.SaveChangesAsync();
+
+        // Act
+        await _function.Run(null);
+
+        // Assert
+        _fileExtractQueueClientMock.Verify(q => q.EnqueueFileExtractAsync(It.Is<MeshFile>(f => f.FileId == "file-1")), Times.Once);
+        _fileTransformQueueClientMock.Verify(q => q.EnqueueFileTransformAsync(It.Is<MeshFile>(f => f.FileId == "file-1")), Times.Never);
+
+        var updatedFile = await _dbContext.MeshFiles.FindAsync("file-1");
+        Assert.True(updatedFile!.LastUpdatedUtc > DateTime.UtcNow.AddMinutes(-1));
+    }
+
+    [Theory]
+    [InlineData(MeshFileStatus.Extracted)]
+    [InlineData(MeshFileStatus.Transforming)]
+    public async Task Run_EnqueuesExtractedOrTransformingFilesOlderThan12Hours(MeshFileStatus testStatus)
+    {
+        var file = new MeshFile
+        {
+            FileType = MeshFileType.NbssAppointmentEvents,
+            MailboxId = "test-mailbox",
+            FileId = "file-1",
+            Status = testStatus,
+            LastUpdatedUtc = DateTime.UtcNow.AddHours(-13)
+        };
+
+        _dbContext.MeshFiles.Add(file);
+        await _dbContext.SaveChangesAsync();
+
+        // Act
+        await _function.Run(null);
+
+        // Assert
+        _fileTransformQueueClientMock.Verify(q => q.EnqueueFileTransformAsync(It.Is<MeshFile>(f => f.FileId == "file-1")), Times.Once);
+        _fileExtractQueueClientMock.Verify(q => q.EnqueueFileExtractAsync(It.Is<MeshFile>(f => f.FileId == "file-1")), Times.Never);
+
+
+        var updatedFile = await _dbContext.MeshFiles.FindAsync("file-1");
+        Assert.True(updatedFile!.LastUpdatedUtc > DateTime.UtcNow.AddMinutes(-1));
+    }
+
+    [Theory]
+    [InlineData(MeshFileStatus.Discovered)]
+    [InlineData(MeshFileStatus.Extracting)]
+    [InlineData(MeshFileStatus.Extracted)]
+    [InlineData(MeshFileStatus.Transforming)]
+    public async Task Run_SkipsFreshFiles(MeshFileStatus testStatus)
+    {
+        // Arrange
+        var lastUpdatedUtc = DateTime.UtcNow.AddHours(-1);
+
+        var file = new MeshFile
+        {
+            FileType = MeshFileType.NbssAppointmentEvents,
+            MailboxId = "test-mailbox",
+            FileId = "file-2",
+            Status = testStatus,
+            LastUpdatedUtc = lastUpdatedUtc
+        };
+        _dbContext.MeshFiles.Add(file);
+        await _dbContext.SaveChangesAsync();
+
+        // Act
+        await _function.Run(null);
+
+        // Assert
+        _fileExtractQueueClientMock.Verify(q => q.EnqueueFileExtractAsync(It.IsAny<MeshFile>()), Times.Never);
+        _fileTransformQueueClientMock.Verify(q => q.EnqueueFileTransformAsync(It.IsAny<MeshFile>()), Times.Never);
+
+        var updatedFile = await _dbContext.MeshFiles.FindAsync("file-2");
+        Assert.True(updatedFile!.LastUpdatedUtc == lastUpdatedUtc);
+    }
+
+    [Fact]
+    public async Task Run_IgnoresFilesInOtherStatuses()
+    {
+        // Arrange
+        var file = new MeshFile
+        {
+            FileType = MeshFileType.NbssAppointmentEvents,
+            MailboxId = "test-mailbox",
+            FileId = "file-5",
+            Status = MeshFileStatus.Transformed,
+            LastUpdatedUtc = DateTime.UtcNow.AddHours(-20)
+        };
+        _dbContext.MeshFiles.Add(file);
+        await _dbContext.SaveChangesAsync();
+
+        // Act
+        await _function.Run(null);
+
+        // Assert
+        _fileExtractQueueClientMock.Verify(q => q.EnqueueFileExtractAsync(It.IsAny<MeshFile>()), Times.Never);
+    }
+
+    [Fact]
+    public async Task Run_IfNoFilesFoundDoNothing()
+    {
+        // Act
+        await _function.Run(null);
+
+        // Assert
+        _fileExtractQueueClientMock.Verify(q => q.EnqueueFileExtractAsync(It.IsAny<MeshFile>()), Times.Never);
+        _fileTransformQueueClientMock.Verify(q => q.EnqueueFileTransformAsync(It.IsAny<MeshFile>()), Times.Never);
+    }
+
+    [Fact]
+    public async Task Run_ProcessesMultipleEligibleFiles()
+    {
+        // Arrange
+        var files = new[]
+        {
+            new MeshFile
+            {
+                FileType = MeshFileType.NbssAppointmentEvents,
+                MailboxId = "test-mailbox",
+                FileId = "file-6",
+                Status = MeshFileStatus.Discovered,
+                LastUpdatedUtc = DateTime.UtcNow.AddHours(-13)
+            },
+            new MeshFile
+            {
+                FileType = MeshFileType.NbssAppointmentEvents,
+                MailboxId = "test-mailbox",
+                FileId = "file-7",
+                Status = MeshFileStatus.Extracted,
+                LastUpdatedUtc = DateTime.UtcNow.AddHours(-13)
+            },
+            new MeshFile
+            {
+                FileType = MeshFileType.NbssAppointmentEvents,
+                MailboxId = "test-mailbox",
+                FileId = "file-8",
+                Status = MeshFileStatus.Transforming,
+                LastUpdatedUtc = DateTime.UtcNow.AddHours(-13)
+            }
+        };
+
+        _dbContext.MeshFiles.AddRange(files);
+        await _dbContext.SaveChangesAsync();
+
+        // Act
+        await _function.Run(null);
+
+        // Assert
+        _fileExtractQueueClientMock.Verify(q => q.EnqueueFileExtractAsync(It.Is<MeshFile>(f => f.FileId == "file-6")), Times.Once);
+        _fileExtractQueueClientMock.Verify(q => q.EnqueueFileExtractAsync(It.IsAny<MeshFile>()), Times.Once);
+
+        foreach (var fileId in new[] { "file-6", "file-7", "file-8" })
+        {
+            var updated = await _dbContext.MeshFiles.FindAsync(fileId);
+            Assert.True(updated!.LastUpdatedUtc > DateTime.UtcNow.AddMinutes(-1));
+        }
+    }
+}