fix: adding automatic retries to durable function (#1706)

* fix: adding automatic retries to durrable function

* fix: fixing unit tests

* fix: fixing sonar cube issue

* fix: changing MaxRetryCount to a private varible

* fix: adding MaxRetryCount to config and cod clean up

Author: SamRobinson75684

application/CohortManager/compose.core.yaml

@@ -337,6 +337,7 @@ services:
       - ExceptionFunctionURL=http://create-exception:7070/api/CreateException
       - AzureWebJobsStorage=${AZURITE_CONNECTION_STRING}
       - AcceptableLatencyThresholdMs=500
+      - MaxRetryCount=3
 
   # Service Now Integration Service
   servicenow-message-handler:

application/CohortManager/src/Functions/DemographicServices/DemographicDurableFunction/DemographicDurableFunctionConfig.cs

@@ -1,9 +1,12 @@
-namespace NHS.Screening.DemographicDurableFunction;
+namespace NHS.CohortManager.DemographicServices;
 
 using System.ComponentModel.DataAnnotations;
 
 public class DemographicDurableFunctionConfig
 {
     [Required]
-    public string DemographicDataServiceURL {get; set;}
+    public string DemographicDataServiceURL { get; set; }
+
+    [Required]
+    public int MaxRetryCount { get; set; }
 }

application/CohortManager/src/Functions/DemographicServices/DemographicDurableFunction/DurableDemographicFunction.cs

@@ -1,16 +1,17 @@
 namespace NHS.CohortManager.DemographicServices;
 
 using System.Net;
+
 using System.Text;
 using System.Text.Json;
 using Common;
 using DataServices.Client;
-using Microsoft.AspNetCore.Components;
 using Microsoft.Azure.Functions.Worker;
 using Microsoft.Azure.Functions.Worker.Http;
 using Microsoft.DurableTask;
 using Microsoft.DurableTask.Client;
 using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
 using Model;
 
 public class DurableDemographicFunction
@@ -19,12 +20,15 @@ public class DurableDemographicFunction
     private readonly ILogger<DurableDemographicFunction> _logger;
     private readonly ICreateResponse _createResponse;
 
+    private readonly DemographicDurableFunctionConfig _config;
+
 
-    public DurableDemographicFunction(IDataServiceClient<ParticipantDemographic> dataServiceClient, ILogger<DurableDemographicFunction> logger, ICreateResponse createResponse)
+    public DurableDemographicFunction(IDataServiceClient<ParticipantDemographic> dataServiceClient, ILogger<DurableDemographicFunction> logger, ICreateResponse createResponse, IOptions<DemographicDurableFunctionConfig> demographicDurableFunctionConfig)
     {
         _participantDemographic = dataServiceClient;
         _logger = logger;
         _createResponse = createResponse;
+        _config = demographicDurableFunctionConfig.Value;
     }
 
     /// <summary>
@@ -34,11 +38,9 @@ public DurableDemographicFunction(IDataServiceClient<ParticipantDemographic> dat
     /// <returns></returns>
     /// <exception cref="TimeoutException"></exception>
     [Function(nameof(DurableDemographicFunction))]
-    public async Task<bool> RunOrchestrator(
+    public async Task RunOrchestrator(
         [OrchestrationTrigger] TaskOrchestrationContext context)
     {
-
-
         try
         {
             var demographicJsonData = context.GetInput<string>();
@@ -48,24 +50,17 @@ public async Task<bool> RunOrchestrator(
                 throw new InvalidDataException("demographicJsonData was null or empty in Orchestration function");
             }
 
-            var retryOptions = TaskOptions.FromRetryPolicy(new RetryPolicy(
-                maxNumberOfAttempts: 1, // this means the function will not retry and therefore add duplicates
-                firstRetryInterval: TimeSpan.FromSeconds(100))
-            );
+            TaskOptions retryOptions = TaskOptions.FromRetryHandler(retryContext =>
+            {
+                _logger.LogWarning("Retrying batch after failure. Current Retry count: {RetryCount} ", retryContext.LastAttemptNumber);
+                return retryContext.LastAttemptNumber < _config.MaxRetryCount;
+            });
 
-            // Add timeout-aware logic
-            var recordsInserted = await context.CallActivityAsync<bool>(
+            await context.CallActivityAsync<bool>(
                 nameof(InsertDemographicData),
                 demographicJsonData,
                 options: retryOptions
             );
-
-            if (!recordsInserted)
-            {
-                throw new InvalidOperationException("Demographic records were not added to the database in the orchestration function");
-            }
-            return true;
-
         }
         catch (Exception ex)
         {
@@ -81,18 +76,16 @@ public async Task<bool> RunOrchestrator(
     /// <param name="executionContext"></param>
     /// <returns></returns>
     [Function(nameof(InsertDemographicData))]
-    public async Task<bool> InsertDemographicData([ActivityTrigger] string demographicJsonData, FunctionContext executionContext)
+    public async Task InsertDemographicData([ActivityTrigger] string demographicJsonData, FunctionContext executionContext)
     {
-        try
-        {
-            var participantData = JsonSerializer.Deserialize<List<ParticipantDemographic>>(demographicJsonData);
-            return await _participantDemographic.AddRange(participantData);
-        }
-        catch (Exception ex)
+        var participantData = JsonSerializer.Deserialize<List<ParticipantDemographic>>(demographicJsonData);
+        var recordsInserted = await _participantDemographic.AddRange(participantData!);
+
+        if (!recordsInserted)
         {
-            _logger.LogError(ex, "Inserting demographic data failed");
-            return false;
+            throw new InvalidOperationException("Demographic records were not added to the database in the orchestration function");
         }
+        _logger.LogInformation("InsertDemographicData function has successfully completed");
     }
 
 

application/CohortManager/src/Functions/DemographicServices/DemographicDurableFunction/Program.cs

@@ -4,8 +4,8 @@
 using Data.Database;
 using Model;
 using DataServices.Client;
-using NHS.Screening.DemographicDurableFunction;
 using HealthChecks.Extensions;
+using NHS.CohortManager.DemographicServices;
 
 var host = new HostBuilder()
     .AddConfiguration<DemographicDurableFunctionConfig>(out DemographicDurableFunctionConfig config)

infrastructure/tf-core/environments/development.tfvars

@@ -865,6 +865,7 @@ function_apps = {
       ]
       env_vars_static = {
         AcceptableLatencyThresholdMs = "500"
+        MaxRetryCount=3
       }
     }
 

infrastructure/tf-core/environments/integration.tfvars

@@ -866,6 +866,7 @@ function_apps = {
       ]
       env_vars_static = {
         AcceptableLatencyThresholdMs = "500"
+        MaxRetryCount=3
       }
     }
 

infrastructure/tf-core/environments/nft.tfvars

@@ -865,6 +865,7 @@ function_apps = {
       ]
       env_vars_static = {
         AcceptableLatencyThresholdMs = "500"
+        MaxRetryCount=3
       }
     }
 

infrastructure/tf-core/environments/preprod.tfvars

@@ -850,6 +850,7 @@ function_apps = {
       ]
       env_vars_static = {
         AcceptableLatencyThresholdMs = "500"
+        MaxRetryCount=3
       }
     }
 

infrastructure/tf-core/environments/production.tfvars

@@ -884,6 +884,7 @@ function_apps = {
       ]
       env_vars_static = {
         AcceptableLatencyThresholdMs = "500"
+        MaxRetryCount=3
       }
     }
 

infrastructure/tf-core/environments/sandbox.tfvars

@@ -917,6 +917,7 @@ function_apps = {
       ]
       env_vars_static = {
         AcceptableLatencyThresholdMs = "500"
+        MaxRetryCount=3
       }
     }