FUND-2327 nrc_listener rollback Polly retry with AddRetryOnHttpStatusCodes and DBG retry-logging (#142)

* AddRetryOnHttpStatusCodes restored with Polly DBG logging

* Update src/OneGround.ZGW.Notificaties.Messaging/ServiceConfiguration.cs

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* Update src/OneGround.ZGW.Notificaties.Messaging/ServiceConfiguration.cs

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* copilot fixes

---------

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

Author: heuvea

src/OneGround.ZGW.Common.ServiceAgent/Configuration/HttpResiliencePipelineOptions.cs

@@ -7,5 +7,7 @@ public class HttpResiliencePipelineOptions
     public HttpRetryStrategyOptions Retry { get; set; } = HttpResiliencePipelineDefaults.RetryStrategyOptions;
     public HttpTimeoutStrategyOptions Timeout { get; set; } = HttpResiliencePipelineDefaults.TimeoutStrategyOptions;
 
+    public string AddRetryOnHttpStatusCodes { get; set; } = "";
+
     public static string GetKey(string serviceName) => $"PollyConfig:{serviceName}";
 }

src/OneGround.ZGW.Notificaties.Messaging/ServiceConfiguration.cs

@@ -1,9 +1,13 @@
+using System.Linq;
+using System.Net;
 using Hangfire;
 using Hangfire.Console;
 using Hangfire.PostgreSql;
 using MassTransit;
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Http.Resilience;
+using Microsoft.Extensions.Logging;
 using OneGround.ZGW.Common.Batching;
 using OneGround.ZGW.Common.CorrelationId;
 using OneGround.ZGW.Common.Extensions;
@@ -21,6 +25,7 @@
 using OneGround.ZGW.Notificaties.Messaging.Jobs.Notificatie;
 using OneGround.ZGW.Notificaties.Messaging.Services;
 using Polly;
+using Polly.Retry;
 
 namespace OneGround.ZGW.Notificaties.Messaging;
 
@@ -64,10 +69,44 @@ public void ConfigureServices(IServiceCollection services)
                 serviceName,
                 (builder, context) =>
                 {
-                    context.EnableReloads<HttpResiliencePipelineOptions>(optionsKey);
-                    var options = context.GetOptions<HttpResiliencePipelineOptions>(optionsKey);
+                    // Enable dynamic reloads of this pipeline whenever the named ResiliencePipelineNotificaties change
+                    context.EnableReloads<HttpResiliencePipelineOptions>("PollyConfig:NotificatiesSender");
+
+                    // Retrieve the named options
+                    var options = context.GetOptions<HttpResiliencePipelineOptions>("PollyConfig:NotificatiesSender");
+
+                    builder.AddRetry(
+                        new RetryStrategyOptions<HttpResponseMessage>
+                        {
+                            MaxRetryAttempts = options.Retry.MaxRetryAttempts,
+                            BackoffType = options.Retry.BackoffType,
+                            UseJitter = options.Retry.UseJitter,
+                            Delay = options.Retry.Delay,
+                            ShouldHandle = arg =>
+                            {
+                                if (arg.Outcome.Result == null) // This flow is when service did not respond at all (gives no HTTP statuscode)
+                                {
+                                    // Always retry on this flow
+                                    return ValueTask.FromResult(true);
+                                }
+
+                                // Retry depending on the HTTP status code
+                                var shouldRetry = DefaultRetryOnHttpStatusCodes
+                                    .Concat(HttpStatusCodesStringToEnumerable(options.AddRetryOnHttpStatusCodes))
+                                    .Any(statuscode => arg.Outcome.Result.StatusCode == statuscode);
+
+                                return ValueTask.FromResult(shouldRetry);
+                            },
+                            OnRetry = arg =>
+                            {
+                                LogRetry(context, arg);
+
+                                // Event handlers can be asynchronous; here, we return an empty ValueTask.
+                                return default;
+                            },
+                        }
+                    );
 
-                    builder.AddRetry(options.Retry);
                     builder.AddTimeout(options.Timeout);
                 }
             );
@@ -166,6 +205,24 @@ public void ConfigureServices(IServiceCollection services)
         );
     }
 
+    private static IEnumerable<HttpStatusCode> HttpStatusCodesStringToEnumerable(string addRetryOnHttpStatusCodes)
+    {
+        var result = addRetryOnHttpStatusCodes
+            .Split(';', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)
+            .Select(codeString =>
+            {
+                if (Enum.TryParse<HttpStatusCode>(codeString, out var statusCode))
+                {
+                    return (HttpStatusCode?)statusCode;
+                }
+                return null;
+            })
+            .Where(code => code.HasValue)
+            .Select(code => code!.Value);
+
+        return result;
+    }
+
     private bool IsExpireFailedJobsScannerEnabled =>
         !string.IsNullOrWhiteSpace(_hangfireConfiguration.ExpireFailedJobsScanAt)
         && !DisabledValues.Contains(_hangfireConfiguration.ExpireFailedJobsScanAt.Trim());
@@ -228,4 +285,57 @@ private AutomaticRetryAttribute GetRetryPolicyFromConfig()
             LogEvents = false,
         };
     }
+
+    private static void LogRetry(ResilienceHandlerContext context, OnRetryArguments<HttpResponseMessage> arg)
+    {
+        using var scope = context.ServiceProvider.CreateScope();
+
+        var logger = scope.ServiceProvider.GetRequiredService<ILogger<ServiceConfiguration>>();
+
+        // Get the context in which a retry should be taken and log...
+        if (
+            arg.Context.Properties.TryGetValue(
+                new ResiliencePropertyKey<HttpRequestMessage>("Resilience.Http.RequestMessage"),
+                out var httpRequestMessage
+            )
+        )
+        {
+            if (arg.Outcome.Result is { } httpResponseMessage)
+            {
+                logger.LogDebug(
+                    "OnRetry, Attempt# {AttemptNumber} on {Method} {RequestUri} => {ReasonPhrase} [{StatusCode}]. Next over {TotalSeconds} second(s).",
+                    arg.AttemptNumber,
+                    httpRequestMessage.Method,
+                    httpRequestMessage.RequestUri,
+                    httpResponseMessage.ReasonPhrase,
+                    (int)httpResponseMessage.StatusCode,
+                    arg.RetryDelay.TotalSeconds
+                );
+            }
+            else
+            {
+                logger.LogDebug(
+                    "OnRetry, Attempt# {AttemptNumber} on {Method} {RequestUri} => (no response). Next over {TotalSeconds} second(s).",
+                    arg.AttemptNumber,
+                    httpRequestMessage.Method,
+                    httpRequestMessage.RequestUri,
+                    arg.RetryDelay.TotalSeconds
+                );
+            }
+        }
+        else
+        {
+            logger.LogDebug("OnRetry, Attempt# {AttemptNumber}. Next over {TotalSeconds} second(s).", arg.AttemptNumber, arg.RetryDelay.TotalSeconds);
+        }
+    }
+
+    private static IEnumerable<HttpStatusCode> DefaultRetryOnHttpStatusCodes =>
+        [
+            HttpStatusCode.RequestTimeout,
+            HttpStatusCode.TooManyRequests,
+            HttpStatusCode.InternalServerError,
+            HttpStatusCode.BadGateway,
+            HttpStatusCode.ServiceUnavailable,
+            HttpStatusCode.GatewayTimeout,
+        ];
 }