Updating RetryProxyHandler strategy and retry counts (#10566)

Author: fabiocav

release_notes.md

@@ -15,4 +15,5 @@
 - Update PowerShell 7.2 worker to [4.0.4025](https://github.com/Azure/azure-functions-powershell-worker/releases/tag/v4.0.4025)
 - Update PowerShell 7.4 worker to [4.0.4026](https://github.com/Azure/azure-functions-powershell-worker/releases/tag/v4.0.4026)
 - Added support for identity-based connections to Diagnostic Events (#10438)
-- Updating Microsoft.Azure.WebJobs.Logging.ApplicationInsights to 3.0.42-12121
+- Updating Microsoft.Azure.WebJobs.Logging.ApplicationInsights to 3.0.42-12121
+- Updated retry logic in Worker HTTP proxy to allow for longer worker HTTP listener initialization times (#10566).

src/WebJobs.Script.Grpc/Server/RetryProxyHandler.cs

@@ -12,10 +12,14 @@ namespace Microsoft.Azure.WebJobs.Script.Grpc
     internal sealed class RetryProxyHandler : DelegatingHandler
     {
         // The maximum number of retries
-        private readonly int _maxRetries = 3;
+        internal const int MaxRetries = 10;
 
         // The initial delay in milliseconds
-        private readonly int _initialDelay = 50;
+        internal const int InitialDelay = 50;
+
+        // The maximum delay in milliseconds
+        internal const int MaximumDelay = 250;
+
         private readonly ILogger _logger;
 
         public RetryProxyHandler(HttpMessageHandler innerHandler, ILogger logger)
@@ -26,32 +30,29 @@ public RetryProxyHandler(HttpMessageHandler innerHandler, ILogger logger)
 
         protected override async Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
         {
-            var currentDelay = _initialDelay;
-            for (int attemptCount = 1; attemptCount <= _maxRetries; attemptCount++)
+            var currentDelay = InitialDelay;
+            for (int attemptCount = 1; attemptCount <= MaxRetries; attemptCount++)
             {
                 try
                 {
                     return await base.SendAsync(request, cancellationToken);
                 }
-                catch (HttpRequestException) when (attemptCount < _maxRetries)
+                catch (HttpRequestException) when (attemptCount < MaxRetries)
                 {
-                    currentDelay *= attemptCount;
-
                     _logger.LogWarning("Failed to proxy request to the worker. Retrying in {delay}ms. Attempt {attemptCount} of {maxRetries}.",
-                        currentDelay, attemptCount, _maxRetries);
+                        currentDelay, attemptCount, MaxRetries);
 
                     await Task.Delay(currentDelay, cancellationToken);
+
+                    currentDelay = Math.Min(currentDelay * 2, MaximumDelay);
                 }
                 catch (Exception ex)
                 {
-                    if (attemptCount == _maxRetries)
-                    {
-                        _logger.LogWarning("Reached the maximum retry count for worker request proxying. Error: {exception}", ex);
-                    }
-                    else
-                    {
-                        _logger.LogWarning($"Unsupported exception type in {nameof(RetryProxyHandler)}. Request will not be retried. Exception: {{exception}}", ex);
-                    }
+                    var message = attemptCount == MaxRetries
+                        ? "Reached the maximum retry count for worker request proxying. Error: {exception}"
+                        : $"Unsupported exception type in {nameof(RetryProxyHandler)}. Request will not be retried. Exception: {{exception}}";
+
+                    _logger.LogWarning(message, ex);
 
                     throw;
                 }

test/WebJobs.Script.Tests/Workers/RetryProxyHandlerTests.cs

@@ -0,0 +1,51 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT License. See License.txt in the project root for license information.
+
+using System;
+using System.Collections.Generic;
+using System.Net.Http;
+using System.Reflection;
+using System.Threading;
+using System.Threading.Tasks;
+using System.Web.Http;
+using Microsoft.Azure.WebJobs.Script.Grpc;
+using Microsoft.Extensions.Logging.Abstractions;
+using Xunit;
+
+namespace Microsoft.Azure.WebJobs.Script.Tests.Workers
+{
+    public class RetryProxyHandlerTests
+    {
+        [Fact]
+        public async Task SendAsync_RetriesToMax()
+        {
+            var inner = new TestHandler();
+            var handler = new RetryProxyHandler(inner, NullLogger.Instance);
+            var request = new HttpRequestMessage();
+
+            var response = typeof(RetryProxyHandler)!
+                .GetMethod("SendAsync", BindingFlags.NonPublic | BindingFlags.Instance)!
+                .Invoke(handler, new object[] { request, CancellationToken.None })
+                as Task<HttpResponseMessage>;
+
+            var result = await response.ContinueWith(t => t);
+
+            Assert.True(result.IsFaulted);
+            Assert.True(result.Exception.InnerException is HttpRequestException);
+            Assert.Equal(RetryProxyHandler.MaxRetries, inner.Attempts);
+        }
+
+        private class TestHandler : HttpMessageHandler
+        {
+            public int Attempts { get; set; }
+
+            protected override Task<HttpResponseMessage> SendAsync(HttpRequestMessage request,
+                CancellationToken cancellationToken)
+            {
+                Attempts++;
+
+                throw new HttpRequestException();
+            }
+        }
+    }
+}