Add retry policy support for worker-indexing scenarios (#9265)

Author: satvu

src/WebJobs.Script.Grpc/Channel/GrpcWorkerChannel.cs

@@ -863,6 +863,25 @@ internal void ProcessFunctionMetadataResponses(FunctionMetadataResponse function
                         Language = metadata.Language
                     };
 
+                    if (metadata.RetryOptions is not null)
+                    {
+                        functionMetadata.Retry = new RetryOptions
+                        {
+                            MaxRetryCount = metadata.RetryOptions.MaxRetryCount,
+                            Strategy = metadata.RetryOptions.RetryStrategy.ToRetryStrategy()
+                        };
+
+                        if (functionMetadata.Retry.Strategy is RetryStrategy.FixedDelay)
+                        {
+                            functionMetadata.Retry.DelayInterval = metadata.RetryOptions.DelayInterval?.ToTimeSpan();
+                        }
+                        else
+                        {
+                            functionMetadata.Retry.MinimumInterval = metadata.RetryOptions.MinimumInterval?.ToTimeSpan();
+                            functionMetadata.Retry.MaximumInterval = metadata.RetryOptions.MaximumInterval?.ToTimeSpan();
+                        }
+                    }
+
                     functionMetadata.SetFunctionId(metadata.FunctionId);
 
                     foreach (var property in metadata.Properties)

src/WebJobs.Script.Grpc/MessageExtensions/GrpcMessageConversionExtensions.cs

@@ -357,6 +357,14 @@ internal static TypedData ToRpcLongArray(this long[] arrLong)
             return typedData;
         }
 
+        internal static RetryStrategy ToRetryStrategy(this RpcRetryOptions.Types.RetryStrategy retryStrategy) =>
+            retryStrategy switch
+            {
+                RpcRetryOptions.Types.RetryStrategy.FixedDelay => RetryStrategy.FixedDelay,
+                RpcRetryOptions.Types.RetryStrategy.ExponentialBackoff => RetryStrategy.ExponentialBackoff,
+                _ => throw new InvalidOperationException($"Unknown RetryStrategy RpcDataType: {retryStrategy}.")
+            };
+
         private static bool ShouldIncludeEmptyEntriesInMessagePayload(GrpcCapabilities capabilities)
         {
             return !string.IsNullOrWhiteSpace(capabilities.GetCapabilityState(RpcWorkerConstants.IncludeEmptyEntriesInMessagePayload));

src/WebJobs.Script/Host/WorkerFunctionMetadataProvider.cs

@@ -193,10 +193,15 @@ internal IEnumerable<FunctionMetadata> ValidateMetadata(IEnumerable<RawFunctionM
                         }
                     }
 
-                    // retry option validation
+                    // populate retry options if json string representation is provided
                     if (!string.IsNullOrEmpty(rawFunction.RetryOptions))
                     {
                         function.Retry = JObject.Parse(rawFunction.RetryOptions).ToObject<RetryOptions>();
+                    }
+
+                    // retry option validation
+                    if (function.Retry is not null)
+                    {
                         Utility.ValidateRetryOptions(function.Retry);
                     }
 

src/WebJobs.Script/Utility.cs

@@ -912,11 +912,11 @@ public static void ValidateRetryOptions(RetryOptions
                 case RetryStrategy.ExponentialBackoff:
                     if (!retryOptions.MinimumInterval.HasValue)
                     {
-                        throw new ArgumentNullException(nameof(retryOptions.DelayInterval));
+                        throw new ArgumentNullException(nameof(retryOptions.MinimumInterval));
                     }
                     if (!retryOptions.MaximumInterval.HasValue)
                     {
-                        throw new ArgumentNullException(nameof(retryOptions.DelayInterval));
+                        throw new ArgumentNullException(nameof(retryOptions.MaximumInterval));
                     }
                     // ensure values specified to create ExponentialBackoffRetryAttribute are valid
                     _ = new ExponentialBackoffRetryAttribute(retryOptions.MaxRetryCount.Value, retryOptions.MinimumInterval.ToString(), retryOptions.MaximumInterval.ToString());

test/WebJobs.Script.Tests/UtilityTests.cs

@@ -1022,6 +1022,82 @@ public void TryReadAsBool_ReturnsExpectedBoolValue(object value, bool expectedVa
             Assert.Equal(resultValue, expectedValueResult);
         }
 
+        [Fact]
+        public void ValidateRetryOptions_ThrowsWhenMaxRetryIsNull()
+        {
+            var retryOptions = new RetryOptions
+            {
+                Strategy = RetryStrategy.FixedDelay
+            };
+
+            var ex = Assert.Throws<ArgumentNullException>(() => Utility.ValidateRetryOptions(retryOptions));
+            Assert.Equal("Value cannot be null. (Parameter 'MaxRetryCount')", ex.Message);
+        }
+
+        [Fact]
+        public void ValidateRetryOptions_ThrowsWhenFixedDelayArgsNull()
+        {
+            var retryOptions = new RetryOptions
+            {
+                Strategy = RetryStrategy.FixedDelay,
+                MaxRetryCount = 5
+            };
+
+            var ex = Assert.Throws<ArgumentNullException>(() => Utility.ValidateRetryOptions(retryOptions));
+            Assert.Equal("Value cannot be null. (Parameter 'DelayInterval')", ex.Message);
+        }
+
+        [Fact]
+        public void ValidateRetryOptions_ThrowsWhenExponentialBackoffArgsNull()
+        {
+            var retryOptions1 = new RetryOptions
+            {
+                Strategy = RetryStrategy.ExponentialBackoff,
+                MaxRetryCount = 5,
+                MinimumInterval = TimeSpan.FromSeconds(5)
+            };
+
+            var retryOptions2 = new RetryOptions
+            {
+                Strategy = RetryStrategy.ExponentialBackoff,
+                MaxRetryCount = 5,
+                MaximumInterval = TimeSpan.MaxValue
+            };
+
+            var ex1 = Assert.Throws<ArgumentNullException>(() => Utility.ValidateRetryOptions(retryOptions1));
+            Assert.Equal("Value cannot be null. (Parameter 'MaximumInterval')", ex1.Message);
+
+            var ex2 = Assert.Throws<ArgumentNullException>(() => Utility.ValidateRetryOptions(retryOptions2));
+            Assert.Equal("Value cannot be null. (Parameter 'MinimumInterval')", ex2.Message);
+        }
+
+        [Fact]
+        public void ValidateRetryOptions_FixedDelaySuccess()
+        {
+            var retryOptions = new RetryOptions
+            {
+                Strategy = RetryStrategy.FixedDelay,
+                MaxRetryCount = 5,
+                DelayInterval = TimeSpan.FromSeconds(600)
+            };
+
+            Utility.ValidateRetryOptions(retryOptions);
+        }
+
+        [Fact]
+        public void ValidateRetryOptions_ExponentialBackoffSuccess()
+        {
+            var retryOptions = new RetryOptions
+            {
+                Strategy = RetryStrategy.ExponentialBackoff,
+                MaxRetryCount = 5,
+                MinimumInterval = TimeSpan.FromSeconds(10),
+                MaximumInterval = TimeSpan.MaxValue
+            };
+
+            Utility.ValidateRetryOptions(retryOptions);
+        }
+
         private static void VerifyLogLevel(IList<LogMessage> allLogs, string msg, LogLevel expectedLevel)
         {
             var message = allLogs.FirstOrDefault(l => l.FormattedMessage.Contains(msg));