Dts Grpc Client Retry Support (#403)

Author: YunchuWang

src/Client/AzureManaged/DurableTaskSchedulerClientOptions.cs

@@ -6,6 +6,7 @@
 using Azure.Identity;
 using Grpc.Core;
 using Grpc.Net.Client;
+using Grpc.Net.Client.Configuration;
 
 namespace Microsoft.DurableTask;
 
@@ -112,6 +113,7 @@ this.Credential is not null
         {
             Credentials = ChannelCredentials.Create(channelCreds, managedBackendCreds),
             UnsafeUseInsecureChannelCallCredentials = this.AllowInsecureCredentials,
+            ServiceConfig = GrpcRetryPolicyDefaults.DefaultServiceConfig,
         });
     }
 
@@ -127,7 +129,7 @@ this.Credential is not null
             case "managedidentity":
                 return new ManagedIdentityCredential(connectionString.ClientId);
             case "workloadidentity":
-                var opts = new WorkloadIdentityCredentialOptions();
+                WorkloadIdentityCredentialOptions opts = new WorkloadIdentityCredentialOptions();
                 if (!string.IsNullOrEmpty(connectionString.ClientId))
                 {
                     opts.ClientId = connectionString.ClientId;

src/Shared/AzureManaged/GrpcRetryPolicyDefaults.cs

@@ -0,0 +1,81 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+using Grpc.Core;
+using Grpc.Net.Client.Configuration;
+
+namespace Microsoft.DurableTask;
+
+/// <summary>
+/// Provides default retry policy configurations for gRPC client connections.
+/// </summary>
+sealed class GrpcRetryPolicyDefaults
+{
+    /// <summary>
+    /// Default maximum number of retry attempts.
+    /// </summary>
+    public const int DefaultMaxAttempts = 10;
+
+    /// <summary>
+    /// Default initial backoff in milliseconds.
+    /// </summary>
+    public const int DefaultInitialBackoffMs = 50;
+
+    /// <summary>
+    /// Default maximum backoff in milliseconds.
+    /// </summary>
+    public const int DefaultMaxBackoffMs = 250;
+
+    /// <summary>
+    /// Default backoff multiplier for exponential backoff.
+    /// </summary>
+    public const double DefaultBackoffMultiplier = 2;
+
+    /// <summary>
+    /// The default service configuration that includes the method configuration.
+    /// </summary>
+    /// <remarks>
+    /// This can be applied to a gRPC channel to enable automatic retries for all methods.
+    /// </remarks>
+    public static readonly ServiceConfig DefaultServiceConfig;
+
+
+    /// <summary>
+    /// The default retry policy for gRPC operations.
+    /// </summary>
+    /// <remarks>
+    /// - Retries only for Unavailable status codes (typically connection issues).
+    /// </remarks>
+    static readonly Grpc.Net.Client.Configuration.RetryPolicy Default;
+
+    /// <summary>
+    /// The default method configuration that applies the retry policy to all methods.
+    /// </summary>
+    /// <remarks>
+    /// Uses MethodName.Default to apply the retry policy to all gRPC methods.
+    /// </remarks>
+    static readonly MethodConfig DefaultMethodConfig;
+
+    static GrpcRetryPolicyDefaults()
+    {
+        Default = new Grpc.Net.Client.Configuration.RetryPolicy
+        {
+            MaxAttempts = DefaultMaxAttempts,
+            InitialBackoff = TimeSpan.FromMilliseconds(DefaultInitialBackoffMs),
+            MaxBackoff = TimeSpan.FromMilliseconds(DefaultMaxBackoffMs),
+            BackoffMultiplier = DefaultBackoffMultiplier,
+            RetryableStatusCodes = { StatusCode.Unavailable },
+        };
+
+        DefaultMethodConfig = new MethodConfig
+        {
+            Names = { MethodName.Default },
+            RetryPolicy = Default,
+        };
+
+        DefaultServiceConfig = new ServiceConfig
+        {
+            MethodConfigs = { DefaultMethodConfig },
+        };
+    }
+}

src/Worker/AzureManaged/DurableTaskSchedulerWorkerOptions.cs

@@ -6,6 +6,7 @@
 using Azure.Identity;
 using Grpc.Core;
 using Grpc.Net.Client;
+using Grpc.Net.Client.Configuration;
 
 namespace Microsoft.DurableTask;
 
@@ -119,6 +120,7 @@ this.Credential is not null
         {
             Credentials = ChannelCredentials.Create(channelCreds, managedBackendCreds),
             UnsafeUseInsecureChannelCallCredentials = this.AllowInsecureCredentials,
+            ServiceConfig = GrpcRetryPolicyDefaults.DefaultServiceConfig,
         });
     }
 
@@ -134,7 +136,7 @@ this.Credential is not null
             case "managedidentity":
                 return new ManagedIdentityCredential(connectionString.ClientId);
             case "workloadidentity":
-                var opts = new WorkloadIdentityCredentialOptions();
+                WorkloadIdentityCredentialOptions opts = new WorkloadIdentityCredentialOptions();
                 if (!string.IsNullOrEmpty(connectionString.ClientId))
                 {
                     opts.ClientId = connectionString.ClientId;

test/Shared/AzureManaged.Tests/GrpcRetryPolicyDefaultsTests.cs

@@ -0,0 +1,34 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+using FluentAssertions;
+using Grpc.Core;
+using Grpc.Net.Client.Configuration;
+using Xunit;
+
+namespace Microsoft.DurableTask.Tests;
+
+public class GrpcRetryPolicyDefaultsTests
+{
+    [Fact]
+    public void DefaultServiceConfig_ShouldHaveExpectedConfiguration()
+    {
+        ServiceConfig serviceConfig = GrpcRetryPolicyDefaults.DefaultServiceConfig;
+
+        serviceConfig.Should().NotBeNull();
+        serviceConfig.MethodConfigs.Should().HaveCount(1);
+
+        MethodConfig methodConfig = serviceConfig.MethodConfigs[0];
+        methodConfig.Names.Should().Contain(MethodName.Default);
+        methodConfig.RetryPolicy.Should().NotBeNull();
+
+        RetryPolicy? retryPolicy = methodConfig.RetryPolicy;
+        retryPolicy.Should().NotBeNull();
+        _ = retryPolicy!.MaxAttempts.Should().Be(GrpcRetryPolicyDefaults.DefaultMaxAttempts);
+        retryPolicy.InitialBackoff.Should().Be(TimeSpan.FromMilliseconds(GrpcRetryPolicyDefaults.DefaultInitialBackoffMs));
+        retryPolicy.MaxBackoff.Should().Be(TimeSpan.FromMilliseconds(GrpcRetryPolicyDefaults.DefaultMaxBackoffMs));
+        retryPolicy.BackoffMultiplier.Should().Be(GrpcRetryPolicyDefaults.DefaultBackoffMultiplier);
+        retryPolicy.RetryableStatusCodes.Should().ContainSingle()
+            .Which.Should().Be(StatusCode.Unavailable);
+    }
+} 

test/Shared/AzureManaged.Tests/Shared.AzureManaged.Tests.csproj

@@ -12,6 +12,8 @@
   <ItemGroup>
     <PackageReference Include="Microsoft.Extensions.DependencyInjection" />
     <PackageReference Include="Azure.Identity" />
+    <PackageReference Include="Grpc.Core" />
+    <PackageReference Include="Grpc.Net.Client" />
   </ItemGroup>
 
   <ItemGroup>