Dts Grpc client retry support

Author: YunchuWang

src/Client/AzureManaged/DurableTaskSchedulerClientOptions.cs

@@ -6,6 +6,7 @@
 using Azure.Identity;
 using Grpc.Core;
 using Grpc.Net.Client;
+using Grpc.Net.Client.Configuration;
 
 namespace Microsoft.DurableTask;
 
@@ -112,6 +113,24 @@ this.Credential is not null
         {
             Credentials = ChannelCredentials.Create(channelCreds, managedBackendCreds),
             UnsafeUseInsecureChannelCallCredentials = this.AllowInsecureCredentials,
+            ServiceConfig = new ServiceConfig
+            {
+                MethodConfigs =
+                {
+                    new MethodConfig
+                    {
+                        Names = { MethodName.Default },
+                        RetryPolicy = new Grpc.Net.Client.Configuration.RetryPolicy()
+                        {
+                            MaxAttempts = 10,
+                            InitialBackoff = TimeSpan.FromMilliseconds(50),
+                            MaxBackoff = TimeSpan.FromMilliseconds(250),
+                            BackoffMultiplier = 2,
+                            RetryableStatusCodes = { StatusCode.Unavailable },
+                        },
+                    },
+                },
+            },
         });
     }
 
@@ -127,7 +146,7 @@ this.Credential is not null
             case "managedidentity":
                 return new ManagedIdentityCredential(connectionString.ClientId);
             case "workloadidentity":
-                var opts = new WorkloadIdentityCredentialOptions();
+                WorkloadIdentityCredentialOptions opts = new WorkloadIdentityCredentialOptions();
                 if (!string.IsNullOrEmpty(connectionString.ClientId))
                 {
                     opts.ClientId = connectionString.ClientId;

src/Worker/AzureManaged/DurableTaskSchedulerWorkerOptions.cs

@@ -6,6 +6,7 @@
 using Azure.Identity;
 using Grpc.Core;
 using Grpc.Net.Client;
+using Grpc.Net.Client.Configuration;
 
 namespace Microsoft.DurableTask;
 
@@ -119,6 +120,24 @@ this.Credential is not null
         {
             Credentials = ChannelCredentials.Create(channelCreds, managedBackendCreds),
             UnsafeUseInsecureChannelCallCredentials = this.AllowInsecureCredentials,
+            ServiceConfig = new ServiceConfig
+            {
+                MethodConfigs =
+                {
+                    new MethodConfig
+                    {
+                        Names = { MethodName.Default },
+                        RetryPolicy = new Grpc.Net.Client.Configuration.RetryPolicy()
+                        {
+                            MaxAttempts = 10,
+                            InitialBackoff = TimeSpan.FromMilliseconds(50),
+                            MaxBackoff = TimeSpan.FromMilliseconds(250),
+                            BackoffMultiplier = 2,
+                            RetryableStatusCodes = { StatusCode.Unavailable },
+                        },
+                    },
+                },
+            },
         });
     }
 
@@ -134,7 +153,7 @@ this.Credential is not null
             case "managedidentity":
                 return new ManagedIdentityCredential(connectionString.ClientId);
             case "workloadidentity":
-                var opts = new WorkloadIdentityCredentialOptions();
+                WorkloadIdentityCredentialOptions opts = new WorkloadIdentityCredentialOptions();
                 if (!string.IsNullOrEmpty(connectionString.ClientId))
                 {
                     opts.ClientId = connectionString.ClientId;