diff --git a/src/Client/AzureManaged/DurableTaskSchedulerClientOptions.cs b/src/Client/AzureManaged/DurableTaskSchedulerClientOptions.cs index c1a0da7e8..c2c74a7a5 100644 --- a/src/Client/AzureManaged/DurableTaskSchedulerClientOptions.cs +++ b/src/Client/AzureManaged/DurableTaskSchedulerClientOptions.cs @@ -6,6 +6,7 @@ using Azure.Identity; using Grpc.Core; using Grpc.Net.Client; +using Grpc.Net.Client.Configuration; namespace Microsoft.DurableTask; @@ -112,6 +113,7 @@ this.Credential is not null { Credentials = ChannelCredentials.Create(channelCreds, managedBackendCreds), UnsafeUseInsecureChannelCallCredentials = this.AllowInsecureCredentials, + ServiceConfig = GrpcRetryPolicyDefaults.DefaultServiceConfig, }); } @@ -127,7 +129,7 @@ this.Credential is not null case "managedidentity": return new ManagedIdentityCredential(connectionString.ClientId); case "workloadidentity": - var opts = new WorkloadIdentityCredentialOptions(); + WorkloadIdentityCredentialOptions opts = new WorkloadIdentityCredentialOptions(); if (!string.IsNullOrEmpty(connectionString.ClientId)) { opts.ClientId = connectionString.ClientId; diff --git a/src/Shared/AzureManaged/GrpcRetryPolicyDefaults.cs b/src/Shared/AzureManaged/GrpcRetryPolicyDefaults.cs new file mode 100644 index 000000000..6217d1867 --- /dev/null +++ b/src/Shared/AzureManaged/GrpcRetryPolicyDefaults.cs @@ -0,0 +1,81 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using Grpc.Core; +using Grpc.Net.Client.Configuration; + +namespace Microsoft.DurableTask; + +/// +/// Provides default retry policy configurations for gRPC client connections. +/// +sealed class GrpcRetryPolicyDefaults +{ + /// + /// Default maximum number of retry attempts. + /// + public const int DefaultMaxAttempts = 10; + + /// + /// Default initial backoff in milliseconds. + /// + public const int DefaultInitialBackoffMs = 50; + + /// + /// Default maximum backoff in milliseconds. + /// + public const int DefaultMaxBackoffMs = 250; + + /// + /// Default backoff multiplier for exponential backoff. + /// + public const double DefaultBackoffMultiplier = 2; + + /// + /// The default service configuration that includes the method configuration. + /// + /// + /// This can be applied to a gRPC channel to enable automatic retries for all methods. + /// + public static readonly ServiceConfig DefaultServiceConfig; + + + /// + /// The default retry policy for gRPC operations. + /// + /// + /// - Retries only for Unavailable status codes (typically connection issues). + /// + static readonly Grpc.Net.Client.Configuration.RetryPolicy Default; + + /// + /// The default method configuration that applies the retry policy to all methods. + /// + /// + /// Uses MethodName.Default to apply the retry policy to all gRPC methods. + /// + static readonly MethodConfig DefaultMethodConfig; + + static GrpcRetryPolicyDefaults() + { + Default = new Grpc.Net.Client.Configuration.RetryPolicy + { + MaxAttempts = DefaultMaxAttempts, + InitialBackoff = TimeSpan.FromMilliseconds(DefaultInitialBackoffMs), + MaxBackoff = TimeSpan.FromMilliseconds(DefaultMaxBackoffMs), + BackoffMultiplier = DefaultBackoffMultiplier, + RetryableStatusCodes = { StatusCode.Unavailable }, + }; + + DefaultMethodConfig = new MethodConfig + { + Names = { MethodName.Default }, + RetryPolicy = Default, + }; + + DefaultServiceConfig = new ServiceConfig + { + MethodConfigs = { DefaultMethodConfig }, + }; + } +} diff --git a/src/Worker/AzureManaged/DurableTaskSchedulerWorkerOptions.cs b/src/Worker/AzureManaged/DurableTaskSchedulerWorkerOptions.cs index 023b2d8b8..cc795a1ed 100644 --- a/src/Worker/AzureManaged/DurableTaskSchedulerWorkerOptions.cs +++ b/src/Worker/AzureManaged/DurableTaskSchedulerWorkerOptions.cs @@ -6,6 +6,7 @@ using Azure.Identity; using Grpc.Core; using Grpc.Net.Client; +using Grpc.Net.Client.Configuration; namespace Microsoft.DurableTask; @@ -119,6 +120,7 @@ this.Credential is not null { Credentials = ChannelCredentials.Create(channelCreds, managedBackendCreds), UnsafeUseInsecureChannelCallCredentials = this.AllowInsecureCredentials, + ServiceConfig = GrpcRetryPolicyDefaults.DefaultServiceConfig, }); } @@ -134,7 +136,7 @@ this.Credential is not null case "managedidentity": return new ManagedIdentityCredential(connectionString.ClientId); case "workloadidentity": - var opts = new WorkloadIdentityCredentialOptions(); + WorkloadIdentityCredentialOptions opts = new WorkloadIdentityCredentialOptions(); if (!string.IsNullOrEmpty(connectionString.ClientId)) { opts.ClientId = connectionString.ClientId; diff --git a/test/Shared/AzureManaged.Tests/GrpcRetryPolicyDefaultsTests.cs b/test/Shared/AzureManaged.Tests/GrpcRetryPolicyDefaultsTests.cs new file mode 100644 index 000000000..3251e9cf7 --- /dev/null +++ b/test/Shared/AzureManaged.Tests/GrpcRetryPolicyDefaultsTests.cs @@ -0,0 +1,34 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using FluentAssertions; +using Grpc.Core; +using Grpc.Net.Client.Configuration; +using Xunit; + +namespace Microsoft.DurableTask.Tests; + +public class GrpcRetryPolicyDefaultsTests +{ + [Fact] + public void DefaultServiceConfig_ShouldHaveExpectedConfiguration() + { + ServiceConfig serviceConfig = GrpcRetryPolicyDefaults.DefaultServiceConfig; + + serviceConfig.Should().NotBeNull(); + serviceConfig.MethodConfigs.Should().HaveCount(1); + + MethodConfig methodConfig = serviceConfig.MethodConfigs[0]; + methodConfig.Names.Should().Contain(MethodName.Default); + methodConfig.RetryPolicy.Should().NotBeNull(); + + RetryPolicy? retryPolicy = methodConfig.RetryPolicy; + retryPolicy.Should().NotBeNull(); + _ = retryPolicy!.MaxAttempts.Should().Be(GrpcRetryPolicyDefaults.DefaultMaxAttempts); + retryPolicy.InitialBackoff.Should().Be(TimeSpan.FromMilliseconds(GrpcRetryPolicyDefaults.DefaultInitialBackoffMs)); + retryPolicy.MaxBackoff.Should().Be(TimeSpan.FromMilliseconds(GrpcRetryPolicyDefaults.DefaultMaxBackoffMs)); + retryPolicy.BackoffMultiplier.Should().Be(GrpcRetryPolicyDefaults.DefaultBackoffMultiplier); + retryPolicy.RetryableStatusCodes.Should().ContainSingle() + .Which.Should().Be(StatusCode.Unavailable); + } +} \ No newline at end of file diff --git a/test/Shared/AzureManaged.Tests/Shared.AzureManaged.Tests.csproj b/test/Shared/AzureManaged.Tests/Shared.AzureManaged.Tests.csproj index 849b655f8..f9822995b 100644 --- a/test/Shared/AzureManaged.Tests/Shared.AzureManaged.Tests.csproj +++ b/test/Shared/AzureManaged.Tests/Shared.AzureManaged.Tests.csproj @@ -12,6 +12,8 @@ + +