diff --git a/CHANGELOG.md b/CHANGELOG.md index 6d72ce036..471ae3d83 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ ## (Unreleased) +- Expose gRPC retry options in Azure Managed extensions ([#447](https://github.com/microsoft/durabletask-dotnet/pull/447)) + ## v1.12.0 - Activity tag support ([#426](https://github.com/microsoft/durabletask-dotnet/pull/426)) diff --git a/src/Client/AzureManaged/DurableTaskSchedulerClientOptions.cs b/src/Client/AzureManaged/DurableTaskSchedulerClientOptions.cs index 360995cc3..3a00ba7a1 100644 --- a/src/Client/AzureManaged/DurableTaskSchedulerClientOptions.cs +++ b/src/Client/AzureManaged/DurableTaskSchedulerClientOptions.cs @@ -6,8 +6,8 @@ using Azure.Identity; using Grpc.Core; using Grpc.Net.Client; -using Microsoft.DurableTask; using Microsoft.DurableTask.Client; +using GrpcConfig = Grpc.Net.Client.Configuration; namespace Microsoft.DurableTask; @@ -46,6 +46,11 @@ public class DurableTaskSchedulerClientOptions /// public bool AllowInsecureCredentials { get; set; } + /// + /// Gets or sets the options that determine how and when calls made to the scheduler will be retried. + /// + public ClientRetryOptions? RetryOptions { get; set; } + /// /// Creates a new instance of from a connection string. /// @@ -109,6 +114,45 @@ this.Credential is not null metadata.Add("Authorization", $"Bearer {token.Token}"); }); + GrpcConfig.ServiceConfig? serviceConfig = GrpcRetryPolicyDefaults.DefaultServiceConfig; + if (this.RetryOptions != null) + { + GrpcConfig.RetryPolicy retryPolicy = new GrpcConfig.RetryPolicy + { + MaxAttempts = this.RetryOptions.MaxRetries ?? GrpcRetryPolicyDefaults.DefaultMaxAttempts, + InitialBackoff = TimeSpan.FromMilliseconds(this.RetryOptions.InitialBackoffMs ?? GrpcRetryPolicyDefaults.DefaultInitialBackoffMs), + MaxBackoff = TimeSpan.FromMilliseconds(this.RetryOptions.MaxBackoffMs ?? GrpcRetryPolicyDefaults.DefaultMaxBackoffMs), + BackoffMultiplier = this.RetryOptions.BackoffMultiplier ?? GrpcRetryPolicyDefaults.DefaultBackoffMultiplier, + RetryableStatusCodes = { StatusCode.Unavailable }, // Always retry on Unavailable. + }; + + if (this.RetryOptions.RetryableStatusCodes != null) + { + foreach (StatusCode statusCode in this.RetryOptions.RetryableStatusCodes) + { + // Added by default, don't need to have it added twice. + if (statusCode == StatusCode.Unavailable) + { + continue; + } + + retryPolicy.RetryableStatusCodes.Add(statusCode); + } + } + + GrpcConfig.MethodConfig methodConfig = new GrpcConfig.MethodConfig + { + // MethodName.Default applies this retry policy configuration to all gRPC methods on the channel. + Names = { GrpcConfig.MethodName.Default }, + RetryPolicy = retryPolicy, + }; + + serviceConfig = new GrpcConfig.ServiceConfig + { + MethodConfigs = { methodConfig }, + }; + } + // Production will use HTTPS, but local testing will use HTTP ChannelCredentials channelCreds = endpoint.StartsWith("https://", StringComparison.OrdinalIgnoreCase) ? ChannelCredentials.SecureSsl : @@ -117,7 +161,7 @@ this.Credential is not null { Credentials = ChannelCredentials.Create(channelCreds, managedBackendCreds), UnsafeUseInsecureChannelCallCredentials = this.AllowInsecureCredentials, - ServiceConfig = GrpcRetryPolicyDefaults.DefaultServiceConfig, + ServiceConfig = serviceConfig, }); } @@ -173,4 +217,35 @@ this.Credential is not null nameof(connectionString)); } } + + /// + /// Options used to configure retries used when making calls to the Scheduler. + /// + public class ClientRetryOptions + { + /// + /// Gets or sets the maximum number of times a call should be retried. + /// + public int? MaxRetries { get; set; } + + /// + /// Gets or sets the initial backoff in milliseconds. + /// + public int? InitialBackoffMs { get; set; } + + /// + /// Gets or sets the maximum backoff in milliseconds. + /// + public int? MaxBackoffMs { get; set; } + + /// + /// Gets or sets the backoff multiplier for exponential backoff. + /// + public double? BackoffMultiplier { get; set; } + + /// + /// Gets or sets the list of status codes that can be retried. + /// + public IList? RetryableStatusCodes { get; set; } + } } diff --git a/test/Client/AzureManaged.Tests/Client.AzureManaged.Tests.csproj b/test/Client/AzureManaged.Tests/Client.AzureManaged.Tests.csproj index 22142fa2d..4d43922f1 100644 --- a/test/Client/AzureManaged.Tests/Client.AzureManaged.Tests.csproj +++ b/test/Client/AzureManaged.Tests/Client.AzureManaged.Tests.csproj @@ -10,6 +10,7 @@ + diff --git a/test/Client/AzureManaged.Tests/DurableTaskSchedulerClientExtensionsTests.cs b/test/Client/AzureManaged.Tests/DurableTaskSchedulerClientExtensionsTests.cs index d625703ca..7aad801c3 100644 --- a/test/Client/AzureManaged.Tests/DurableTaskSchedulerClientExtensionsTests.cs +++ b/test/Client/AzureManaged.Tests/DurableTaskSchedulerClientExtensionsTests.cs @@ -4,6 +4,7 @@ using Azure.Core; using Azure.Identity; using FluentAssertions; +using Grpc.Core; using Microsoft.DurableTask.Client.Grpc; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Options; @@ -109,7 +110,7 @@ public void UseDurableTaskScheduler_WithNullParameters_ShouldThrowOptionsValidat // Assert var action = () => provider.GetRequiredService>().Value; action.Should().Throw() - .WithMessage(endpoint == null + .WithMessage(endpoint == null ? "DataAnnotation validation failed for 'DurableTaskSchedulerClientOptions' members: 'EndpointAddress' with the error: 'Endpoint address is required'." : "DataAnnotation validation failed for 'DurableTaskSchedulerClientOptions' members: 'TaskHubName' with the error: 'Task hub name is required'."); } @@ -193,4 +194,90 @@ public void UseDurableTaskScheduler_WithNamedOptions_ShouldConfigureCorrectly() options.ResourceId.Should().Be("https://durabletask.io"); options.AllowInsecureCredentials.Should().BeFalse(); } + + [Fact] + public void UseDurableTaskScheduler_WithEndpointAndCredentialAndRetryOptions_ShouldConfigureCorrectly() + { + // Arrange + ServiceCollection services = new ServiceCollection(); + Mock mockBuilder = new Mock(); + mockBuilder.Setup(b => b.Services).Returns(services); + DefaultAzureCredential credential = new DefaultAzureCredential(); + + // Act + mockBuilder.Object.UseDurableTaskScheduler(ValidEndpoint, ValidTaskHub, credential, options => + options.RetryOptions = new DurableTaskSchedulerClientOptions.ClientRetryOptions + { + MaxRetries = 5, + InitialBackoffMs = 100, + MaxBackoffMs = 1000, + BackoffMultiplier = 2.0, + RetryableStatusCodes = new List { StatusCode.Unknown } + } + ); + + // Assert + ServiceProvider provider = services.BuildServiceProvider(); + IOptions? options = provider.GetService>(); + options.Should().NotBeNull(); + + // Validate the configured options + DurableTaskSchedulerClientOptions clientOptions = provider.GetRequiredService>().Value; + clientOptions.EndpointAddress.Should().Be(ValidEndpoint); + clientOptions.TaskHubName.Should().Be(ValidTaskHub); + clientOptions.Credential.Should().BeOfType(); + clientOptions.RetryOptions.Should().NotBeNull(); + // The assert not null doesn't clear the syntax warning about null checks. + if (clientOptions.RetryOptions != null) + { + clientOptions.RetryOptions.MaxRetries.Should().Be(5); + clientOptions.RetryOptions.InitialBackoffMs.Should().Be(100); + clientOptions.RetryOptions.MaxBackoffMs.Should().Be(1000); + clientOptions.RetryOptions.BackoffMultiplier.Should().Be(2.0); + clientOptions.RetryOptions.RetryableStatusCodes.Should().Contain(StatusCode.Unknown); + } + } + + [Fact] + public void UseDurableTaskScheduler_WithConnectionStringAndRetryOptions_ShouldConfigureCorrectly() + { + // Arrange + ServiceCollection services = new ServiceCollection(); + Mock mockBuilder = new Mock(); + mockBuilder.Setup(b => b.Services).Returns(services); + string connectionString = $"Endpoint={ValidEndpoint};Authentication=DefaultAzure;TaskHub={ValidTaskHub}"; + + // Act + mockBuilder.Object.UseDurableTaskScheduler(connectionString, options => + options.RetryOptions = new DurableTaskSchedulerClientOptions.ClientRetryOptions + { + MaxRetries = 5, + InitialBackoffMs = 100, + MaxBackoffMs = 1000, + BackoffMultiplier = 2.0, + RetryableStatusCodes = new List { StatusCode.Unknown } + } + ); + + // Assert + ServiceProvider provider = services.BuildServiceProvider(); + IOptions? options = provider.GetService>(); + options.Should().NotBeNull(); + + // Validate the configured options + DurableTaskSchedulerClientOptions clientOptions = provider.GetRequiredService>().Value; + clientOptions.EndpointAddress.Should().Be(ValidEndpoint); + clientOptions.TaskHubName.Should().Be(ValidTaskHub); + clientOptions.Credential.Should().BeOfType(); + clientOptions.RetryOptions.Should().NotBeNull(); + // The assert not null doesn't clear the syntax warning about null checks. + if (clientOptions.RetryOptions != null) + { + clientOptions.RetryOptions.MaxRetries.Should().Be(5); + clientOptions.RetryOptions.InitialBackoffMs.Should().Be(100); + clientOptions.RetryOptions.MaxBackoffMs.Should().Be(1000); + clientOptions.RetryOptions.BackoffMultiplier.Should().Be(2.0); + clientOptions.RetryOptions.RetryableStatusCodes.Should().Contain(StatusCode.Unknown); + } + } }