diff --git a/CHANGELOG.md b/CHANGELOG.md
index 6d72ce036..471ae3d83 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,6 +2,8 @@
## (Unreleased)
+- Expose gRPC retry options in Azure Managed extensions ([#447](https://github.com/microsoft/durabletask-dotnet/pull/447))
+
## v1.12.0
- Activity tag support ([#426](https://github.com/microsoft/durabletask-dotnet/pull/426))
diff --git a/src/Client/AzureManaged/DurableTaskSchedulerClientOptions.cs b/src/Client/AzureManaged/DurableTaskSchedulerClientOptions.cs
index 360995cc3..3a00ba7a1 100644
--- a/src/Client/AzureManaged/DurableTaskSchedulerClientOptions.cs
+++ b/src/Client/AzureManaged/DurableTaskSchedulerClientOptions.cs
@@ -6,8 +6,8 @@
using Azure.Identity;
using Grpc.Core;
using Grpc.Net.Client;
-using Microsoft.DurableTask;
using Microsoft.DurableTask.Client;
+using GrpcConfig = Grpc.Net.Client.Configuration;
namespace Microsoft.DurableTask;
@@ -46,6 +46,11 @@ public class DurableTaskSchedulerClientOptions
///
public bool AllowInsecureCredentials { get; set; }
+ ///
+ /// Gets or sets the options that determine how and when calls made to the scheduler will be retried.
+ ///
+ public ClientRetryOptions? RetryOptions { get; set; }
+
///
/// Creates a new instance of from a connection string.
///
@@ -109,6 +114,45 @@ this.Credential is not null
metadata.Add("Authorization", $"Bearer {token.Token}");
});
+ GrpcConfig.ServiceConfig? serviceConfig = GrpcRetryPolicyDefaults.DefaultServiceConfig;
+ if (this.RetryOptions != null)
+ {
+ GrpcConfig.RetryPolicy retryPolicy = new GrpcConfig.RetryPolicy
+ {
+ MaxAttempts = this.RetryOptions.MaxRetries ?? GrpcRetryPolicyDefaults.DefaultMaxAttempts,
+ InitialBackoff = TimeSpan.FromMilliseconds(this.RetryOptions.InitialBackoffMs ?? GrpcRetryPolicyDefaults.DefaultInitialBackoffMs),
+ MaxBackoff = TimeSpan.FromMilliseconds(this.RetryOptions.MaxBackoffMs ?? GrpcRetryPolicyDefaults.DefaultMaxBackoffMs),
+ BackoffMultiplier = this.RetryOptions.BackoffMultiplier ?? GrpcRetryPolicyDefaults.DefaultBackoffMultiplier,
+ RetryableStatusCodes = { StatusCode.Unavailable }, // Always retry on Unavailable.
+ };
+
+ if (this.RetryOptions.RetryableStatusCodes != null)
+ {
+ foreach (StatusCode statusCode in this.RetryOptions.RetryableStatusCodes)
+ {
+ // Added by default, don't need to have it added twice.
+ if (statusCode == StatusCode.Unavailable)
+ {
+ continue;
+ }
+
+ retryPolicy.RetryableStatusCodes.Add(statusCode);
+ }
+ }
+
+ GrpcConfig.MethodConfig methodConfig = new GrpcConfig.MethodConfig
+ {
+ // MethodName.Default applies this retry policy configuration to all gRPC methods on the channel.
+ Names = { GrpcConfig.MethodName.Default },
+ RetryPolicy = retryPolicy,
+ };
+
+ serviceConfig = new GrpcConfig.ServiceConfig
+ {
+ MethodConfigs = { methodConfig },
+ };
+ }
+
// Production will use HTTPS, but local testing will use HTTP
ChannelCredentials channelCreds = endpoint.StartsWith("https://", StringComparison.OrdinalIgnoreCase) ?
ChannelCredentials.SecureSsl :
@@ -117,7 +161,7 @@ this.Credential is not null
{
Credentials = ChannelCredentials.Create(channelCreds, managedBackendCreds),
UnsafeUseInsecureChannelCallCredentials = this.AllowInsecureCredentials,
- ServiceConfig = GrpcRetryPolicyDefaults.DefaultServiceConfig,
+ ServiceConfig = serviceConfig,
});
}
@@ -173,4 +217,35 @@ this.Credential is not null
nameof(connectionString));
}
}
+
+ ///
+ /// Options used to configure retries used when making calls to the Scheduler.
+ ///
+ public class ClientRetryOptions
+ {
+ ///
+ /// Gets or sets the maximum number of times a call should be retried.
+ ///
+ public int? MaxRetries { get; set; }
+
+ ///
+ /// Gets or sets the initial backoff in milliseconds.
+ ///
+ public int? InitialBackoffMs { get; set; }
+
+ ///
+ /// Gets or sets the maximum backoff in milliseconds.
+ ///
+ public int? MaxBackoffMs { get; set; }
+
+ ///
+ /// Gets or sets the backoff multiplier for exponential backoff.
+ ///
+ public double? BackoffMultiplier { get; set; }
+
+ ///
+ /// Gets or sets the list of status codes that can be retried.
+ ///
+ public IList? RetryableStatusCodes { get; set; }
+ }
}
diff --git a/test/Client/AzureManaged.Tests/Client.AzureManaged.Tests.csproj b/test/Client/AzureManaged.Tests/Client.AzureManaged.Tests.csproj
index 22142fa2d..4d43922f1 100644
--- a/test/Client/AzureManaged.Tests/Client.AzureManaged.Tests.csproj
+++ b/test/Client/AzureManaged.Tests/Client.AzureManaged.Tests.csproj
@@ -10,6 +10,7 @@
+
diff --git a/test/Client/AzureManaged.Tests/DurableTaskSchedulerClientExtensionsTests.cs b/test/Client/AzureManaged.Tests/DurableTaskSchedulerClientExtensionsTests.cs
index d625703ca..7aad801c3 100644
--- a/test/Client/AzureManaged.Tests/DurableTaskSchedulerClientExtensionsTests.cs
+++ b/test/Client/AzureManaged.Tests/DurableTaskSchedulerClientExtensionsTests.cs
@@ -4,6 +4,7 @@
using Azure.Core;
using Azure.Identity;
using FluentAssertions;
+using Grpc.Core;
using Microsoft.DurableTask.Client.Grpc;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Options;
@@ -109,7 +110,7 @@ public void UseDurableTaskScheduler_WithNullParameters_ShouldThrowOptionsValidat
// Assert
var action = () => provider.GetRequiredService>().Value;
action.Should().Throw()
- .WithMessage(endpoint == null
+ .WithMessage(endpoint == null
? "DataAnnotation validation failed for 'DurableTaskSchedulerClientOptions' members: 'EndpointAddress' with the error: 'Endpoint address is required'."
: "DataAnnotation validation failed for 'DurableTaskSchedulerClientOptions' members: 'TaskHubName' with the error: 'Task hub name is required'.");
}
@@ -193,4 +194,90 @@ public void UseDurableTaskScheduler_WithNamedOptions_ShouldConfigureCorrectly()
options.ResourceId.Should().Be("https://durabletask.io");
options.AllowInsecureCredentials.Should().BeFalse();
}
+
+ [Fact]
+ public void UseDurableTaskScheduler_WithEndpointAndCredentialAndRetryOptions_ShouldConfigureCorrectly()
+ {
+ // Arrange
+ ServiceCollection services = new ServiceCollection();
+ Mock mockBuilder = new Mock();
+ mockBuilder.Setup(b => b.Services).Returns(services);
+ DefaultAzureCredential credential = new DefaultAzureCredential();
+
+ // Act
+ mockBuilder.Object.UseDurableTaskScheduler(ValidEndpoint, ValidTaskHub, credential, options =>
+ options.RetryOptions = new DurableTaskSchedulerClientOptions.ClientRetryOptions
+ {
+ MaxRetries = 5,
+ InitialBackoffMs = 100,
+ MaxBackoffMs = 1000,
+ BackoffMultiplier = 2.0,
+ RetryableStatusCodes = new List { StatusCode.Unknown }
+ }
+ );
+
+ // Assert
+ ServiceProvider provider = services.BuildServiceProvider();
+ IOptions? options = provider.GetService>();
+ options.Should().NotBeNull();
+
+ // Validate the configured options
+ DurableTaskSchedulerClientOptions clientOptions = provider.GetRequiredService>().Value;
+ clientOptions.EndpointAddress.Should().Be(ValidEndpoint);
+ clientOptions.TaskHubName.Should().Be(ValidTaskHub);
+ clientOptions.Credential.Should().BeOfType();
+ clientOptions.RetryOptions.Should().NotBeNull();
+ // The assert not null doesn't clear the syntax warning about null checks.
+ if (clientOptions.RetryOptions != null)
+ {
+ clientOptions.RetryOptions.MaxRetries.Should().Be(5);
+ clientOptions.RetryOptions.InitialBackoffMs.Should().Be(100);
+ clientOptions.RetryOptions.MaxBackoffMs.Should().Be(1000);
+ clientOptions.RetryOptions.BackoffMultiplier.Should().Be(2.0);
+ clientOptions.RetryOptions.RetryableStatusCodes.Should().Contain(StatusCode.Unknown);
+ }
+ }
+
+ [Fact]
+ public void UseDurableTaskScheduler_WithConnectionStringAndRetryOptions_ShouldConfigureCorrectly()
+ {
+ // Arrange
+ ServiceCollection services = new ServiceCollection();
+ Mock mockBuilder = new Mock();
+ mockBuilder.Setup(b => b.Services).Returns(services);
+ string connectionString = $"Endpoint={ValidEndpoint};Authentication=DefaultAzure;TaskHub={ValidTaskHub}";
+
+ // Act
+ mockBuilder.Object.UseDurableTaskScheduler(connectionString, options =>
+ options.RetryOptions = new DurableTaskSchedulerClientOptions.ClientRetryOptions
+ {
+ MaxRetries = 5,
+ InitialBackoffMs = 100,
+ MaxBackoffMs = 1000,
+ BackoffMultiplier = 2.0,
+ RetryableStatusCodes = new List { StatusCode.Unknown }
+ }
+ );
+
+ // Assert
+ ServiceProvider provider = services.BuildServiceProvider();
+ IOptions? options = provider.GetService>();
+ options.Should().NotBeNull();
+
+ // Validate the configured options
+ DurableTaskSchedulerClientOptions clientOptions = provider.GetRequiredService>().Value;
+ clientOptions.EndpointAddress.Should().Be(ValidEndpoint);
+ clientOptions.TaskHubName.Should().Be(ValidTaskHub);
+ clientOptions.Credential.Should().BeOfType();
+ clientOptions.RetryOptions.Should().NotBeNull();
+ // The assert not null doesn't clear the syntax warning about null checks.
+ if (clientOptions.RetryOptions != null)
+ {
+ clientOptions.RetryOptions.MaxRetries.Should().Be(5);
+ clientOptions.RetryOptions.InitialBackoffMs.Should().Be(100);
+ clientOptions.RetryOptions.MaxBackoffMs.Should().Be(1000);
+ clientOptions.RetryOptions.BackoffMultiplier.Should().Be(2.0);
+ clientOptions.RetryOptions.RetryableStatusCodes.Should().Contain(StatusCode.Unknown);
+ }
+ }
}