diff --git a/src/Dapr.Messaging/PublishSubscribe/DaprSubscriptionOptions.cs b/src/Dapr.Messaging/PublishSubscribe/DaprSubscriptionOptions.cs index 73838b605..bc9e058ae 100644 --- a/src/Dapr.Messaging/PublishSubscribe/DaprSubscriptionOptions.cs +++ b/src/Dapr.Messaging/PublishSubscribe/DaprSubscriptionOptions.cs @@ -40,5 +40,11 @@ public sealed record DaprSubscriptionOptions(MessageHandlingPolicy MessageHandli /// been signaled. /// public TimeSpan MaximumCleanupTimeout { get; init; } = TimeSpan.FromSeconds(30); + + /// + /// An optional callback invoked when errors occur during an active subscription. + /// If not set, runtime errors from background tasks will be silently swallowed. + /// + public SubscriptionErrorHandler? ErrorHandler { get; init; } } diff --git a/src/Dapr.Messaging/PublishSubscribe/PublishSubscribeReceiver.cs b/src/Dapr.Messaging/PublishSubscribe/PublishSubscribeReceiver.cs index 4b0d608ff..0ec5622f3 100644 --- a/src/Dapr.Messaging/PublishSubscribe/PublishSubscribeReceiver.cs +++ b/src/Dapr.Messaging/PublishSubscribe/PublishSubscribeReceiver.cs @@ -12,6 +12,7 @@ // ------------------------------------------------------------------------ using System.Threading.Channels; +using Dapr; using Dapr.AppCallback.Autogen.Grpc.v1; using Grpc.Core; using P = Dapr.Client.Autogen.Grpc.v1; @@ -118,18 +119,36 @@ internal async Task SubscribeAsync(CancellationToken cancellationToken = default return; } - var stream = await GetStreamAsync(cancellationToken); + AsyncDuplexStreamingCall stream; + try + { + stream = await GetStreamAsync(cancellationToken); + } + catch (RpcException ex) + { + // Reset so the caller can retry after the sidecar becomes available + Interlocked.Exchange(ref hasInitialized, 0); + throw new DaprException( + $"Unable to subscribe to topic '{topicName}' on pubsub '{pubSubName}'. The Dapr sidecar may be unavailable.", + ex); + } + catch (Exception) + { + // Reset so the caller can retry regardless of the exception type + Interlocked.Exchange(ref hasInitialized, 0); + throw; + } //Retrieve the messages from the sidecar and write to the messages channel - start without awaiting so this isn't blocking _ = FetchDataFromSidecarAsync(stream, topicMessagesChannel.Writer, cancellationToken) - .ContinueWith(HandleTaskCompletion, null, cancellationToken, TaskContinuationOptions.OnlyOnFaulted, + .ContinueWith(HandleTaskCompletion, null, CancellationToken.None, TaskContinuationOptions.OnlyOnFaulted, TaskScheduler.Default); //Process the messages as they're written to either channel _ = ProcessAcknowledgementChannelMessagesAsync(stream, cancellationToken).ContinueWith(HandleTaskCompletion, - null, cancellationToken, TaskContinuationOptions.OnlyOnFaulted, TaskScheduler.Default); + null, CancellationToken.None, TaskContinuationOptions.OnlyOnFaulted, TaskScheduler.Default); _ = ProcessTopicChannelMessagesAsync(cancellationToken).ContinueWith(HandleTaskCompletion, null, - cancellationToken, + CancellationToken.None, TaskContinuationOptions.OnlyOnFaulted, TaskScheduler.Default); } @@ -149,11 +168,28 @@ internal async Task WriteAcknowledgementToChannelAsync(TopicAcknowledgement ackn } //Exposed for testing purposes only - internal static void HandleTaskCompletion(Task task, object? state) + internal void HandleTaskCompletion(Task task, object? state) { - if (task.Exception != null) + if (task.Exception is null) + { + return; + } + + // Allow the caller to retry after a background task failure + Interlocked.Exchange(ref hasInitialized, 0); + + var innerException = task.Exception.InnerException ?? task.Exception; + var daprException = new DaprException( + $"An error occurred during an active subscription to topic '{topicName}' on pubsub '{pubSubName}'.", + innerException); + + try + { + options.ErrorHandler?.Invoke(daprException); + } + catch (Exception) { - throw task.Exception; + // Prevent a faulty error handler from becoming an unobserved task exception } } diff --git a/src/Dapr.Messaging/PublishSubscribe/SubscriptionErrorHandler.cs b/src/Dapr.Messaging/PublishSubscribe/SubscriptionErrorHandler.cs new file mode 100644 index 000000000..86cb56e1a --- /dev/null +++ b/src/Dapr.Messaging/PublishSubscribe/SubscriptionErrorHandler.cs @@ -0,0 +1,20 @@ +// ------------------------------------------------------------------------ +// Copyright 2024 The Dapr Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ------------------------------------------------------------------------ + +namespace Dapr.Messaging.PublishSubscribe; + +/// +/// A delegate that handles errors occurring during an active subscription. +/// +/// The wrapping the original error. +public delegate void SubscriptionErrorHandler(DaprException exception); diff --git a/test/Dapr.Messaging.Test/PublishSubscribe/PublishSubscribeReceiverTests.cs b/test/Dapr.Messaging.Test/PublishSubscribe/PublishSubscribeReceiverTests.cs index f8070aa66..2375650d8 100644 --- a/test/Dapr.Messaging.Test/PublishSubscribe/PublishSubscribeReceiverTests.cs +++ b/test/Dapr.Messaging.Test/PublishSubscribe/PublishSubscribeReceiverTests.cs @@ -12,6 +12,7 @@ // ------------------------------------------------------------------------ using System.Threading.Channels; +using Dapr; using Dapr.AppCallback.Autogen.Grpc.v1; using Dapr.Messaging.PublishSubscribe; using Grpc.Core; @@ -193,14 +194,194 @@ public async Task DisposeAsync_ShouldCompleteChannels() } [Fact] - public void HandleTaskCompletion_ShouldThrowException_WhenTaskHasException() + public void HandleTaskCompletion_ShouldInvokeErrorHandler_WhenTaskHasException() { + const string pubSubName = "testPubSub"; + const string topicName = "testTopic"; + DaprException? receivedException = null; + var options = + new DaprSubscriptionOptions(new MessageHandlingPolicy(TimeSpan.FromSeconds(5), TopicResponseAction.Success)) + { + ErrorHandler = ex => receivedException = ex + }; + + var messageHandler = new TopicMessageHandler((message, token) => Task.FromResult(TopicResponseAction.Success)); + var mockDaprClient = new Mock(); + var receiver = new PublishSubscribeReceiver(pubSubName, topicName, options, messageHandler, mockDaprClient.Object); + + var task = Task.FromException(new InvalidOperationException("Test exception")); + + receiver.HandleTaskCompletion(task, null); + + Assert.NotNull(receivedException); + Assert.IsType(receivedException.InnerException); + Assert.Equal("Test exception", receivedException.InnerException.Message); + Assert.Contains("testTopic", receivedException.Message); + Assert.Contains("testPubSub", receivedException.Message); + } + + [Fact] + public void HandleTaskCompletion_ShouldNotThrow_WhenNoErrorHandler() + { + const string pubSubName = "testPubSub"; + const string topicName = "testTopic"; + var options = + new DaprSubscriptionOptions(new MessageHandlingPolicy(TimeSpan.FromSeconds(5), TopicResponseAction.Success)); + + var messageHandler = new TopicMessageHandler((message, token) => Task.FromResult(TopicResponseAction.Success)); + var mockDaprClient = new Mock(); + var receiver = new PublishSubscribeReceiver(pubSubName, topicName, options, messageHandler, mockDaprClient.Object); + + var task = Task.FromException(new InvalidOperationException("Test exception")); + + var exception = Record.Exception(() => receiver.HandleTaskCompletion(task, null)); + + Assert.Null(exception); + } + + [Fact] + public void HandleTaskCompletion_ShouldNotThrow_WhenErrorHandlerThrows() + { + const string pubSubName = "testPubSub"; + const string topicName = "testTopic"; + var options = + new DaprSubscriptionOptions(new MessageHandlingPolicy(TimeSpan.FromSeconds(5), TopicResponseAction.Success)) + { + ErrorHandler = _ => throw new InvalidOperationException("Handler failed") + }; + + var messageHandler = new TopicMessageHandler((message, token) => Task.FromResult(TopicResponseAction.Success)); + var mockDaprClient = new Mock(); + var receiver = new PublishSubscribeReceiver(pubSubName, topicName, options, messageHandler, mockDaprClient.Object); + var task = Task.FromException(new InvalidOperationException("Test exception")); - var exception = Assert.Throws(() => - PublishSubscribeReceiver.HandleTaskCompletion(task, null)); - - Assert.IsType(exception.InnerException); - Assert.Equal("Test exception", exception.InnerException.Message); + var exception = Record.Exception(() => receiver.HandleTaskCompletion(task, null)); + + Assert.Null(exception); + } + + [Fact] + public void HandleTaskCompletion_ShouldNotInvokeErrorHandler_WhenTaskSucceeded() + { + const string pubSubName = "testPubSub"; + const string topicName = "testTopic"; + var handlerInvoked = false; + var options = + new DaprSubscriptionOptions(new MessageHandlingPolicy(TimeSpan.FromSeconds(5), TopicResponseAction.Success)) + { + ErrorHandler = _ => handlerInvoked = true + }; + + var messageHandler = new TopicMessageHandler((message, token) => Task.FromResult(TopicResponseAction.Success)); + var mockDaprClient = new Mock(); + var receiver = new PublishSubscribeReceiver(pubSubName, topicName, options, messageHandler, mockDaprClient.Object); + + receiver.HandleTaskCompletion(Task.CompletedTask, null); + + Assert.False(handlerInvoked); + } + + [Fact] + public async Task SubscribeAsync_ShouldThrowDaprException_WhenSidecarUnavailable() + { + const string pubSubName = "testPubSub"; + const string topicName = "testTopic"; + var options = + new DaprSubscriptionOptions(new MessageHandlingPolicy(TimeSpan.FromSeconds(5), TopicResponseAction.Success)); + + var messageHandler = new TopicMessageHandler((message, token) => Task.FromResult(TopicResponseAction.Success)); + var mockDaprClient = new Mock(); + + // Setup the mock to throw RpcException (simulating unavailable sidecar) + mockDaprClient.Setup(client => + client.SubscribeTopicEventsAlpha1(null, null, It.IsAny())) + .Throws(new RpcException(new Status(StatusCode.Unavailable, "Connect Failed"))); + + var receiver = new PublishSubscribeReceiver(pubSubName, topicName, options, messageHandler, mockDaprClient.Object); + + var exception = await Assert.ThrowsAsync(() => receiver.SubscribeAsync()); + + Assert.Contains("testTopic", exception.Message); + Assert.Contains("testPubSub", exception.Message); + Assert.IsType(exception.InnerException); + } + + [Fact] + public async Task SubscribeAsync_ShouldAllowRetry_AfterSidecarFailure() + { + const string pubSubName = "testPubSub"; + const string topicName = "testTopic"; + var options = + new DaprSubscriptionOptions(new MessageHandlingPolicy(TimeSpan.FromSeconds(5), TopicResponseAction.Success)); + + var messageHandler = new TopicMessageHandler((message, token) => Task.FromResult(TopicResponseAction.Success)); + var mockDaprClient = new Mock(); + + // First call throws RpcException + mockDaprClient.Setup(client => + client.SubscribeTopicEventsAlpha1(null, null, It.IsAny())) + .Throws(new RpcException(new Status(StatusCode.Unavailable, "Connect Failed"))); + + var receiver = new PublishSubscribeReceiver(pubSubName, topicName, options, messageHandler, mockDaprClient.Object); + + await Assert.ThrowsAsync(() => receiver.SubscribeAsync()); + + // Now setup the mock to succeed on retry + var mockRequestStream = new Mock>(); + var mockResponseStream = new Mock>(); + var mockCall = + new AsyncDuplexStreamingCall( + mockRequestStream.Object, mockResponseStream.Object, Task.FromResult(new Metadata()), + () => new Status(), () => new Metadata(), () => { }); + + mockDaprClient.Setup(client => + client.SubscribeTopicEventsAlpha1(null, null, It.IsAny())) + .Returns(mockCall); + + // Second call should succeed (hasInitialized was reset) + var retryException = await Record.ExceptionAsync(() => receiver.SubscribeAsync()); + Assert.Null(retryException); + + // Verify the client was called twice + mockDaprClient.Verify(client => + client.SubscribeTopicEventsAlpha1(null, null, It.IsAny()), Times.Exactly(2)); + } + + [Fact] + public async Task SubscribeAsync_ShouldResetHasInitialized_WhenNonRpcExceptionThrown() + { + const string pubSubName = "testPubSub"; + const string topicName = "testTopic"; + var options = + new DaprSubscriptionOptions(new MessageHandlingPolicy(TimeSpan.FromSeconds(5), TopicResponseAction.Success)); + + var messageHandler = new TopicMessageHandler((message, token) => Task.FromResult(TopicResponseAction.Success)); + var mockDaprClient = new Mock(); + + // First call throws a non-RPC exception + mockDaprClient.Setup(client => + client.SubscribeTopicEventsAlpha1(null, null, It.IsAny())) + .Throws(new ObjectDisposedException("client")); + + var receiver = new PublishSubscribeReceiver(pubSubName, topicName, options, messageHandler, mockDaprClient.Object); + + await Assert.ThrowsAsync(() => receiver.SubscribeAsync()); + + // Now setup the mock to succeed on retry + var mockRequestStream = new Mock>(); + var mockResponseStream = new Mock>(); + var mockCall = + new AsyncDuplexStreamingCall( + mockRequestStream.Object, mockResponseStream.Object, Task.FromResult(new Metadata()), + () => new Status(), () => new Metadata(), () => { }); + + mockDaprClient.Setup(client => + client.SubscribeTopicEventsAlpha1(null, null, It.IsAny())) + .Returns(mockCall); + + // Second call should succeed (hasInitialized was reset) + var retryException = await Record.ExceptionAsync(() => receiver.SubscribeAsync()); + Assert.Null(retryException); } }