Fix SubscribeAsync silently failing when Dapr sidecar is unavailable (#1663)

- Throw DaprException wrapping RpcException on initial connection failure
- Add SubscriptionErrorHandler callback for runtime errors in background tasks
- Reset hasInitialized on any failure (initial or mid-stream) to allow retry
- Guard ErrorHandler invocation with try/catch to prevent unobserved exceptions
- Use CancellationToken.None for ContinueWith to ensure error handlers fire
- Include topic/pubsub names in error messages for multi-subscription debugging

Resolves #1663

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

Author: GeorgeTsiokos

src/Dapr.Messaging/PublishSubscribe/DaprSubscriptionOptions.cs

@@ -40,5 +40,11 @@ public sealed record DaprSubscriptionOptions(MessageHandlingPolicy MessageHandli
     /// been signaled.
     /// </summary>
     public TimeSpan MaximumCleanupTimeout { get; init; } = TimeSpan.FromSeconds(30);
+
+    /// <summary>
+    /// An optional callback invoked when errors occur during an active subscription.
+    /// If not set, runtime errors from background tasks will be silently swallowed.
+    /// </summary>
+    public SubscriptionErrorHandler? ErrorHandler { get; init; }
 }
 

src/Dapr.Messaging/PublishSubscribe/PublishSubscribeReceiver.cs

@@ -12,6 +12,7 @@
 // ------------------------------------------------------------------------
 
 using System.Threading.Channels;
+using Dapr;
 using Dapr.AppCallback.Autogen.Grpc.v1;
 using Grpc.Core;
 using P = Dapr.Client.Autogen.Grpc.v1;
@@ -118,18 +119,36 @@ internal async Task SubscribeAsync(CancellationToken cancellationToken = default
             return;
         }
 
-        var stream = await GetStreamAsync(cancellationToken);
+        AsyncDuplexStreamingCall<P.SubscribeTopicEventsRequestAlpha1, P.SubscribeTopicEventsResponseAlpha1> stream;
+        try
+        {
+            stream = await GetStreamAsync(cancellationToken);
+        }
+        catch (RpcException ex)
+        {
+            // Reset so the caller can retry after the sidecar becomes available
+            Interlocked.Exchange(ref hasInitialized, 0);
+            throw new DaprException(
+                $"Unable to subscribe to topic '{topicName}' on pubsub '{pubSubName}'. The Dapr sidecar may be unavailable.",
+                ex);
+        }
+        catch (Exception)
+        {
+            // Reset so the caller can retry regardless of the exception type
+            Interlocked.Exchange(ref hasInitialized, 0);
+            throw;
+        }
 
         //Retrieve the messages from the sidecar and write to the messages channel - start without awaiting so this isn't blocking
         _ = FetchDataFromSidecarAsync(stream, topicMessagesChannel.Writer, cancellationToken)
-            .ContinueWith(HandleTaskCompletion, null, cancellationToken, TaskContinuationOptions.OnlyOnFaulted,
+            .ContinueWith(HandleTaskCompletion, null, CancellationToken.None, TaskContinuationOptions.OnlyOnFaulted,
                 TaskScheduler.Default);
 
         //Process the messages as they're written to either channel
         _ = ProcessAcknowledgementChannelMessagesAsync(stream, cancellationToken).ContinueWith(HandleTaskCompletion,
-            null, cancellationToken, TaskContinuationOptions.OnlyOnFaulted, TaskScheduler.Default);
+            null, CancellationToken.None, TaskContinuationOptions.OnlyOnFaulted, TaskScheduler.Default);
         _ = ProcessTopicChannelMessagesAsync(cancellationToken).ContinueWith(HandleTaskCompletion, null,
-            cancellationToken,
+            CancellationToken.None,
             TaskContinuationOptions.OnlyOnFaulted, TaskScheduler.Default);
     }
 
@@ -149,11 +168,28 @@ internal async Task WriteAcknowledgementToChannelAsync(TopicAcknowledgement ackn
     }
 
     //Exposed for testing purposes only
-    internal static void HandleTaskCompletion(Task task, object? state)
+    internal void HandleTaskCompletion(Task task, object? state)
     {
-        if (task.Exception != null)
+        if (task.Exception is null)
+        {
+            return;
+        }
+
+        // Allow the caller to retry after a background task failure
+        Interlocked.Exchange(ref hasInitialized, 0);
+
+        var innerException = task.Exception.InnerException ?? task.Exception;
+        var daprException = new DaprException(
+            $"An error occurred during an active subscription to topic '{topicName}' on pubsub '{pubSubName}'.",
+            innerException);
+
+        try
+        {
+            options.ErrorHandler?.Invoke(daprException);
+        }
+        catch (Exception)
         {
-            throw task.Exception;
+            // Prevent a faulty error handler from becoming an unobserved task exception
         }
     }
 

src/Dapr.Messaging/PublishSubscribe/SubscriptionErrorHandler.cs

@@ -0,0 +1,20 @@
+// ------------------------------------------------------------------------
+// Copyright 2024 The Dapr Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//     http://www.apache.org/licenses/LICENSE-2.0
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+// ------------------------------------------------------------------------
+
+namespace Dapr.Messaging.PublishSubscribe;
+
+/// <summary>
+/// A delegate that handles errors occurring during an active subscription.
+/// </summary>
+/// <param name="exception">The <see cref="DaprException"/> wrapping the original error.</param>
+public delegate void SubscriptionErrorHandler(DaprException exception);

test/Dapr.Messaging.Test/PublishSubscribe/PublishSubscribeReceiverTests.cs

@@ -12,6 +12,7 @@
 // ------------------------------------------------------------------------
 
 using System.Threading.Channels;
+using Dapr;
 using Dapr.AppCallback.Autogen.Grpc.v1;
 using Dapr.Messaging.PublishSubscribe;
 using Grpc.Core;
@@ -193,14 +194,194 @@ public async Task DisposeAsync_ShouldCompleteChannels()
     }
 
     [Fact]
-    public void HandleTaskCompletion_ShouldThrowException_WhenTaskHasException()
+    public void HandleTaskCompletion_ShouldInvokeErrorHandler_WhenTaskHasException()
     {
+        const string pubSubName = "testPubSub";
+        const string topicName = "testTopic";
+        DaprException? receivedException = null;
+        var options =
+            new DaprSubscriptionOptions(new MessageHandlingPolicy(TimeSpan.FromSeconds(5), TopicResponseAction.Success))
+            {
+                ErrorHandler = ex => receivedException = ex
+            };
+
+        var messageHandler = new TopicMessageHandler((message, token) => Task.FromResult(TopicResponseAction.Success));
+        var mockDaprClient = new Mock<P.Dapr.DaprClient>();
+        var receiver = new PublishSubscribeReceiver(pubSubName, topicName, options, messageHandler, mockDaprClient.Object);
+
+        var task = Task.FromException(new InvalidOperationException("Test exception"));
+
+        receiver.HandleTaskCompletion(task, null);
+
+        Assert.NotNull(receivedException);
+        Assert.IsType<InvalidOperationException>(receivedException.InnerException);
+        Assert.Equal("Test exception", receivedException.InnerException.Message);
+        Assert.Contains("testTopic", receivedException.Message);
+        Assert.Contains("testPubSub", receivedException.Message);
+    }
+
+    [Fact]
+    public void HandleTaskCompletion_ShouldNotThrow_WhenNoErrorHandler()
+    {
+        const string pubSubName = "testPubSub";
+        const string topicName = "testTopic";
+        var options =
+            new DaprSubscriptionOptions(new MessageHandlingPolicy(TimeSpan.FromSeconds(5), TopicResponseAction.Success));
+
+        var messageHandler = new TopicMessageHandler((message, token) => Task.FromResult(TopicResponseAction.Success));
+        var mockDaprClient = new Mock<P.Dapr.DaprClient>();
+        var receiver = new PublishSubscribeReceiver(pubSubName, topicName, options, messageHandler, mockDaprClient.Object);
+
+        var task = Task.FromException(new InvalidOperationException("Test exception"));
+
+        var exception = Record.Exception(() => receiver.HandleTaskCompletion(task, null));
+
+        Assert.Null(exception);
+    }
+
+    [Fact]
+    public void HandleTaskCompletion_ShouldNotThrow_WhenErrorHandlerThrows()
+    {
+        const string pubSubName = "testPubSub";
+        const string topicName = "testTopic";
+        var options =
+            new DaprSubscriptionOptions(new MessageHandlingPolicy(TimeSpan.FromSeconds(5), TopicResponseAction.Success))
+            {
+                ErrorHandler = _ => throw new InvalidOperationException("Handler failed")
+            };
+
+        var messageHandler = new TopicMessageHandler((message, token) => Task.FromResult(TopicResponseAction.Success));
+        var mockDaprClient = new Mock<P.Dapr.DaprClient>();
+        var receiver = new PublishSubscribeReceiver(pubSubName, topicName, options, messageHandler, mockDaprClient.Object);
+
         var task = Task.FromException(new InvalidOperationException("Test exception"));
 
-        var exception = Assert.Throws<AggregateException>(() => 
-            PublishSubscribeReceiver.HandleTaskCompletion(task, null));
-            
-        Assert.IsType<InvalidOperationException>(exception.InnerException);
-        Assert.Equal("Test exception", exception.InnerException.Message);
+        var exception = Record.Exception(() => receiver.HandleTaskCompletion(task, null));
+
+        Assert.Null(exception);
+    }
+
+    [Fact]
+    public void HandleTaskCompletion_ShouldNotInvokeErrorHandler_WhenTaskSucceeded()
+    {
+        const string pubSubName = "testPubSub";
+        const string topicName = "testTopic";
+        var handlerInvoked = false;
+        var options =
+            new DaprSubscriptionOptions(new MessageHandlingPolicy(TimeSpan.FromSeconds(5), TopicResponseAction.Success))
+            {
+                ErrorHandler = _ => handlerInvoked = true
+            };
+
+        var messageHandler = new TopicMessageHandler((message, token) => Task.FromResult(TopicResponseAction.Success));
+        var mockDaprClient = new Mock<P.Dapr.DaprClient>();
+        var receiver = new PublishSubscribeReceiver(pubSubName, topicName, options, messageHandler, mockDaprClient.Object);
+
+        receiver.HandleTaskCompletion(Task.CompletedTask, null);
+
+        Assert.False(handlerInvoked);
+    }
+
+    [Fact]
+    public async Task SubscribeAsync_ShouldThrowDaprException_WhenSidecarUnavailable()
+    {
+        const string pubSubName = "testPubSub";
+        const string topicName = "testTopic";
+        var options =
+            new DaprSubscriptionOptions(new MessageHandlingPolicy(TimeSpan.FromSeconds(5), TopicResponseAction.Success));
+
+        var messageHandler = new TopicMessageHandler((message, token) => Task.FromResult(TopicResponseAction.Success));
+        var mockDaprClient = new Mock<P.Dapr.DaprClient>();
+
+        // Setup the mock to throw RpcException (simulating unavailable sidecar)
+        mockDaprClient.Setup(client =>
+                client.SubscribeTopicEventsAlpha1(null, null, It.IsAny<CancellationToken>()))
+            .Throws(new RpcException(new Status(StatusCode.Unavailable, "Connect Failed")));
+
+        var receiver = new PublishSubscribeReceiver(pubSubName, topicName, options, messageHandler, mockDaprClient.Object);
+
+        var exception = await Assert.ThrowsAsync<DaprException>(() => receiver.SubscribeAsync());
+
+        Assert.Contains("testTopic", exception.Message);
+        Assert.Contains("testPubSub", exception.Message);
+        Assert.IsType<RpcException>(exception.InnerException);
+    }
+
+    [Fact]
+    public async Task SubscribeAsync_ShouldAllowRetry_AfterSidecarFailure()
+    {
+        const string pubSubName = "testPubSub";
+        const string topicName = "testTopic";
+        var options =
+            new DaprSubscriptionOptions(new MessageHandlingPolicy(TimeSpan.FromSeconds(5), TopicResponseAction.Success));
+
+        var messageHandler = new TopicMessageHandler((message, token) => Task.FromResult(TopicResponseAction.Success));
+        var mockDaprClient = new Mock<P.Dapr.DaprClient>();
+
+        // First call throws RpcException
+        mockDaprClient.Setup(client =>
+                client.SubscribeTopicEventsAlpha1(null, null, It.IsAny<CancellationToken>()))
+            .Throws(new RpcException(new Status(StatusCode.Unavailable, "Connect Failed")));
+
+        var receiver = new PublishSubscribeReceiver(pubSubName, topicName, options, messageHandler, mockDaprClient.Object);
+
+        await Assert.ThrowsAsync<DaprException>(() => receiver.SubscribeAsync());
+
+        // Now setup the mock to succeed on retry
+        var mockRequestStream = new Mock<IClientStreamWriter<P.SubscribeTopicEventsRequestAlpha1>>();
+        var mockResponseStream = new Mock<IAsyncStreamReader<P.SubscribeTopicEventsResponseAlpha1>>();
+        var mockCall =
+            new AsyncDuplexStreamingCall<P.SubscribeTopicEventsRequestAlpha1, P.SubscribeTopicEventsResponseAlpha1>(
+                mockRequestStream.Object, mockResponseStream.Object, Task.FromResult(new Metadata()),
+                () => new Status(), () => new Metadata(), () => { });
+
+        mockDaprClient.Setup(client =>
+                client.SubscribeTopicEventsAlpha1(null, null, It.IsAny<CancellationToken>()))
+            .Returns(mockCall);
+
+        // Second call should succeed (hasInitialized was reset)
+        var retryException = await Record.ExceptionAsync(() => receiver.SubscribeAsync());
+        Assert.Null(retryException);
+
+        // Verify the client was called twice
+        mockDaprClient.Verify(client =>
+            client.SubscribeTopicEventsAlpha1(null, null, It.IsAny<CancellationToken>()), Times.Exactly(2));
+    }
+
+    [Fact]
+    public async Task SubscribeAsync_ShouldResetHasInitialized_WhenNonRpcExceptionThrown()
+    {
+        const string pubSubName = "testPubSub";
+        const string topicName = "testTopic";
+        var options =
+            new DaprSubscriptionOptions(new MessageHandlingPolicy(TimeSpan.FromSeconds(5), TopicResponseAction.Success));
+
+        var messageHandler = new TopicMessageHandler((message, token) => Task.FromResult(TopicResponseAction.Success));
+        var mockDaprClient = new Mock<P.Dapr.DaprClient>();
+
+        // First call throws a non-RPC exception
+        mockDaprClient.Setup(client =>
+                client.SubscribeTopicEventsAlpha1(null, null, It.IsAny<CancellationToken>()))
+            .Throws(new ObjectDisposedException("client"));
+
+        var receiver = new PublishSubscribeReceiver(pubSubName, topicName, options, messageHandler, mockDaprClient.Object);
+
+        await Assert.ThrowsAsync<ObjectDisposedException>(() => receiver.SubscribeAsync());
+
+        // Now setup the mock to succeed on retry
+        var mockRequestStream = new Mock<IClientStreamWriter<P.SubscribeTopicEventsRequestAlpha1>>();
+        var mockResponseStream = new Mock<IAsyncStreamReader<P.SubscribeTopicEventsResponseAlpha1>>();
+        var mockCall =
+            new AsyncDuplexStreamingCall<P.SubscribeTopicEventsRequestAlpha1, P.SubscribeTopicEventsResponseAlpha1>(
+                mockRequestStream.Object, mockResponseStream.Object, Task.FromResult(new Metadata()),
+                () => new Status(), () => new Metadata(), () => { });
+
+        mockDaprClient.Setup(client =>
+                client.SubscribeTopicEventsAlpha1(null, null, It.IsAny<CancellationToken>()))
+            .Returns(mockCall);
+
+        // Second call should succeed (hasInitialized was reset)
+        var retryException = await Record.ExceptionAsync(() => receiver.SubscribeAsync());
+        Assert.Null(retryException);
     }
 }