Add retries to WaitForInstanceCompletion (#412)

For certain backends (like DTS), there can be a hard limit of 4 minutes for idle requests. As a WaitForInstanceCompletion sits idle while waiting, it can receive a 504. This change allows for the SDK to retry instead of returning that error to the caller.

Signed-off-by: halspang <[email protected]>

Author: halspang

CHANGELOG.md

@@ -1,9 +1,15 @@
 # Changelog
 
+## (Unreleased)
+
+- Add automatic retry on gateway timeout in `GrpcDurableTaskClient.WaitForInstanceCompletionAsync` in [#412](https://github.com/microsoft/durabletask-dotnet/pull/412))
+
 ## v1.10.0
-- Update DurableTask.Core to v3.1.0 and Bump version to v1.10.0 by @nytian in([#411](https://github.com/microsoft/durabletask-dotnet/pull/411))
+
+- Update DurableTask.Core to v3.1.0 and Bump version to v1.10.0 by @nytian in ([#411](https://github.com/microsoft/durabletask-dotnet/pull/411))
 
 ## v1.9.1
+
 - Add basic orchestration and activity execution logs by @cgillum in ([#405](https://github.com/microsoft/durabletask-dotnet/pull/405))
 - Add default version in `TaskOrchestrationContext` by @halspang in ([#408](https://github.com/microsoft/durabletask-dotnet/pull/408))
 

src/Client/Grpc/GrpcDurableTaskClient.cs

@@ -348,17 +348,27 @@ public override async Task<OrchestrationMetadata> WaitForInstanceCompletionAsync
             GetInputsAndOutputs = getInputsAndOutputs,
         };
 
-        try
+        while (!cancellation.IsCancellationRequested)
         {
-            P.GetInstanceResponse response = await this.sidecarClient.WaitForInstanceCompletionAsync(
-                request, cancellationToken: cancellation);
-            return this.CreateMetadata(response.OrchestrationState, getInputsAndOutputs);
-        }
-        catch (RpcException e) when (e.StatusCode == StatusCode.Cancelled)
-        {
-            throw new OperationCanceledException(
-                $"The {nameof(this.WaitForInstanceCompletionAsync)} operation was canceled.", e, cancellation);
+            try
+            {
+                P.GetInstanceResponse response = await this.sidecarClient.WaitForInstanceCompletionAsync(
+                    request, cancellationToken: cancellation);
+                return this.CreateMetadata(response.OrchestrationState, getInputsAndOutputs);
+            }
+            catch (RpcException e) when (e.StatusCode == StatusCode.Cancelled)
+            {
+                throw new OperationCanceledException(
+                    $"The {nameof(this.WaitForInstanceCompletionAsync)} operation was canceled.", e, cancellation);
+            }
+            catch (RpcException e) when (e.StatusCode == StatusCode.DeadlineExceeded)
+            {
+                // Gateway timeout/deadline exceeded can happen before the request is completed. Do nothing and retry.
+            }
         }
+
+        // If the operation was cancelled in between requests, we should still throw instead of returning a null value.
+        throw new OperationCanceledException($"The {nameof(this.WaitForInstanceCompletionAsync)} operation was canceled.");
     }
 
     /// <inheritdoc/>