Add retries to WaitForInstanceCompletion

CHANGELOG.md

@@ -1,9 +1,15 @@
 # Changelog
 
+## (Unreleased)
+
+- Add automatic retry on gateway timeout in `GrpcDurableTaskClient.WaitForInstanceCompletionAsync` in [#412](https://github.com/microsoft/durabletask-dotnet/pull/412))
+
 ## v1.10.0
-- Update DurableTask.Core to v3.1.0 and Bump version to v1.10.0 by @nytian in([#411](https://github.com/microsoft/durabletask-dotnet/pull/411))
+
+- Update DurableTask.Core to v3.1.0 and Bump version to v1.10.0 by @nytian in ([#411](https://github.com/microsoft/durabletask-dotnet/pull/411))
 
 ## v1.9.1
+
 - Add basic orchestration and activity execution logs by @cgillum in ([#405](https://github.com/microsoft/durabletask-dotnet/pull/405))
 - Add default version in `TaskOrchestrationContext` by @halspang in ([#408](https://github.com/microsoft/durabletask-dotnet/pull/408))
 

src/Client/Grpc/GrpcDurableTaskClient.cs

@@ -107,7 +107,7 @@
 
         if (Activity.Current?.Id != null || Activity.Current?.TraceStateString != null)
         {
             if (request.ParentTraceContext == null)
             {
                 request.ParentTraceContext = new P.TraceContext();
             }
@@ -125,7 +125,7 @@
 
         DateTimeOffset? startAt = options?.StartAt;
         this.logger.SchedulingOrchestration(
             request.InstanceId,
             orchestratorName,
             sizeInBytes: request.Input != null ? Encoding.UTF8.GetByteCount(request.Input) : 0,
             startAt.GetValueOrDefault(DateTimeOffset.UtcNow));
@@ -348,17 +348,27 @@
             GetInputsAndOutputs = getInputsAndOutputs,
         };
 
-        try
+        while (!cancellation.IsCancellationRequested)
         {
-            P.GetInstanceResponse response = await this.sidecarClient.WaitForInstanceCompletionAsync(
-                request, cancellationToken: cancellation);
-            return this.CreateMetadata(response.OrchestrationState, getInputsAndOutputs);
-        }
-        catch (RpcException e) when (e.StatusCode == StatusCode.Cancelled)
-        {
-            throw new OperationCanceledException(
-                $"The {nameof(this.WaitForInstanceCompletionAsync)} operation was canceled.", e, cancellation);
+            try
+            {
+                P.GetInstanceResponse response = await this.sidecarClient.WaitForInstanceCompletionAsync(
+                    request, cancellationToken: cancellation);
+                return this.CreateMetadata(response.OrchestrationState, getInputsAndOutputs);
+            }
+            catch (RpcException e) when (e.StatusCode == StatusCode.Cancelled)
+            {
+                throw new OperationCanceledException(
+                    $"The {nameof(this.WaitForInstanceCompletionAsync)} operation was canceled.", e, cancellation);
+            }
+            catch (RpcException e) when (e.StatusCode == StatusCode.DeadlineExceeded)
+            {
+                // Gateway timeout/deadline exceeded can happen before the request is completed. Do nothing and retry.
+            }
         }
+
+        // If the operation was cancelled in between requests, we should still throw instead of returning a null value.
+        throw new OperationCanceledException($"The {nameof(this.WaitForInstanceCompletionAsync)} operation was canceled.");
     }
 
     /// <inheritdoc/>