diff --git a/CHANGELOG.md b/CHANGELOG.md index 79ce7fb2d..68b500cc5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,9 +1,15 @@ # Changelog +## (Unreleased) + +- Add automatic retry on gateway timeout in `GrpcDurableTaskClient.WaitForInstanceCompletionAsync` in [#412](https://github.com/microsoft/durabletask-dotnet/pull/412)) + ## v1.10.0 -- Update DurableTask.Core to v3.1.0 and Bump version to v1.10.0 by @nytian in([#411](https://github.com/microsoft/durabletask-dotnet/pull/411)) + +- Update DurableTask.Core to v3.1.0 and Bump version to v1.10.0 by @nytian in ([#411](https://github.com/microsoft/durabletask-dotnet/pull/411)) ## v1.9.1 + - Add basic orchestration and activity execution logs by @cgillum in ([#405](https://github.com/microsoft/durabletask-dotnet/pull/405)) - Add default version in `TaskOrchestrationContext` by @halspang in ([#408](https://github.com/microsoft/durabletask-dotnet/pull/408)) diff --git a/src/Client/Grpc/GrpcDurableTaskClient.cs b/src/Client/Grpc/GrpcDurableTaskClient.cs index 4d32093e9..a4960ae93 100644 --- a/src/Client/Grpc/GrpcDurableTaskClient.cs +++ b/src/Client/Grpc/GrpcDurableTaskClient.cs @@ -348,17 +348,27 @@ public override async Task WaitForInstanceCompletionAsync GetInputsAndOutputs = getInputsAndOutputs, }; - try + while (!cancellation.IsCancellationRequested) { - P.GetInstanceResponse response = await this.sidecarClient.WaitForInstanceCompletionAsync( - request, cancellationToken: cancellation); - return this.CreateMetadata(response.OrchestrationState, getInputsAndOutputs); - } - catch (RpcException e) when (e.StatusCode == StatusCode.Cancelled) - { - throw new OperationCanceledException( - $"The {nameof(this.WaitForInstanceCompletionAsync)} operation was canceled.", e, cancellation); + try + { + P.GetInstanceResponse response = await this.sidecarClient.WaitForInstanceCompletionAsync( + request, cancellationToken: cancellation); + return this.CreateMetadata(response.OrchestrationState, getInputsAndOutputs); + } + catch (RpcException e) when (e.StatusCode == StatusCode.Cancelled) + { + throw new OperationCanceledException( + $"The {nameof(this.WaitForInstanceCompletionAsync)} operation was canceled.", e, cancellation); + } + catch (RpcException e) when (e.StatusCode == StatusCode.DeadlineExceeded) + { + // Gateway timeout/deadline exceeded can happen before the request is completed. Do nothing and retry. + } } + + // If the operation was cancelled in between requests, we should still throw instead of returning a null value. + throw new OperationCanceledException($"The {nameof(this.WaitForInstanceCompletionAsync)} operation was canceled."); } ///