Improve error handling for invocation request cancellations (#9158)

Author: liliankasem

src/WebJobs.Script.Grpc/Channel/GrpcWorkerChannel.cs

@@ -738,7 +738,7 @@ internal async Task SendInvocationRequest(ScriptInvocationContext context)
                 // do not send an invocation request if cancellation has been requested
                 if (context.CancellationToken.IsCancellationRequested)
                 {
-                    _workerChannelLogger.LogWarning("Cancellation has been requested. The invocation request with id '{invocationId}' is cancelled and will not be sent to the worker.", invocationId);
+                    _workerChannelLogger.LogWarning("Cancellation has been requested. The invocation request with id '{invocationId}' is canceled and will not be sent to the worker.", invocationId);
                     context.ResultSource.TrySetCanceled();
                     return;
                 }

src/WebJobs.Script.WebHost/WebJobsScriptHostService.cs

@@ -180,7 +180,7 @@ public async Task StartAsync(CancellationToken cancellationToken)
             }
             catch (OperationCanceledException ex)
             {
-                _logger.LogWarning("Host startup operation has been cancelled", ex);
+                _logger.LogWarning("Host startup operation has been canceled", ex);
 
                 if (cancellationToken.IsCancellationRequested)
                 {

src/WebJobs.Script/Description/Workers/FunctionInvocationCanceledException.cs

@@ -0,0 +1,14 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT License. See License.txt in the project root for license information.
+
+using System;
+using Microsoft.Azure.WebJobs.Host;
+
+namespace Microsoft.Azure.WebJobs.Script.Description
+{
+    internal class FunctionInvocationCanceledException : FunctionInvocationException
+    {
+        public FunctionInvocationCanceledException(string invocationId, Exception innerException)
+            : base($"The invocation request with id '{invocationId}' was canceled before the request was sent to the worker.", innerException) { }
+    }
+}

src/WebJobs.Script/Description/Workers/WorkerFunctionInvoker.cs

@@ -91,11 +91,23 @@ protected override async Task<object> InvokeCore(object[] parameters, FunctionIn
             };
 
             string invocationId = context.ExecutionContext.InvocationId.ToString();
-            _logger.LogTrace($"Sending invocation id:{invocationId}");
+            ScriptInvocationResult result = new();
+
+            _logger.LogTrace("Sending invocation id: '{id}", invocationId);
             await _functionDispatcher.InvokeAsync(invocationContext);
-            var result = await invocationContext.ResultSource.Task;
+
+            try
+            {
+                result = await invocationContext.ResultSource.Task;
+            }
+            catch (OperationCanceledException ex)
+            {
+                // Only catch the exception when the task is cancelled, otherwise let it be handled by the ExceptionMiddleware
+                throw new FunctionInvocationCanceledException(invocationId, ex);
+            }
 
             await BindOutputsAsync(triggerValue, context.Binder, result);
+
             return result.Return;
         }
 

test/WebJobs.Script.Tests/Workers/Rpc/GrpcWorkerChannelTests.cs

@@ -450,11 +450,11 @@ public async Task SendInvocationRequest_SignalCancellation_WithoutCapability_NoA
         }
 
         [Fact]
-        public async Task SendInvocationRequest_CancellationAlreadyRequested_ResultSourceCancelled()
+        public async Task SendInvocationRequest_CancellationAlreadyRequested_ResultSourceCanceled()
         {
             var cancellationWaitTimeMs = 3000;
             var invocationId = Guid.NewGuid();
-            var expectedCancellationLog = $"Cancellation has been requested. The invocation request with id '{invocationId}' is cancelled and will not be sent to the worker.";
+            var expectedCancellationLog = $"Cancellation has been requested. The invocation request with id '{invocationId}' is canceled and will not be sent to the worker.";
 
             var cts = new CancellationTokenSource();
             cts.CancelAfter(cancellationWaitTimeMs);