Fix GrpcWorkerChannel.StartWorkerProcessAsync timeout

Directory.Build.targets

@@ -2,4 +2,8 @@
 
   <Import Project="$(EngBuildRoot)Engineering.targets" />
 
+  <PropertyGroup>
+    <VSTestResultsDirectory>$(ArtifactsPath)/log/$(ArtifactsProjectName)/tests_$(ArtifactsPivots)/</VSTestResultsDirectory>
+  </PropertyGroup>
+
 </Project>

eng/ci/templates/jobs/run-unit-tests.yml

@@ -16,7 +16,7 @@ jobs:
       displayName: Publish deps.json
       path: $(Build.ArtifactStagingDirectory)
       artifact: WebHost_Deps
-      condition: failed()
+      condition: and(failed(), eq(variables['System.JobAttempt'], 1)) # only publish on first attempt
 
   steps:
   - template: /eng/ci/templates/install-dotnet.yml@self

release_notes.md

@@ -5,4 +5,5 @@
 -->
 - Update Python Worker Version to [4.40.2](https://github.com/Azure/azure-functions-python-worker/releases/tag/4.40.2)
 - Add JitTrace Files for v4.1044
+- Throw exception instead of timing out when worker channel exits before initializing gRPC (#10937)
 - Avoid emitting empty tag values for health check metrics (#11393)

src/WebJobs.Script.Grpc/Channel/GrpcWorkerChannel.cs

@@ -23,7 +23,6 @@
 using Microsoft.Azure.WebJobs.Script.Diagnostics;
 using Microsoft.Azure.WebJobs.Script.Diagnostics.OpenTelemetry;
 using Microsoft.Azure.WebJobs.Script.Eventing;
-using Microsoft.Azure.WebJobs.Script.Exceptions;
 using Microsoft.Azure.WebJobs.Script.Extensions;
 using Microsoft.Azure.WebJobs.Script.Grpc.Eventing;
 using Microsoft.Azure.WebJobs.Script.Grpc.Extensions;
@@ -367,19 +366,39 @@ private void DispatchMessage(InboundGrpcEvent msg)
 
         public bool IsChannelReadyForInvocations()
         {
-            return !_disposing && !_disposed && _state.HasFlag(RpcWorkerChannelState.InvocationBuffersInitialized | RpcWorkerChannelState.Initialized);
+            return !_disposing && !_disposed
+                && _state.HasFlag(
+                    RpcWorkerChannelState.InvocationBuffersInitialized | RpcWorkerChannelState.Initialized);
         }
 
         public async Task StartWorkerProcessAsync(CancellationToken cancellationToken)
         {
-            RegisterCallbackForNextGrpcMessage(MsgType.StartStream, _workerConfig.CountOptions.ProcessStartupTimeout, 1, SendWorkerInitRequest, HandleWorkerStartStreamError);
-            // note: it is important that the ^^^ StartStream is in place *before* we start process the loop, otherwise we get a race condition
+            RegisterCallbackForNextGrpcMessage(
+                MsgType.StartStream,
+                _workerConfig.CountOptions.ProcessStartupTimeout,
+                count: 1,
+                SendWorkerInitRequest,
+                HandleWorkerStartStreamError);
+
+            // note: it is important that the ^^^ StartStream is in place *before* we start process the loop,
+            // otherwise we get a race condition
             _ = ProcessInbound();
 
             _workerChannelLogger.LogDebug("Initiating Worker Process start up");
-            await _rpcWorkerProcess.StartProcessAsync();
-            _state = _state | RpcWorkerChannelState.Initializing;
-            await _workerInitTask.Task;
+            await _rpcWorkerProcess.StartProcessAsync(cancellationToken);
+            _state |= RpcWorkerChannelState.Initializing;
+            Task<int> exited = _rpcWorkerProcess.WaitForExitAsync(cancellationToken);
+            Task winner = await Task.WhenAny(_workerInitTask.Task, exited).WaitAsync(cancellationToken);
+            await winner;
+
+            if (winner == exited)
+            {
+                // Process exited without throwing. We need to throw to indicate process is not running.
+                throw new WorkerProcessExitException("Worker process exited before initializing.")
+                {
+                    ExitCode = await exited,
+                };
+            }
         }
 
         public async Task<WorkerStatus> GetWorkerStatusAsync()

src/WebJobs.Script/DependencyInjection/ScriptStartupTypeLocator.cs

@@ -36,29 +36,37 @@ public sealed class ScriptStartupTypeLocator : IWebJobsStartupTypeLocator
         private readonly IExtensionBundleManager _extensionBundleManager;
         private readonly IFunctionMetadataManager _functionMetadataManager;
         private readonly IMetricsLogger _metricsLogger;
-        private readonly Lazy<IEnumerable<Type>> _startupTypes;
+        private readonly IEnvironment _environment;
         private readonly IOptions<ExtensionRequirementOptions> _extensionRequirementOptions;
+        private readonly Lazy<IEnumerable<Type>> _startupTypes;
         private static string[] _builtinExtensionAssemblies = GetBuiltinExtensionAssemblies();
 
-        public ScriptStartupTypeLocator(string rootScriptPath, ILogger<ScriptStartupTypeLocator> logger, IExtensionBundleManager extensionBundleManager,
-            IFunctionMetadataManager functionMetadataManager, IMetricsLogger metricsLogger, IOptions<ExtensionRequirementOptions> extensionRequirementOptions)
+        public ScriptStartupTypeLocator(
+            string rootScriptPath,
+            ILogger<ScriptStartupTypeLocator> logger,
+            IExtensionBundleManager extensionBundleManager,
+            IFunctionMetadataManager functionMetadataManager,
+            IMetricsLogger metricsLogger,
+            IEnvironment environment,
+            IOptions<ExtensionRequirementOptions> extensionRequirementOptions)
         {
             _rootScriptPath = rootScriptPath ?? throw new ArgumentNullException(nameof(rootScriptPath));
             _extensionBundleManager = extensionBundleManager ?? throw new ArgumentNullException(nameof(extensionBundleManager));
             _logger = logger;
             _functionMetadataManager = functionMetadataManager;
             _metricsLogger = metricsLogger;
-            _startupTypes = new Lazy<IEnumerable<Type>>(() => GetExtensionsStartupTypesAsync().ConfigureAwait(false).GetAwaiter().GetResult());
+            _environment = environment;
             _extensionRequirementOptions = extensionRequirementOptions;
+            _startupTypes = new Lazy<IEnumerable<Type>>(() => GetExtensionsStartupTypesAsync().ConfigureAwait(false).GetAwaiter().GetResult());
         }
 
         private static string[] GetBuiltinExtensionAssemblies()
         {
-            return new[]
-            {
+            return
+            [
                 typeof(WebJobs.Extensions.Http.HttpWebJobsStartup).Assembly.GetName().Name,
                 typeof(WebJobs.Extensions.ExtensionsWebJobsStartup).Assembly.GetName().Name
-            };
+            ];
         }
 
         public Type[] GetStartupTypes()
@@ -102,11 +110,11 @@ public async Task<IEnumerable<Type>> GetExtensionsStartupTypesAsync()
                 }
             }
 
-            bool isDotnetIsolatedApp = Utility.IsDotnetIsolatedApp(SystemEnvironment.Instance, functionMetadataCollection);
+            bool isDotnetIsolatedApp = Utility.IsDotnetIsolatedApp(_environment, functionMetadataCollection);
             bool isDotnetApp = isPrecompiledFunctionApp || isDotnetIsolatedApp;
-            var isLogicApp = SystemEnvironment.Instance.IsLogicApp();
+            var isLogicApp = _environment.IsLogicApp();
 
-            if (SystemEnvironment.Instance.IsPlaceholderModeEnabled())
+            if (_environment.IsPlaceholderModeEnabled())
             {
                 // Do not move this.
                 // Calling this log statement in the placeholder mode to avoid jitting during specialization

src/WebJobs.Script/Extensions/ExceptionExtensions.cs

@@ -1,7 +1,8 @@
 // Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the MIT License. See License.txt in the project root for license information.
 
-using System.Reflection;
+using System.Collections.Generic;
+using System.Runtime.ExceptionServices;
 using System.Runtime.InteropServices;
 using System.Threading;
 using Microsoft.Azure.WebJobs.Host.Diagnostics;
@@ -32,12 +33,22 @@ or SEHException
 
         public static string ToFormattedString(this Exception exception)
         {
-            if (exception == null)
+            ArgumentNullException.ThrowIfNull(exception);
+            return ExceptionFormatter.GetFormattedException(exception);
+        }
+
+        public static void ThrowIfErrorsPresent(IList<Exception> exceptions, string message = null)
+        {
+            switch (exceptions)
             {
-                throw new ArgumentNullException(nameof(exception));
+                case null or []:
+                    return;
+                case [Exception e]:
+                    ExceptionDispatchInfo.Capture(e).Throw();
+                    return;
+                default:
+                    throw new AggregateException(message, exceptions);
             }
-
-            return ExceptionFormatter.GetFormattedException(exception);
         }
     }
 }

src/WebJobs.Script/Host/IWorkerFunctionMetadataProvider.cs

@@ -9,14 +9,14 @@
 namespace Microsoft.Azure.WebJobs.Script
 {
     /// <summary>
-    /// Defines an interface for fetching function metadata from Out-of-Proc language workers
+    /// Defines an interface for fetching function metadata from Out-of-Proc language workers.
     /// </summary>
     internal interface IWorkerFunctionMetadataProvider
     {
         ImmutableDictionary<string, ImmutableArray<string>> FunctionErrors { get; }
 
         /// <summary>
-        /// Attempts to get function metadata from Out-of-Proc language workers
+        /// Attempts to get function metadata from Out-of-Proc language workers.
         /// </summary>
         /// <returns>FunctionMetadataResult that either contains the function metadata or indicates that a fall back option for fetching metadata should be used</returns>
         Task<FunctionMetadataResult> GetFunctionMetadataAsync(IEnumerable<RpcWorkerConfig> workerConfigs, bool forceRefresh = false);

src/WebJobs.Script/Host/ScriptHostState.cs

@@ -6,7 +6,7 @@ namespace Microsoft.Azure.WebJobs.Script
     public enum ScriptHostState
     {
         /// <summary>
-        /// The host has not yet been created
+        /// The host has not yet been created.
         /// </summary>
         Default,
 
@@ -28,7 +28,7 @@ public enum ScriptHostState
         Running,
 
         /// <summary>
-        /// The host is in an error state
+        /// The host is in an error state.
         /// </summary>
         Error,
 
@@ -43,7 +43,7 @@ public enum ScriptHostState
         Stopped,
 
         /// <summary>
-        /// The host is offline
+        /// The host is offline.
         /// </summary>
         Offline
     }

src/WebJobs.Script/Host/WorkerFunctionMetadataProvider.cs

@@ -77,7 +77,7 @@ public async Task<FunctionMetadataResult> GetFunctionMetadataAsync(IEnumerable<R
                 // forceRefresh will be false when bundle is not used (dotnet and dotnet-isolated).
                 if (!_environment.IsPlaceholderModeEnabled() && forceRefresh && !_scriptOptions.CurrentValue.IsFileSystemReadOnly)
                 {
-                    _channelManager.ShutdownChannelsAsync().GetAwaiter().GetResult();
+                    await _channelManager.ShutdownChannelsAsync();
                 }
 
                 var channels = _channelManager.GetChannels(_workerRuntime);
@@ -107,6 +107,7 @@ public async Task<FunctionMetadataResult> GetFunctionMetadataAsync(IEnumerable<R
                     throw new InvalidOperationException($"No initialized language worker channel found for runtime: {_workerRuntime}.");
                 }
 
+                List<Exception> errors = null;
                 foreach (string workerId in channels.Keys.ToList())
                 {
                     if (channels.TryGetValue(workerId, out TaskCompletionSource<IRpcWorkerChannel> languageWorkerChannelTask))
@@ -129,7 +130,7 @@ public async Task<FunctionMetadataResult> GetFunctionMetadataAsync(IEnumerable<R
                             }
 
                             _functions = functions.ToImmutableArray();
-                            _logger.FunctionsReturnedByProvider(_functions.IsDefault ? 0 : _functions.Count(), _metadataProviderName);
+                            _logger.FunctionsReturnedByProvider(_functions.Length, _metadataProviderName);
 
                             // Validate if the app has functions in legacy format and add in logs to inform about the mixed app
                             _ = Task.Delay(TimeSpan.FromMinutes(1)).ContinueWith(t => ValidateFunctionAppFormat(_scriptOptions.CurrentValue.ScriptPath, _logger, _environment));
@@ -140,9 +141,13 @@ public async Task<FunctionMetadataResult> GetFunctionMetadataAsync(IEnumerable<R
                         {
                             _logger.LogWarning(ex, "Removing errored webhost language worker channel for runtime: {workerRuntime} workerId:{workerId}", _workerRuntime, workerId);
                             await _channelManager.ShutdownChannelIfExistsAsync(_workerRuntime, workerId, ex);
+                            errors ??= [];
+                            errors.Add(ex);
                         }
                     }
                 }
+
+                ExceptionExtensions.ThrowIfErrorsPresent(errors, "Errors getting function metadata from workers.");
             }
 
             return new FunctionMetadataResult(useDefaultMetadataIndexing: false, _functions);

src/WebJobs.Script/ScriptHostBuilderExtensions.cs

@@ -142,12 +142,13 @@ public static IHostBuilder AddScriptHost(this IHostBuilder builder,
                 var bundleManager = new ExtensionBundleManager(extensionBundleOptions, SystemEnvironment.Instance, loggerFactory, configOption, httpClientFactory);
                 var metadataServiceManager = applicationOptions.RootServiceProvider.GetService<IFunctionMetadataManager>();
 
-                ScriptStartupTypeLocator locator = new(
+                var locator = new ScriptStartupTypeLocator(
                     applicationOptions.ScriptPath,
                     loggerFactory.CreateLogger<ScriptStartupTypeLocator>(),
                     bundleManager,
                     metadataServiceManager,
                     metricsLogger,
+                    SystemEnvironment.Instance,
                     extensionRequirementOptions);
 
                 // The locator (and thus the bundle manager) need to be created now in order to configure app configuration.

src/WebJobs.Script/Workers/ProcessManagement/IWorkerProcess.cs

@@ -2,8 +2,8 @@
 // Licensed under the MIT License. See License.txt in the project root for license information.
 
 using System.Diagnostics;
+using System.Threading;
 using System.Threading.Tasks;
-using Microsoft.Azure.WebJobs.Script.Scale;
 
 namespace Microsoft.Azure.WebJobs.Script.Workers
 {
@@ -13,7 +13,9 @@ public interface IWorkerProcess
 
         Process Process { get; }
 
-        Task StartProcessAsync();
+        Task StartProcessAsync(CancellationToken cancellationToken = default);
+
+        Task<int> WaitForExitAsync(CancellationToken cancellationToken = default);
 
         void WaitForProcessExitInMilliSeconds(int waitTime);
     }