Do not exclude exit code -1 from restart (#5361)

* Do not exclude exit code -1 from restart

* add more logging to swallowed exceptions

* update with arg validation

* rename exception name

Author: mhoeger

src/WebJobs.Script/Workers/Http/HttpWorkerProcess.cs

@@ -51,14 +51,15 @@ internal override Process CreateWorkerProcess()
             return _processFactory.CreateWorkerProcess(workerContext);
         }
 
-        internal override void HandleWorkerProcessExitError(WorkerProcessExitException langExc)
+        internal override void HandleWorkerProcessExitError(WorkerProcessExitException httpWorkerProcessExitException)
         {
-            // The subscriber of WorkerErrorEvent is expected to Dispose() the errored channel
-            if (langExc != null && langExc.ExitCode != -1)
+            if (httpWorkerProcessExitException == null)
             {
-                _workerProcessLogger.LogDebug(langExc, $"Language Worker Process exited.", _workerProcessArguments.ExecutablePath);
-                _eventManager.Publish(new HttpWorkerErrorEvent(_workerId, langExc));
+                throw new ArgumentNullException(nameof(httpWorkerProcessExitException));
             }
+            // The subscriber of WorkerErrorEvent is expected to Dispose() the errored channel
+            _workerProcessLogger.LogDebug(httpWorkerProcessExitException, $"Language Worker Process exited. Pid={httpWorkerProcessExitException.Pid}.", _workerProcessArguments.ExecutablePath);
+            _eventManager.Publish(new HttpWorkerErrorEvent(_workerId, httpWorkerProcessExitException));
         }
 
         internal override void HandleWorkerProcessRestart()

src/WebJobs.Script/Workers/ProcessManagement/WorkerProcess.cs

@@ -136,11 +136,13 @@ private void OnProcessExited(object sender, EventArgs e)
                 {
                     var processExitEx = new WorkerProcessExitException($"{_process.StartInfo.FileName} exited with code {_process.ExitCode}\n {exceptionMessage}");
                     processExitEx.ExitCode = _process.ExitCode;
+                    processExitEx.Pid = _process.Id;
                     HandleWorkerProcessExitError(processExitEx);
                 }
             }
-            catch (Exception)
+            catch (Exception exc)
             {
+                _workerProcessLogger?.LogDebug(exc, "Exception on worker process exit.");
                 // ignore process is already disposed
             }
         }
@@ -182,8 +184,9 @@ public void Dispose()
                     _process.Dispose();
                 }
             }
-            catch (Exception)
+            catch (Exception exc)
             {
+                _workerProcessLogger?.LogDebug(exc, "Exception on worker disposal.");
                 //ignore
             }
         }

src/WebJobs.Script/Workers/ProcessManagement/WorkerProcessExitException.cs

@@ -16,5 +16,7 @@ internal WorkerProcessExitException(string message, Exception innerException) :
         }
 
         internal int ExitCode { get; set; }
+
+        internal int Pid { get; set; }
     }
 }

src/WebJobs.Script/Workers/Rpc/FunctionRegistration/RpcFunctionInvocationDispatcher.cs

@@ -278,9 +278,17 @@ public async void WorkerError(WorkerErrorEvent workerError)
         {
             if (!_disposing)
             {
-                _logger.LogDebug("Handling WorkerErrorEvent for runtime:{runtime}, workerId:{workerId}", workerError.Language, workerError.WorkerId);
-                AddOrUpdateErrorBucket(workerError);
-                await DisposeAndRestartWorkerChannel(workerError.Language, workerError.WorkerId);
+                if (string.Equals(_workerRuntime, workerError.Language))
+                {
+                    _logger.LogDebug("Handling WorkerErrorEvent for runtime:{runtime}, workerId:{workerId}", workerError.Language, workerError.WorkerId);
+                    AddOrUpdateErrorBucket(workerError);
+                    await DisposeAndRestartWorkerChannel(workerError.Language, workerError.WorkerId);
+                }
+                else
+                {
+                    _logger.LogDebug("Received WorkerErrorEvent for runtime:{runtime}, workerId:{workerId}", workerError.Language, workerError.WorkerId);
+                    _logger.LogDebug("WorkerErrorEvent runtime:{runtime} does not match current runtime:{currentRuntime}. Failed with: {exception}", workerError.Language, _workerRuntime, workerError.Exception);
+                }
             }
         }
 
@@ -297,21 +305,37 @@ private async Task DisposeAndRestartWorkerChannel(string runtime, string workerI
         {
             bool isWebHostChannel = await _webHostLanguageWorkerChannelManager.ShutdownChannelIfExistsAsync(runtime, workerId);
             bool isJobHostChannel = false;
-            if (!isWebHostChannel)
+
+            // Dispose old worker
+            if (isWebHostChannel)
+            {
+                _logger.LogDebug("Disposing WebHost channel for workerId: {channelId}, for runtime:{language}", workerId, runtime);
+            }
+            else
             {
-                _logger.LogDebug("Disposing channel for workerId: {channelId}, for runtime:{language}", workerId, runtime);
                 var channel = _jobHostLanguageWorkerChannelManager.GetChannels().Where(ch => ch.Id == workerId).FirstOrDefault();
                 if (channel != null)
                 {
+                    _logger.LogDebug("Disposing JobHost channel for workerId: {channelId}, for runtime:{language}", workerId, runtime);
                     isJobHostChannel = true;
                     _jobHostLanguageWorkerChannelManager.DisposeAndRemoveChannel(channel);
                 }
+                else
+                {
+                    _logger.LogDebug("Did not find WebHost or JobHost channel to dispose for workerId: {channelId}, runtime:{language}", workerId, runtime);
+                }
             }
+
             if (ShouldRestartWorkerChannel(runtime, isWebHostChannel, isJobHostChannel))
             {
                 _logger.LogDebug("Restarting worker channel for runtime:{runtime}", runtime);
                 await RestartWorkerChannel(runtime, workerId);
             }
+            else
+            {
+                _logger.LogDebug("Skipping worker channel restart for errored worker runtime:{runtime}, current runtime:{currentRuntime}, isWebHostChannel:{isWebHostChannel}, isJobHostChannel:{isJobHostChannel}",
+                    runtime, _workerRuntime, isWebHostChannel, isJobHostChannel);
+            }
         }
 
         internal bool ShouldRestartWorkerChannel(string runtime, bool isWebHostChannel, bool isJobHostChannel)

src/WebJobs.Script/Workers/Rpc/RpcWorkerProcess.cs

@@ -47,14 +47,15 @@ internal override Process CreateWorkerProcess()
             return _processFactory.CreateWorkerProcess(workerContext);
         }
 
-        internal override void HandleWorkerProcessExitError(WorkerProcessExitException langExc)
+        internal override void HandleWorkerProcessExitError(WorkerProcessExitException rpcWorkerProcessExitException)
         {
-            // The subscriber of WorkerErrorEvent is expected to Dispose() the errored channel
-            if (langExc != null && langExc.ExitCode != -1)
+            if (rpcWorkerProcessExitException == null)
             {
-                _workerProcessLogger.LogDebug(langExc, $"Language Worker Process exited.", _workerProcessArguments.ExecutablePath);
-                _eventManager.Publish(new WorkerErrorEvent(_runtime, _workerId, langExc));
+                throw new ArgumentNullException(nameof(rpcWorkerProcessExitException));
             }
+            // The subscriber of WorkerErrorEvent is expected to Dispose() the errored channel
+            _workerProcessLogger.LogDebug(rpcWorkerProcessExitException, $"Language Worker Process exited. Pid={rpcWorkerProcessExitException.Pid}.", _workerProcessArguments.ExecutablePath);
+            _eventManager.Publish(new WorkerErrorEvent(_runtime, _workerId, rpcWorkerProcessExitException));
         }
 
         internal override void HandleWorkerProcessRestart()