catching exceptions due to race with worker process exit (#8917)

Author: brettsam

src/WebJobs.Script/Workers/ProcessManagement/WorkerProcess.cs

@@ -11,7 +11,6 @@
 using Microsoft.Azure.WebJobs.Logging;
 using Microsoft.Azure.WebJobs.Script.Diagnostics;
 using Microsoft.Azure.WebJobs.Script.Eventing;
-using Microsoft.Azure.WebJobs.Script.Workers.Rpc;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Logging;
 
@@ -140,12 +139,13 @@ private void LogError(string msg)
 
         private void OnProcessExited(object sender, EventArgs e)
         {
+            _workerProcessLogger.LogDebug("Process {processId} has exited with code {exitCode}.", Process?.Id, Process?.ExitCode);
+
             if (Disposing)
             {
                 // No action needed
                 return;
             }
-            string exceptionMessage = string.Join(",", _processStdErrDataQueue.Where(s => !string.IsNullOrEmpty(s)));
 
             try
             {
@@ -160,6 +160,7 @@ private void OnProcessExited(object sender, EventArgs e)
                 }
                 else
                 {
+                    string exceptionMessage = string.Join(",", _processStdErrDataQueue.Where(s => !string.IsNullOrEmpty(s)));
                     var processExitEx = new WorkerProcessExitException($"{Process.StartInfo.FileName} exited with code {Process.ExitCode} (0x{Process.ExitCode.ToString("X")})", new Exception(exceptionMessage));
                     processExitEx.ExitCode = Process.ExitCode;
                     processExitEx.Pid = Process.Id;
@@ -168,7 +169,7 @@ private void OnProcessExited(object sender, EventArgs e)
             }
             catch (Exception exc)
             {
-                _workerProcessLogger?.LogDebug(exc, "Exception on worker process exit.");
+                _workerProcessLogger?.LogDebug(exc, "Exception on worker process exit. Process id: {processId}", Process?.Id);
                 // ignore process is already disposed
             }
             finally
@@ -208,7 +209,17 @@ internal void BuildAndLogConsoleLog(string msg, LogLevel level)
 
         public void WaitForProcessExitInMilliSeconds(int waitTime)
         {
-            Process.WaitForExit(waitTime);
+            try
+            {
+                if (!Process.HasExited)
+                {
+                    Process.WaitForExit(waitTime);
+                }
+            }
+            catch (Exception ex)
+            {
+                _workerProcessLogger.LogDebug(ex, "An exception was thrown while waiting for process {processId} to exit. It is possible that the process had already exited and this can be ignored.", Process?.Id);
+            }
         }
 
         public void Dispose()
@@ -229,7 +240,7 @@ public void Dispose()
                         Process.Kill();
                         if (!Process.WaitForExit(processExitTimeoutInMilliseconds))
                         {
-                            _workerProcessLogger.LogWarning($"Worker process has not exited despite waiting for {processExitTimeoutInMilliseconds} ms");
+                            _workerProcessLogger.LogWarning("Worker process {processId} has not exited despite waiting for {processExitTimeoutInMilliseconds} ms", Process?.Id, processExitTimeoutInMilliseconds);
                         }
                     }
                     Process.Dispose();