handling null refs in channel disposal (#9339)

Author: brettsam

src/WebJobs.Script.Grpc/Channel/GrpcWorkerChannel.cs

@@ -1261,16 +1261,44 @@ protected virtual void Dispose(bool disposing)
                     _timer?.Dispose();
 
                     // unlink function inputs
-                    foreach (var link in _inputLinks)
+                    if (_inputLinks is not null)
                     {
-                        link.Dispose();
+                        foreach (var link in _inputLinks)
+                        {
+                            if (link is null)
+                            {
+                                // This log is temporarily added for diagnostic purposes.
+                                _workerChannelLogger.LogDebug("An input link is null. Skipping disposal.");
+                            }
+
+                            link?.Dispose();
+                        }
+                    }
+                    else
+                    {
+                        // This log is temporarily added for diagnostic purposes.
+                        _workerChannelLogger.LogDebug("The input links collection is null. Skipping disposal of any individual input links.");
                     }
 
                     (_rpcWorkerProcess as IDisposable)?.Dispose();
 
-                    foreach (var sub in _eventSubscriptions)
+                    if (_eventSubscriptions is not null)
+                    {
+                        foreach (var sub in _eventSubscriptions)
+                        {
+                            if (sub is null)
+                            {
+                                // This log is temporarily added for diagnostic purposes.
+                                _workerChannelLogger.LogDebug("An event subscription is null. Skipping disposal.");
+                            }
+
+                            sub?.Dispose();
+                        }
+                    }
+                    else
                     {
-                        sub.Dispose();
+                        // This log is temporarily added for diagnostic purposes.
+                        _workerChannelLogger.LogDebug("The event subscriptions collection is null. Skipping disposal of any individual subscriptions.");
                     }
 
                     // shut down the channels

src/WebJobs.Script/Workers/Rpc/FunctionRegistration/RpcFunctionInvocationDispatcher.cs

@@ -507,16 +507,29 @@ private async Task DisposeAndRestartWorkerChannel(string runtime, string workerI
             }
 
             _logger.LogDebug("Attempting to dispose webhost or jobhost channel for workerId: '{channelId}', runtime: '{language}'", workerId, runtime);
-            bool isWebHostChannelDisposed = await _webHostLanguageWorkerChannelManager.ShutdownChannelIfExistsAsync(runtime, workerId, workerException);
+
+            bool isWebHostChannelDisposed = false;
             bool isJobHostChannelDisposed = false;
-            if (!isWebHostChannelDisposed)
+
+            try
             {
-                isJobHostChannelDisposed = await _jobHostLanguageWorkerChannelManager.ShutdownChannelIfExistsAsync(workerId, workerException);
-            }
+                isWebHostChannelDisposed = await _webHostLanguageWorkerChannelManager.ShutdownChannelIfExistsAsync(runtime, workerId, workerException);
+                if (!isWebHostChannelDisposed)
+                {
+                    isJobHostChannelDisposed = await _jobHostLanguageWorkerChannelManager.ShutdownChannelIfExistsAsync(workerId, workerException);
+                }
 
-            if (!isWebHostChannelDisposed && !isJobHostChannelDisposed)
+                if (!isWebHostChannelDisposed && !isJobHostChannelDisposed)
+                {
+                    _logger.LogDebug("Did not find WebHost or JobHost channel to dispose for workerId: '{channelId}', runtime: '{language}'", workerId, runtime);
+                }
+            }
+            catch (Exception ex)
             {
-                _logger.LogDebug("Did not find WebHost or JobHost channel to dispose for workerId: '{channelId}', runtime: '{language}'", workerId, runtime);
+                // If an exception was thrown while trying to shut down a channel, we're left in an undetermined state. The safest thing to do is
+                // to restart the entire host and let everything come back up from scratch.
+                _logger.LogError(ex, "Error while shutting down channel for workerId '{channelId}'. Shutting down and proactively recycling the Functions Host to recover.", workerId);
+                _applicationLifetime.StopApplication();
             }
 
             if (ShouldRestartWorkerChannel(runtime, isWebHostChannelDisposed, isJobHostChannelDisposed))

src/WebJobs.Script/Workers/Rpc/JobHostRpcWorkerChannelManager.cs

@@ -47,9 +47,13 @@ public Task<bool> ShutdownChannelIfExistsAsync(string channelId, Exception worke
                 {
                     if (channels.TryRemove(channelId, out IRpcWorkerChannel rpcChannel))
                     {
-                        _logger.LogDebug("Disposing language worker channel with id:{workerId}", rpcChannel.Id);
+                        string id = rpcChannel.Id;
+                        _logger.LogDebug("Disposing language worker channel with id:{workerId}", id);
                         rpcChannel.TryFailExecutions(workerException);
+
                         (rpcChannel as IDisposable)?.Dispose();
+                        _logger.LogDebug("Disposed language worker channel with id:{workerId}", id);
+
                         return Task.FromResult(true);
                     }
                 }