Hotfix4.28.3 - remove condition exception/fixing race condition (#9742)

v-imohammad · mathewc · brettsam · web-flow · commit 359eb50f65ed · 2023-12-13T15:09:58.000-08:00
* Removing conditional exception handling from FileLogger (#9739) * fixing race condition in worker shutdown (#9738) * updating common prop --------- Co-authored-by: Mathew Charles <mathewc@microsoft.com> Co-authored-by: Brett Samblanet <brettsam@microsoft.com>
diff --git a/build/common.props b/build/common.props
@@ -5,7 +5,7 @@
     <LangVersion>latest</LangVersion>
     <MajorVersion>4</MajorVersion>
     <MinorVersion>28</MinorVersion>
-    <PatchVersion>2</PatchVersion>
+    <PatchVersion>3</PatchVersion>
     <BuildNumber Condition="'$(BuildNumber)' == '' ">0</BuildNumber>
     <PreviewVersion></PreviewVersion>
     
diff --git a/src/WebJobs.Script/Diagnostics/FileLogger.cs b/src/WebJobs.Script/Diagnostics/FileLogger.cs
@@ -93,7 +93,7 @@ public void Log<TState>(LogLevel logLevel, EventId eventId, TState state, Except
             {
                 _fileWriter.AppendLine(formattedMessage);
             }
-            catch (Exception ex) when (!ex.IsFatal())
+            catch (Exception)
             {
                 // Make sure the Logger doesn't throw if there are Exceptions (disk full, etc).
             }
diff --git a/src/WebJobs.Script/Workers/Rpc/FunctionRegistration/RpcFunctionInvocationDispatcher.cs b/src/WebJobs.Script/Workers/Rpc/FunctionRegistration/RpcFunctionInvocationDispatcher.cs
@@ -187,7 +187,7 @@ internal async void ShutdownWebhostLanguageWorkerChannels()
             await _webHostLanguageWorkerChannelManager?.ShutdownChannelsAsync();
         }
 
-        private void SetDispatcherStateToInitialized(Dictionary<string, TaskCompletionSource<IRpcWorkerChannel>> webhostLanguageWorkerChannel = null)
+        private void SetDispatcherStateToInitialized(IDictionary<string, TaskCompletionSource<IRpcWorkerChannel>> webhostLanguageWorkerChannel = null)
         {
             // RanToCompletion indicates successful process startup
             if (State != FunctionInvocationDispatcherState.Initialized
@@ -198,7 +198,7 @@ private void SetDispatcherStateToInitialized(Dictionary<string, TaskCompletionSo
             }
         }
 
-        private void StartWorkerProcesses(int startIndex, Func<IEnumerable<string>, Task> startAction, bool initializeDispatcher = false, Dictionary<string, TaskCompletionSource<IRpcWorkerChannel>> webhostLanguageWorkerChannel = null, IEnumerable<string> functionLanguages = null)
+        private void StartWorkerProcesses(int startIndex, Func<IEnumerable<string>, Task> startAction, bool initializeDispatcher = false, IDictionary<string, TaskCompletionSource<IRpcWorkerChannel>> webhostLanguageWorkerChannel = null, IEnumerable<string> functionLanguages = null)
         {
             Task.Run(async () =>
             {
@@ -309,7 +309,7 @@ public async Task InitializeAsync(IEnumerable<FunctionMetadata> functions, Cance
             if (Utility.IsSupportedRuntime(_workerRuntime, _workerConfigs) || _environment.IsMultiLanguageRuntimeEnvironment())
             {
                 State = FunctionInvocationDispatcherState.Initializing;
-                Dictionary<string, TaskCompletionSource<IRpcWorkerChannel>> webhostLanguageWorkerChannels = _webHostLanguageWorkerChannelManager.GetChannels(_workerRuntime);
+                IDictionary<string, TaskCompletionSource<IRpcWorkerChannel>> webhostLanguageWorkerChannels = _webHostLanguageWorkerChannelManager.GetChannels(_workerRuntime);
                 if (webhostLanguageWorkerChannels != null)
                 {
                     int workerProcessCount = 0;
diff --git a/src/WebJobs.Script/Workers/Rpc/IWebHostRpcWorkerChannelManager.cs b/src/WebJobs.Script/Workers/Rpc/IWebHostRpcWorkerChannelManager.cs
@@ -11,7 +11,7 @@ public interface IWebHostRpcWorkerChannelManager
     {
         Task<IRpcWorkerChannel> InitializeChannelAsync(IEnumerable<RpcWorkerConfig> workerConfigs, string language);
 
-        Dictionary<string, TaskCompletionSource<IRpcWorkerChannel>> GetChannels(string language);
+        IDictionary<string, TaskCompletionSource<IRpcWorkerChannel>> GetChannels(string language);
 
         Task SpecializeAsync();
 
diff --git a/src/WebJobs.Script/Workers/Rpc/WebHostRpcWorkerChannelManager.cs b/src/WebJobs.Script/Workers/Rpc/WebHostRpcWorkerChannelManager.cs
@@ -7,9 +7,7 @@
 using System.Linq;
 using System.Reactive.Linq;
 using System.Threading.Tasks;
-using Microsoft.Azure.AppService.Proxy.Common.Extensions;
 using Microsoft.Azure.AppService.Proxy.Common.Infra;
-using Microsoft.Azure.AppService.Proxy.Runtime;
 using Microsoft.Azure.WebJobs.Script.Config;
 using Microsoft.Azure.WebJobs.Script.Diagnostics;
 using Microsoft.Azure.WebJobs.Script.Eventing;
@@ -36,7 +34,7 @@ public class WebHostRpcWorkerChannelManager : IWebHostRpcWorkerChannelManager
         private Action _shutdownStandbyWorkerChannels;
         private IConfiguration _config;
 
-        private ConcurrentDictionary<string, Dictionary<string, TaskCompletionSource<IRpcWorkerChannel>>> _workerChannels = new ConcurrentDictionary<string, Dictionary<string, TaskCompletionSource<IRpcWorkerChannel>>>(StringComparer.OrdinalIgnoreCase);
+        private ConcurrentDictionary<string, ConcurrentDictionary<string, TaskCompletionSource<IRpcWorkerChannel>>> _workerChannels = new(StringComparer.OrdinalIgnoreCase);
 
         public WebHostRpcWorkerChannelManager(IScriptEventManager eventManager,
                                               IEnvironment environment,
@@ -101,7 +99,7 @@ await rpcWorkerChannel.StartWorkerProcessAsync().ContinueWith(processStartTask =
 
         internal Task<IRpcWorkerChannel> GetChannelAsync(string language)
         {
-            if (!string.IsNullOrEmpty(language) && _workerChannels.TryGetValue(language, out Dictionary<string, TaskCompletionSource<IRpcWorkerChannel>> workerChannels))
+            if (!string.IsNullOrEmpty(language) && _workerChannels.TryGetValue(language, out ConcurrentDictionary<string, TaskCompletionSource<IRpcWorkerChannel>> workerChannels))
             {
                 if (workerChannels.Count > 0 && workerChannels.TryGetValue(workerChannels.Keys.First(), out TaskCompletionSource<IRpcWorkerChannel> valueTask))
                 {
@@ -111,9 +109,9 @@ internal Task<IRpcWorkerChannel> GetChannelAsync(string language)
             return Task.FromResult<IRpcWorkerChannel>(null);
         }
 
-        public Dictionary<string, TaskCompletionSource<IRpcWorkerChannel>> GetChannels(string language)
+        public IDictionary<string, TaskCompletionSource<IRpcWorkerChannel>> GetChannels(string language)
         {
-            if (!string.IsNullOrEmpty(language) && _workerChannels.TryGetValue(language, out Dictionary<string, TaskCompletionSource<IRpcWorkerChannel>> workerChannels))
+            if (!string.IsNullOrEmpty(language) && _workerChannels.TryGetValue(language, out ConcurrentDictionary<string, TaskCompletionSource<IRpcWorkerChannel>> workerChannels))
             {
                 return workerChannels;
             }
@@ -237,7 +235,7 @@ public Task<bool> ShutdownChannelIfExistsAsync(string language, string workerId,
 
             if (_hostingConfigOptions.Value.RevertWorkerShutdownBehaviour)
             {
-                if (_workerChannels.TryRemove(language, out Dictionary<string, TaskCompletionSource<IRpcWorkerChannel>> rpcWorkerChannels))
+                if (_workerChannels.TryRemove(language, out ConcurrentDictionary<string, TaskCompletionSource<IRpcWorkerChannel>> rpcWorkerChannels))
                 {
                     if (rpcWorkerChannels.TryGetValue(workerId, out TaskCompletionSource<IRpcWorkerChannel> value))
                     {
@@ -264,7 +262,7 @@ public Task<bool> ShutdownChannelIfExistsAsync(string language, string workerId,
             }
             else
             {
-                if (_workerChannels.TryGetValue(language, out Dictionary<string, TaskCompletionSource<IRpcWorkerChannel>> rpcWorkerChannels)
+                if (_workerChannels.TryGetValue(language, out ConcurrentDictionary<string, TaskCompletionSource<IRpcWorkerChannel>> rpcWorkerChannels)
                     && rpcWorkerChannels.TryRemove(workerId, out TaskCompletionSource<IRpcWorkerChannel> value))
                 {
                     value?.Task.ContinueWith(channelTask =>
@@ -304,7 +302,7 @@ internal void ScheduleShutdownStandbyChannels()
                         using (_metricsLogger.LatencyEvent(string.Format(MetricEventNames.SpecializationShutdownStandbyChannels, runtime.Key)))
                         {
                             _logger.LogInformation("Disposing standby channel for runtime:{language}", runtime.Key);
-                            if (_workerChannels.TryRemove(runtime.Key, out Dictionary<string, TaskCompletionSource<IRpcWorkerChannel>> standbyChannels))
+                            if (_workerChannels.TryRemove(runtime.Key, out ConcurrentDictionary<string, TaskCompletionSource<IRpcWorkerChannel>> standbyChannels))
                             {
                                 foreach (string workerId in standbyChannels.Keys)
                                 {
@@ -338,7 +336,7 @@ public async Task ShutdownChannelsAsync()
             foreach (string runtime in _workerChannels.Keys)
             {
                 _logger.LogInformation("Shutting down language worker channels for runtime:{runtime}", runtime);
-                if (_workerChannels.TryRemove(runtime, out Dictionary<string, TaskCompletionSource<IRpcWorkerChannel>> standbyChannels))
+                if (_workerChannels.TryRemove(runtime, out ConcurrentDictionary<string, TaskCompletionSource<IRpcWorkerChannel>> standbyChannels))
                 {
                     foreach (string workerId in standbyChannels.Keys)
                     {
@@ -378,21 +376,21 @@ internal void AddOrUpdateWorkerChannels(string initializedRuntime, IRpcWorkerCha
             _workerChannels.AddOrUpdate(initializedRuntime,
                     (runtime) =>
                     {
-                        Dictionary<string, TaskCompletionSource<IRpcWorkerChannel>> newLanguageWorkerChannels = new Dictionary<string, TaskCompletionSource<IRpcWorkerChannel>>();
-                        newLanguageWorkerChannels.Add(initializedLanguageWorkerChannel.Id, new TaskCompletionSource<IRpcWorkerChannel>());
+                        ConcurrentDictionary<string, TaskCompletionSource<IRpcWorkerChannel>> newLanguageWorkerChannels = new(StringComparer.OrdinalIgnoreCase);
+                        newLanguageWorkerChannels.TryAdd(initializedLanguageWorkerChannel.Id, new TaskCompletionSource<IRpcWorkerChannel>());
                         return newLanguageWorkerChannels;
                     },
                     (runtime, existingLanguageWorkerChannels) =>
                     {
-                        existingLanguageWorkerChannels.Add(initializedLanguageWorkerChannel.Id, new TaskCompletionSource<IRpcWorkerChannel>());
+                        existingLanguageWorkerChannels.TryAdd(initializedLanguageWorkerChannel.Id, new TaskCompletionSource<IRpcWorkerChannel>());
                         return existingLanguageWorkerChannels;
                     });
         }
 
         internal void SetInitializedWorkerChannel(string initializedRuntime, IRpcWorkerChannel initializedLanguageWorkerChannel)
         {
             _logger.LogDebug("Adding webhost language worker channel for runtime: {language}. workerId:{id}", initializedRuntime, initializedLanguageWorkerChannel.Id);
-            if (_workerChannels.TryGetValue(initializedRuntime, out Dictionary<string, TaskCompletionSource<IRpcWorkerChannel>> channel))
+            if (_workerChannels.TryGetValue(initializedRuntime, out ConcurrentDictionary<string, TaskCompletionSource<IRpcWorkerChannel>> channel))
             {
                 if (channel.TryGetValue(initializedLanguageWorkerChannel.Id, out TaskCompletionSource<IRpcWorkerChannel> value))
                 {
@@ -404,7 +402,7 @@ internal void SetInitializedWorkerChannel(string initializedRuntime, IRpcWorkerC
         internal void SetExceptionOnInitializedWorkerChannel(string initializedRuntime, IRpcWorkerChannel initializedLanguageWorkerChannel, Exception exception)
         {
             _logger.LogDebug("Failed to initialize webhost language worker channel for runtime: {language}. workerId:{id}", initializedRuntime, initializedLanguageWorkerChannel.Id);
-            if (_workerChannels.TryGetValue(initializedRuntime, out Dictionary<string, TaskCompletionSource<IRpcWorkerChannel>> channel))
+            if (_workerChannels.TryGetValue(initializedRuntime, out ConcurrentDictionary<string, TaskCompletionSource<IRpcWorkerChannel>> channel))
             {
                 if (channel.TryGetValue(initializedLanguageWorkerChannel.Id, out TaskCompletionSource<IRpcWorkerChannel> value))
                 {
diff --git a/test/WebJobs.Script.Tests/Workers/Rpc/TestRpcWorkerChannelManager.cs b/test/WebJobs.Script.Tests/Workers/Rpc/TestRpcWorkerChannelManager.cs
@@ -35,7 +35,7 @@ public IRpcWorkerChannel GetChannel(string language)
             throw new System.NotImplementedException();
         }
 
-        public Dictionary<string, TaskCompletionSource<IRpcWorkerChannel>> GetChannels(string language)
+        public IDictionary<string, TaskCompletionSource<IRpcWorkerChannel>> GetChannels(string language)
         {
             if (_workerChannels.TryGetValue(language, out Dictionary<string, TaskCompletionSource<IRpcWorkerChannel>> workerChannels))
             {
diff --git a/test/WebJobs.Script.Tests/Workers/Rpc/WebHostRpcWorkerChannelManagerTests.cs b/test/WebJobs.Script.Tests/Workers/Rpc/WebHostRpcWorkerChannelManagerTests.cs
@@ -4,6 +4,7 @@
 using System;
 using System.Collections.Generic;
 using System.Linq;
+using System.Threading;
 using System.Threading.Tasks;
 using Microsoft.Azure.WebJobs.Script.Config;
 using Microsoft.Azure.WebJobs.Script.Description;
@@ -407,6 +408,38 @@ public async Task ShutdownChannelsIfExist_Succeeds()
             Assert.Null(initializedChannel);
         }
 
+        [Fact]
+        public void ShutdownChannelsIfExist_Race_Succeeds()
+        {
+            var channel = CreateTestChannel(RpcWorkerConstants.JavaLanguageWorkerName);
+            string id = channel.Id;
+
+            List<Task<bool>> tasks = new();
+            List<Thread> threads = new();
+            for (int i = 0; i < 2; i++)
+            {
+                Thread t = new(static (state) =>
+                {
+                    var (channelManager, tasks, id) = ((WebHostRpcWorkerChannelManager, List<Task<bool>>, string))state;
+                    tasks.Add(channelManager.ShutdownChannelIfExistsAsync(RpcWorkerConstants.JavaLanguageWorkerName, id));
+                });
+                threads.Add(t);
+            }
+
+            foreach (Thread t in threads)
+            {
+                t.Start((_rpcWorkerChannelManager, tasks, id));
+            }
+
+            foreach (Thread t in threads)
+            {
+                t.Join();
+            }
+
+            // only one should successfully shut down
+            Assert.Single(tasks, t => t.Result == true);
+        }
+
         [Fact]
         public async Task ShutdownChannelsIfExistsAsync_StopsWorkerInvocations()
         {

Original file line number	Diff line number	Diff line change
`@@ -93,7 +93,7 @@ public void Log<TState>(LogLevel logLevel, EventId eventId, TState state, Except`
`93`	`93`	`{`
`94`	`94`	`_fileWriter.AppendLine(formattedMessage);`
`95`	`95`	`}`
`96`		`- catch (Exception ex) when (!ex.IsFatal())`
	`96`	`+ catch (Exception)`
`97`	`97`	`{`
`98`	`98`	`// Make sure the Logger doesn't throw if there are Exceptions (disk full, etc).`
`99`	`99`	`}`
Original file line number	Diff line number	Diff line change
`@@ -187,7 +187,7 @@ internal async void ShutdownWebhostLanguageWorkerChannels()`
`187`	`187`	`await _webHostLanguageWorkerChannelManager?.ShutdownChannelsAsync();`
`188`	`188`	`}`
`189`	`189`
`190`		`- private void SetDispatcherStateToInitialized(Dictionary<string, TaskCompletionSource<IRpcWorkerChannel>> webhostLanguageWorkerChannel = null)`
	`190`	`+ private void SetDispatcherStateToInitialized(IDictionary<string, TaskCompletionSource<IRpcWorkerChannel>> webhostLanguageWorkerChannel = null)`
`191`	`191`	`{`
`192`	`192`	`// RanToCompletion indicates successful process startup`
`193`	`193`	`if (State != FunctionInvocationDispatcherState.Initialized`
`@@ -198,7 +198,7 @@ private void SetDispatcherStateToInitialized(Dictionary<string, TaskCompletionSo`
`198`	`198`	`}`
`199`	`199`	`}`
`200`	`200`
`201`		`- private void StartWorkerProcesses(int startIndex, Func<IEnumerable<string>, Task> startAction, bool initializeDispatcher = false, Dictionary<string, TaskCompletionSource<IRpcWorkerChannel>> webhostLanguageWorkerChannel = null, IEnumerable<string> functionLanguages = null)`
	`201`	`+ private void StartWorkerProcesses(int startIndex, Func<IEnumerable<string>, Task> startAction, bool initializeDispatcher = false, IDictionary<string, TaskCompletionSource<IRpcWorkerChannel>> webhostLanguageWorkerChannel = null, IEnumerable<string> functionLanguages = null)`
`202`	`202`	`{`
`203`	`203`	`Task.Run(async () =>`
`204`	`204`	`{`
`@@ -309,7 +309,7 @@ public async Task InitializeAsync(IEnumerable<FunctionMetadata> functions, Cance`
`309`	`309`	`if (Utility.IsSupportedRuntime(_workerRuntime, _workerConfigs) \|\| _environment.IsMultiLanguageRuntimeEnvironment())`
`310`	`310`	`{`
`311`	`311`	`State = FunctionInvocationDispatcherState.Initializing;`
`312`		`- Dictionary<string, TaskCompletionSource<IRpcWorkerChannel>> webhostLanguageWorkerChannels = _webHostLanguageWorkerChannelManager.GetChannels(_workerRuntime);`
	`312`	`+ IDictionary<string, TaskCompletionSource<IRpcWorkerChannel>> webhostLanguageWorkerChannels = _webHostLanguageWorkerChannelManager.GetChannels(_workerRuntime);`
`313`	`313`	`if (webhostLanguageWorkerChannels != null)`
`314`	`314`	`{`
`315`	`315`	`int workerProcessCount = 0;`
Original file line number	Diff line number	Diff line change
`@@ -11,7 +11,7 @@ public interface IWebHostRpcWorkerChannelManager`
`11`	`11`	`{`
`12`	`12`	`Task<IRpcWorkerChannel> InitializeChannelAsync(IEnumerable<RpcWorkerConfig> workerConfigs, string language);`
`13`	`13`
`14`		`- Dictionary<string, TaskCompletionSource<IRpcWorkerChannel>> GetChannels(string language);`
	`14`	`+ IDictionary<string, TaskCompletionSource<IRpcWorkerChannel>> GetChannels(string language);`
`15`	`15`
`16`	`16`	`Task SpecializeAsync();`
`17`	`17`
Original file line number	Diff line number	Diff line change
`@@ -7,9 +7,7 @@`
`7`	`7`	`using System.Linq;`
`8`	`8`	`using System.Reactive.Linq;`
`9`	`9`	`using System.Threading.Tasks;`
`10`		`-using Microsoft.Azure.AppService.Proxy.Common.Extensions;`
`11`	`10`	`using Microsoft.Azure.AppService.Proxy.Common.Infra;`
`12`		`-using Microsoft.Azure.AppService.Proxy.Runtime;`
`13`	`11`	`using Microsoft.Azure.WebJobs.Script.Config;`
`14`	`12`	`using Microsoft.Azure.WebJobs.Script.Diagnostics;`
`15`	`13`	`using Microsoft.Azure.WebJobs.Script.Eventing;`
`@@ -36,7 +34,7 @@ public class WebHostRpcWorkerChannelManager : IWebHostRpcWorkerChannelManager`
`36`	`34`	`private Action _shutdownStandbyWorkerChannels;`
`37`	`35`	`private IConfiguration _config;`
`38`	`36`
`39`		`- private ConcurrentDictionary<string, Dictionary<string, TaskCompletionSource<IRpcWorkerChannel>>> _workerChannels = new ConcurrentDictionary<string, Dictionary<string, TaskCompletionSource<IRpcWorkerChannel>>>(StringComparer.OrdinalIgnoreCase);`
	`37`	`+ private ConcurrentDictionary<string, ConcurrentDictionary<string, TaskCompletionSource<IRpcWorkerChannel>>> _workerChannels = new(StringComparer.OrdinalIgnoreCase);`
`40`	`38`
`41`	`39`	`public WebHostRpcWorkerChannelManager(IScriptEventManager eventManager,`
`42`	`40`	`IEnvironment environment,`
`@@ -101,7 +99,7 @@ await rpcWorkerChannel.StartWorkerProcessAsync().ContinueWith(processStartTask =`
`101`	`99`
`102`	`100`	`internal Task<IRpcWorkerChannel> GetChannelAsync(string language)`
`103`	`101`	`{`
`104`		`- if (!string.IsNullOrEmpty(language) && _workerChannels.TryGetValue(language, out Dictionary<string, TaskCompletionSource<IRpcWorkerChannel>> workerChannels))`
	`102`	`+ if (!string.IsNullOrEmpty(language) && _workerChannels.TryGetValue(language, out ConcurrentDictionary<string, TaskCompletionSource<IRpcWorkerChannel>> workerChannels))`
`105`	`103`	`{`
`106`	`104`	`if (workerChannels.Count > 0 && workerChannels.TryGetValue(workerChannels.Keys.First(), out TaskCompletionSource<IRpcWorkerChannel> valueTask))`
`107`	`105`	`{`
`@@ -111,9 +109,9 @@ internal Task<IRpcWorkerChannel> GetChannelAsync(string language)`
`111`	`109`	`return Task.FromResult<IRpcWorkerChannel>(null);`
`112`	`110`	`}`
`113`	`111`
`114`		`- public Dictionary<string, TaskCompletionSource<IRpcWorkerChannel>> GetChannels(string language)`
	`112`	`+ public IDictionary<string, TaskCompletionSource<IRpcWorkerChannel>> GetChannels(string language)`
`115`	`113`	`{`
`116`		`- if (!string.IsNullOrEmpty(language) && _workerChannels.TryGetValue(language, out Dictionary<string, TaskCompletionSource<IRpcWorkerChannel>> workerChannels))`
	`114`	`+ if (!string.IsNullOrEmpty(language) && _workerChannels.TryGetValue(language, out ConcurrentDictionary<string, TaskCompletionSource<IRpcWorkerChannel>> workerChannels))`
`117`	`115`	`{`
`118`	`116`	`return workerChannels;`
`119`	`117`	`}`
`@@ -237,7 +235,7 @@ public Task<bool> ShutdownChannelIfExistsAsync(string language, string workerId,`
`237`	`235`
`238`	`236`	`if (_hostingConfigOptions.Value.RevertWorkerShutdownBehaviour)`
`239`	`237`	`{`
`240`		`- if (_workerChannels.TryRemove(language, out Dictionary<string, TaskCompletionSource<IRpcWorkerChannel>> rpcWorkerChannels))`
	`238`	`+ if (_workerChannels.TryRemove(language, out ConcurrentDictionary<string, TaskCompletionSource<IRpcWorkerChannel>> rpcWorkerChannels))`
`241`	`239`	`{`
`242`	`240`	`if (rpcWorkerChannels.TryGetValue(workerId, out TaskCompletionSource<IRpcWorkerChannel> value))`
`243`	`241`	`{`
`@@ -264,7 +262,7 @@ public Task<bool> ShutdownChannelIfExistsAsync(string language, string workerId,`
`264`	`262`	`}`
`265`	`263`	`else`
`266`	`264`	`{`
`267`		`- if (_workerChannels.TryGetValue(language, out Dictionary<string, TaskCompletionSource<IRpcWorkerChannel>> rpcWorkerChannels)`
	`265`	`+ if (_workerChannels.TryGetValue(language, out ConcurrentDictionary<string, TaskCompletionSource<IRpcWorkerChannel>> rpcWorkerChannels)`
`268`	`266`	`&& rpcWorkerChannels.TryRemove(workerId, out TaskCompletionSource<IRpcWorkerChannel> value))`
`269`	`267`	`{`
`270`	`268`	`value?.Task.ContinueWith(channelTask =>`
`@@ -304,7 +302,7 @@ internal void ScheduleShutdownStandbyChannels()`
`304`	`302`	`using (_metricsLogger.LatencyEvent(string.Format(MetricEventNames.SpecializationShutdownStandbyChannels, runtime.Key)))`
`305`	`303`	`{`
`306`	`304`	`_logger.LogInformation("Disposing standby channel for runtime:{language}", runtime.Key);`
`307`		`- if (_workerChannels.TryRemove(runtime.Key, out Dictionary<string, TaskCompletionSource<IRpcWorkerChannel>> standbyChannels))`
	`305`	`+ if (_workerChannels.TryRemove(runtime.Key, out ConcurrentDictionary<string, TaskCompletionSource<IRpcWorkerChannel>> standbyChannels))`
`308`	`306`	`{`
`309`	`307`	`foreach (string workerId in standbyChannels.Keys)`
`310`	`308`	`{`
`@@ -338,7 +336,7 @@ public async Task ShutdownChannelsAsync()`
`338`	`336`	`foreach (string runtime in _workerChannels.Keys)`
`339`	`337`	`{`
`340`	`338`	`_logger.LogInformation("Shutting down language worker channels for runtime:{runtime}", runtime);`
`341`		`- if (_workerChannels.TryRemove(runtime, out Dictionary<string, TaskCompletionSource<IRpcWorkerChannel>> standbyChannels))`
	`339`	`+ if (_workerChannels.TryRemove(runtime, out ConcurrentDictionary<string, TaskCompletionSource<IRpcWorkerChannel>> standbyChannels))`
`342`	`340`	`{`
`343`	`341`	`foreach (string workerId in standbyChannels.Keys)`
`344`	`342`	`{`
`@@ -378,21 +376,21 @@ internal void AddOrUpdateWorkerChannels(string initializedRuntime, IRpcWorkerCha`
`378`	`376`	`_workerChannels.AddOrUpdate(initializedRuntime,`
`379`	`377`	`(runtime) =>`
`380`	`378`	`{`
`381`		`- Dictionary<string, TaskCompletionSource<IRpcWorkerChannel>> newLanguageWorkerChannels = new Dictionary<string, TaskCompletionSource<IRpcWorkerChannel>>();`
`382`		`- newLanguageWorkerChannels.Add(initializedLanguageWorkerChannel.Id, new TaskCompletionSource<IRpcWorkerChannel>());`
	`379`	`+ ConcurrentDictionary<string, TaskCompletionSource<IRpcWorkerChannel>> newLanguageWorkerChannels = new(StringComparer.OrdinalIgnoreCase);`
	`380`	`+ newLanguageWorkerChannels.TryAdd(initializedLanguageWorkerChannel.Id, new TaskCompletionSource<IRpcWorkerChannel>());`
`383`	`381`	`return newLanguageWorkerChannels;`
`384`	`382`	`},`
`385`	`383`	`(runtime, existingLanguageWorkerChannels) =>`
`386`	`384`	`{`
`387`		`- existingLanguageWorkerChannels.Add(initializedLanguageWorkerChannel.Id, new TaskCompletionSource<IRpcWorkerChannel>());`
	`385`	`+ existingLanguageWorkerChannels.TryAdd(initializedLanguageWorkerChannel.Id, new TaskCompletionSource<IRpcWorkerChannel>());`
`388`	`386`	`return existingLanguageWorkerChannels;`
`389`	`387`	`});`
`390`	`388`	`}`
`391`	`389`
`392`	`390`	`internal void SetInitializedWorkerChannel(string initializedRuntime, IRpcWorkerChannel initializedLanguageWorkerChannel)`
`393`	`391`	`{`
`394`	`392`	`_logger.LogDebug("Adding webhost language worker channel for runtime: {language}. workerId:{id}", initializedRuntime, initializedLanguageWorkerChannel.Id);`
`395`		`- if (_workerChannels.TryGetValue(initializedRuntime, out Dictionary<string, TaskCompletionSource<IRpcWorkerChannel>> channel))`
	`393`	`+ if (_workerChannels.TryGetValue(initializedRuntime, out ConcurrentDictionary<string, TaskCompletionSource<IRpcWorkerChannel>> channel))`
`396`	`394`	`{`
`397`	`395`	`if (channel.TryGetValue(initializedLanguageWorkerChannel.Id, out TaskCompletionSource<IRpcWorkerChannel> value))`
`398`	`396`	`{`
`@@ -404,7 +402,7 @@ internal void SetInitializedWorkerChannel(string initializedRuntime, IRpcWorkerC`
`404`	`402`	`internal void SetExceptionOnInitializedWorkerChannel(string initializedRuntime, IRpcWorkerChannel initializedLanguageWorkerChannel, Exception exception)`
`405`	`403`	`{`
`406`	`404`	`_logger.LogDebug("Failed to initialize webhost language worker channel for runtime: {language}. workerId:{id}", initializedRuntime, initializedLanguageWorkerChannel.Id);`
`407`		`- if (_workerChannels.TryGetValue(initializedRuntime, out Dictionary<string, TaskCompletionSource<IRpcWorkerChannel>> channel))`
	`405`	`+ if (_workerChannels.TryGetValue(initializedRuntime, out ConcurrentDictionary<string, TaskCompletionSource<IRpcWorkerChannel>> channel))`
`408`	`406`	`{`
`409`	`407`	`if (channel.TryGetValue(initializedLanguageWorkerChannel.Id, out TaskCompletionSource<IRpcWorkerChannel> value))`
`410`	`408`	`{`
Original file line number	Diff line number	Diff line change
`@@ -35,7 +35,7 @@ public IRpcWorkerChannel GetChannel(string language)`
`35`	`35`	`throw new System.NotImplementedException();`
`36`	`36`	`}`
`37`	`37`
`38`		`- public Dictionary<string, TaskCompletionSource<IRpcWorkerChannel>> GetChannels(string language)`
	`38`	`+ public IDictionary<string, TaskCompletionSource<IRpcWorkerChannel>> GetChannels(string language)`
`39`	`39`	`{`
`40`	`40`	`if (_workerChannels.TryGetValue(language, out Dictionary<string, TaskCompletionSource<IRpcWorkerChannel>> workerChannels))`
`41`	`41`	`{`