Remove the double dispatch on Windows for IO (#43449)

The windows IO thread pool was replaced with one that dispatches to the thread pool. As a result, we remove the extra thread pool dispatch that we originally had for windows because continuations for IO on windows ran on
the same thread that polls for IO.

Author: davidfowl

src/Servers/Kestrel/Transport.Sockets/src/Internal/SocketConnection.cs

@@ -33,7 +33,7 @@ internal sealed partial class SocketConnection : TransportConnection
 
     internal SocketConnection(Socket socket,
                               MemoryPool<byte> memoryPool,
-                              PipeScheduler transportScheduler,
+                              PipeScheduler socketScheduler,
                               ILogger logger,
                               SocketSenderPool socketSenderPool,
                               PipeOptions inputOptions,
@@ -55,12 +55,7 @@ internal SocketConnection(Socket socket,
 
         ConnectionClosed = _connectionClosedTokenSource.Token;
 
-        // On *nix platforms, Sockets already dispatches to the ThreadPool.
-        // Yes, the IOQueues are still used for the PipeSchedulers. This is intentional.
-        // https://github.com/aspnet/KestrelHttpServer/issues/2573
-        var awaiterScheduler = OperatingSystem.IsWindows() ? transportScheduler : PipeScheduler.Inline;
-
-        _receiver = new SocketReceiver(awaiterScheduler);
+        _receiver = new SocketReceiver(socketScheduler);
 
         var pair = DuplexPipe.CreateConnectionPair(inputOptions, outputOptions);
 

src/Servers/Kestrel/Transport.Sockets/src/SocketConnectionContextFactory.cs

@@ -56,15 +56,13 @@ public SocketConnectionContextFactory(SocketConnectionFactoryOptions options, IL
             {
                 var memoryPool = _options.MemoryPoolFactory();
                 var transportScheduler = options.UnsafePreferInlineScheduling ? PipeScheduler.Inline : new IOQueue();
-                // https://github.com/aspnet/KestrelHttpServer/issues/2573
-                var awaiterScheduler = OperatingSystem.IsWindows() ? transportScheduler : PipeScheduler.Inline;
 
                 _settings[i] = new QueueSettings()
                 {
                     Scheduler = transportScheduler,
                     InputOptions = new PipeOptions(memoryPool, applicationScheduler, transportScheduler, maxReadBufferSize, maxReadBufferSize / 2, useSynchronizationContext: false),
                     OutputOptions = new PipeOptions(memoryPool, transportScheduler, applicationScheduler, maxWriteBufferSize, maxWriteBufferSize / 2, useSynchronizationContext: false),
-                    SocketSenderPool = new SocketSenderPool(awaiterScheduler),
+                    SocketSenderPool = new SocketSenderPool(transportScheduler),
                     MemoryPool = memoryPool,
                 };
             }
@@ -73,16 +71,14 @@ public SocketConnectionContextFactory(SocketConnectionFactoryOptions options, IL
         {
             var memoryPool = _options.MemoryPoolFactory();
             var transportScheduler = options.UnsafePreferInlineScheduling ? PipeScheduler.Inline : PipeScheduler.ThreadPool;
-            // https://github.com/aspnet/KestrelHttpServer/issues/2573
-            var awaiterScheduler = OperatingSystem.IsWindows() ? transportScheduler : PipeScheduler.Inline;
             _settings = new QueueSettings[]
             {
                 new QueueSettings()
                 {
                     Scheduler = transportScheduler,
                     InputOptions = new PipeOptions(memoryPool, applicationScheduler, transportScheduler, maxReadBufferSize, maxReadBufferSize / 2, useSynchronizationContext: false),
                     OutputOptions = new PipeOptions(memoryPool, transportScheduler, applicationScheduler, maxWriteBufferSize, maxWriteBufferSize / 2, useSynchronizationContext: false),
-                    SocketSenderPool = new SocketSenderPool(awaiterScheduler),
+                    SocketSenderPool = new SocketSenderPool(PipeScheduler.Inline),
                     MemoryPool = memoryPool,
                 }
             };
@@ -101,7 +97,7 @@ public ConnectionContext Create(Socket socket)
 
         var connection = new SocketConnection(socket,
             setting.MemoryPool,
-            setting.Scheduler,
+            PipeScheduler.Inline,
             _logger,
             setting.SocketSenderPool,
             setting.InputOptions,