[3.1] Add latin1 support to IIS (#22798)

jkotalik · Tratcher · web-flow · commit dcd1250f43a6 · 2020-06-12T11:10:00.000-07:00
Co-authored-by: Chris Ross &lt;Tratcher@Outlook.com&gt;
diff --git a/src/Servers/HttpSys/src/Microsoft.AspNetCore.Server.HttpSys.csproj b/src/Servers/HttpSys/src/Microsoft.AspNetCore.Server.HttpSys.csproj
@@ -13,6 +13,7 @@
 
   <ItemGroup>
     <Compile Include="$(SharedSourceRoot)HttpSys\**\*.cs" />
+    <Compile Include="$(SharedSourceRoot)ServerInfrastructure\*.cs" LinkBase="ServerInfrastructure" />
   </ItemGroup>
 
   <ItemGroup>
diff --git a/src/Servers/IIS/IIS/src/Core/IISHttpContext.cs b/src/Servers/IIS/IIS/src/Core/IISHttpContext.cs
@@ -74,8 +74,9 @@ internal unsafe IISHttpContext(
             IntPtr pInProcessHandler,
             IISServerOptions options,
             IISHttpServer server,
-            ILogger logger)
-            : base((HttpApiTypes.HTTP_REQUEST*)NativeMethods.HttpGetRawRequest(pInProcessHandler))
+            ILogger logger,
+            bool useLatin1)
+            : base((HttpApiTypes.HTTP_REQUEST*)NativeMethods.HttpGetRawRequest(pInProcessHandler), useLatin1: useLatin1)
         {
             _memoryPool = memoryPool;
             _pInProcessHandler = pInProcessHandler;
diff --git a/src/Servers/IIS/IIS/src/Core/IISHttpContextOfT.cs b/src/Servers/IIS/IIS/src/Core/IISHttpContextOfT.cs
@@ -3,6 +3,7 @@
 
 using System;
 using System.Buffers;
+using System.Text;
 using System.Threading;
 using System.Threading.Tasks;
 using Microsoft.AspNetCore.Builder;
@@ -15,8 +16,8 @@ internal class IISHttpContextOfT<TContext> : IISHttpContext
     {
         private readonly IHttpApplication<TContext> _application;
 
-        public IISHttpContextOfT(MemoryPool<byte> memoryPool, IHttpApplication<TContext> application, IntPtr pInProcessHandler, IISServerOptions options, IISHttpServer server, ILogger logger)
-            : base(memoryPool, pInProcessHandler, options, server, logger)
+        public IISHttpContextOfT(MemoryPool<byte> memoryPool, IHttpApplication<TContext> application, IntPtr pInProcessHandler, IISServerOptions options, IISHttpServer server, ILogger logger, bool useLatin1)
+            : base(memoryPool, pInProcessHandler, options, server, logger, useLatin1)
         {
             _application = application;
         }
diff --git a/src/Servers/IIS/IIS/src/Core/IISHttpServer.cs b/src/Servers/IIS/IIS/src/Core/IISHttpServer.cs
@@ -214,11 +214,14 @@ private static void OnRequestsDrained(IntPtr serverContext)
 
         private class IISContextFactory<T> : IISContextFactory
         {
+            private const string Latin1Suppport = "Microsoft.AspNetCore.Server.IIS.Latin1RequestHeaders";
+
             private readonly IHttpApplication<T> _application;
             private readonly MemoryPool<byte> _memoryPool;
             private readonly IISServerOptions _options;
             private readonly IISHttpServer _server;
             private readonly ILogger _logger;
+            private readonly bool _useLatin1;
 
             public IISContextFactory(MemoryPool<byte> memoryPool, IHttpApplication<T> application, IISServerOptions options, IISHttpServer server, ILogger logger)
             {
@@ -227,11 +230,12 @@ public IISContextFactory(MemoryPool<byte> memoryPool, IHttpApplication<T> applic
                 _options = options;
                 _server = server;
                 _logger = logger;
+                AppContext.TryGetSwitch(Latin1Suppport, out _useLatin1);
             }
 
             public IISHttpContext CreateHttpContext(IntPtr pInProcessHandler)
             {
-                return new IISHttpContextOfT<T>(_memoryPool, _application, pInProcessHandler, _options, _server, _logger);
+                return new IISHttpContextOfT<T>(_memoryPool, _application, pInProcessHandler, _options, _server, _logger, _useLatin1);
             }
         }
     }
diff --git a/src/Servers/IIS/IIS/src/Microsoft.AspNetCore.Server.IIS.csproj b/src/Servers/IIS/IIS/src/Microsoft.AspNetCore.Server.IIS.csproj
@@ -19,6 +19,7 @@
     <Compile Include="$(SharedSourceRoot)StackTrace\**\*.cs" LinkBase="Shared\" />
     <Compile Include="$(SharedSourceRoot)RazorViews\*.cs" LinkBase="Shared\" />
     <Compile Include="$(SharedSourceRoot)ErrorPage\*.cs" LinkBase="Shared\" />
+    <Compile Include="$(SharedSourceRoot)ServerInfrastructure\*.cs" LinkBase="Shared\" />
   </ItemGroup>
 
   <Target Name="ValidateNativeComponentsBuilt" AfterTargets="Build" >
diff --git a/src/Servers/IIS/IIS/test/Common.FunctionalTests/Inprocess/Latin1Tests.cs b/src/Servers/IIS/IIS/test/Common.FunctionalTests/Inprocess/Latin1Tests.cs
@@ -0,0 +1,82 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using System.Net;
+using System.Net.Http;
+using System.Threading.Tasks;
+using Microsoft.AspNetCore.Server.IIS.FunctionalTests.Utilities;
+using Microsoft.AspNetCore.Server.IntegrationTesting;
+using Microsoft.AspNetCore.Server.IntegrationTesting.IIS;
+using Microsoft.AspNetCore.Testing;
+using Xunit;
+
+namespace Microsoft.AspNetCore.Server.IIS.FunctionalTests.InProcess
+{
+    [Collection(PublishedSitesCollection.Name)]
+    public class Latin1Tests : IISFunctionalTestBase
+    {
+        public Latin1Tests(PublishedSitesFixture fixture) : base(fixture)
+        {
+        }
+
+        [ConditionalFact]
+        [RequiresNewHandler]
+        public async Task Latin1Works()
+        {
+            var deploymentParameters = Fixture.GetBaseDeploymentParameters();
+            deploymentParameters.TransformArguments((a, _) => $"{a} AddLatin1");
+
+            var deploymentResult = await DeployAsync(deploymentParameters);
+
+            var client = new HttpClient(new LoggingHandler(new WinHttpHandler() { SendTimeout = TimeSpan.FromMinutes(3) }, deploymentResult.Logger));
+
+            var requestMessage = new HttpRequestMessage(HttpMethod.Get, $"{deploymentResult.ApplicationBaseUri}Latin1");
+            requestMessage.Headers.Add("foo", "£");
+
+            var result = await client.SendAsync(requestMessage);
+            Assert.Equal(HttpStatusCode.OK, result.StatusCode);
+        }
+
+        [ConditionalFact]
+        [RequiresNewHandler]
+        public async Task Latin1ReplacedWithoutAppContextSwitch()
+        {
+            var deploymentParameters = Fixture.GetBaseDeploymentParameters();
+            deploymentParameters.TransformArguments((a, _) => $"{a}");
+
+            var deploymentResult = await DeployAsync(deploymentParameters);
+
+            var client = new HttpClient(new LoggingHandler(new WinHttpHandler() { SendTimeout = TimeSpan.FromMinutes(3) }, deploymentResult.Logger));
+
+            var requestMessage = new HttpRequestMessage(HttpMethod.Get, $"{deploymentResult.ApplicationBaseUri}InvalidCharacter");
+            requestMessage.Headers.Add("foo", "£");
+
+            var result = await client.SendAsync(requestMessage);
+            Assert.Equal(HttpStatusCode.OK, result.StatusCode);
+        }
+
+        [ConditionalFact]
+        [RequiresNewHandler]
+        public async Task Latin1InvalidCharacters_HttpSysRejects()
+        {
+            var deploymentParameters = Fixture.GetBaseDeploymentParameters();
+            deploymentParameters.TransformArguments((a, _) => $"{a} AddLatin1");
+
+            var deploymentResult = await DeployAsync(deploymentParameters);
+
+            using (var connection = new TestConnection(deploymentResult.HttpClient.BaseAddress.Port))
+            {
+                await connection.Send(
+                    "GET /ReadAndFlushEcho HTTP/1.1",
+                    "Host: localhost",
+                    "Connection: close",
+                    "foo: £\0a",
+                    "",
+                    "");
+
+                await connection.ReceiveStartsWith("HTTP/1.1 400 Bad Request");
+            }
+        }
+    }
+}
diff --git a/src/Servers/IIS/IIS/test/IIS.FunctionalTests/IIS.FunctionalTests.csproj b/src/Servers/IIS/IIS/test/IIS.FunctionalTests/IIS.FunctionalTests.csproj
@@ -1,4 +1,4 @@
-﻿<Project Sdk="Microsoft.NET.Sdk">
+<Project Sdk="Microsoft.NET.Sdk">
 
 
   <PropertyGroup>
@@ -27,6 +27,7 @@
     <Reference Include="Microsoft.Extensions.Logging.Testing" />
     <Reference Include="Microsoft.Extensions.Logging" />
     <Reference Include="System.Diagnostics.EventLog" />
+    <Reference Include="System.Net.Http.WinHttpHandler" />
   </ItemGroup>
 
 </Project>
diff --git a/src/Servers/IIS/IIS/test/IIS.NewHandler.FunctionalTests/IIS.NewHandler.FunctionalTests.csproj b/src/Servers/IIS/IIS/test/IIS.NewHandler.FunctionalTests/IIS.NewHandler.FunctionalTests.csproj
@@ -1,4 +1,4 @@
-﻿<Project Sdk="Microsoft.NET.Sdk">
+<Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
     <TargetFramework>$(DefaultNetCoreTargetFramework)</TargetFramework>
@@ -34,6 +34,7 @@
     <Reference Include="Microsoft.Extensions.Logging" />
     <Reference Include="System.Diagnostics.EventLog" />
     <Reference Include="System.Net.WebSockets.WebSocketProtocol" />
+    <Reference Include="System.Net.Http.WinHttpHandler" />
   </ItemGroup>
 
 </Project>
diff --git a/src/Servers/IIS/IIS/test/IIS.NewShim.FunctionalTests/IIS.NewShim.FunctionalTests.csproj b/src/Servers/IIS/IIS/test/IIS.NewShim.FunctionalTests/IIS.NewShim.FunctionalTests.csproj
@@ -1,4 +1,4 @@
-﻿<Project Sdk="Microsoft.NET.Sdk">
+<Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
     <TargetFramework>$(DefaultNetCoreTargetFramework)</TargetFramework>
@@ -28,6 +28,7 @@
     <Reference Include="Microsoft.Extensions.Logging" />
     <Reference Include="Microsoft.Extensions.Logging.Testing" />
     <Reference Include="System.Diagnostics.EventLog" />
+    <Reference Include="System.Net.Http.WinHttpHandler" />
   </ItemGroup>
 
 </Project>
diff --git a/src/Servers/IIS/IIS/test/IISExpress.FunctionalTests/IISExpress.FunctionalTests.csproj b/src/Servers/IIS/IIS/test/IISExpress.FunctionalTests/IISExpress.FunctionalTests.csproj
@@ -1,4 +1,4 @@
-﻿<Project Sdk="Microsoft.NET.Sdk">
+<Project Sdk="Microsoft.NET.Sdk">
 
   <Import Project="../FunctionalTest.props" />
 
@@ -30,6 +30,7 @@
     <Reference Include="Microsoft.Extensions.Logging" />
     <Reference Include="Microsoft.Extensions.Logging.Testing" />
     <Reference Include="System.Diagnostics.EventLog" />
+    <Reference Include="System.Net.Http.WinHttpHandler" />
   </ItemGroup>
 
 </Project>
diff --git a/src/Servers/IIS/IIS/test/testassets/InProcessWebSite/Program.cs b/src/Servers/IIS/IIS/test/testassets/InProcessWebSite/Program.cs
@@ -145,9 +145,9 @@ public static int Main(string[] args)
                     }
 
                     return 0;
+#if !FORWARDCOMPAT
                 case "ThrowInStartupGenericHost":
                     {
-#if !FORWARDCOMPAT
                         var host = new HostBuilder().ConfigureWebHost((c) =>
                         {
                             c.ConfigureLogging((_, factory) =>
@@ -161,9 +161,26 @@ public static int Main(string[] args)
                                    
 
                         host.Build().Run();
-#endif
+                        return 0;
                     }
-                    return 0;
+                case "AddLatin1":
+                    {
+                        AppContext.SetSwitch("Microsoft.AspNetCore.Server.IIS.Latin1RequestHeaders", isEnabled: true);
+                        var host = new HostBuilder().ConfigureWebHost((c) =>
+                        {
+                            c.ConfigureLogging((_, factory) =>
+                            {
+                                factory.AddConsole();
+                                factory.AddFilter("Console", level => level >= LogLevel.Information);
+                            })
+                            .UseIIS()
+                            .UseStartup<Startup>();
+                        });
+
+                        host.Build().Run();
+                        return 0;
+                    }
+#endif
                 default:
                     return StartServer();
 
diff --git a/src/Servers/IIS/IIS/test/testassets/InProcessWebSite/Startup.cs b/src/Servers/IIS/IIS/test/testassets/InProcessWebSite/Startup.cs
@@ -1005,5 +1005,19 @@ public async Task HTTPS_PORT(HttpContext context)
 
             await context.Response.WriteAsync(httpsPort.HasValue ? httpsPort.Value.ToString() : "NOVALUE");
         }
+
+        public Task Latin1(HttpContext context)
+        {
+            var value = context.Request.Headers["foo"];
+            Assert.Equal("£", value);
+            return Task.CompletedTask;
+        }
+
+        public Task InvalidCharacter(HttpContext context)
+        {
+            var value = context.Request.Headers["foo"];
+            Assert.Equal("�", value);
+            return Task.CompletedTask;
+        }
     }
 }
diff --git a/src/Servers/Kestrel/Core/src/Internal/Infrastructure/HttpUtilities.cs b/src/Servers/Kestrel/Core/src/Internal/Infrastructure/HttpUtilities.cs
@@ -130,67 +130,8 @@ public static unsafe string GetAsciiStringNonNullCharacters(this Span<byte> span
             return asciiString;
         }
 
-        private static unsafe string GetAsciiOrUTF8StringNonNullCharacters(this Span<byte> span)
-        {
-            if (span.IsEmpty)
-            {
-                return string.Empty;
-            }
-
-            var resultString = new string('\0', span.Length);
-
-            fixed (char* output = resultString)
-            fixed (byte* buffer = span)
-            {
-                // StringUtilities.TryGetAsciiString returns null if there are any null (0 byte) characters
-                // in the string
-                if (!StringUtilities.TryGetAsciiString(buffer, output, span.Length))
-                {
-                    // null characters are considered invalid
-                    if (span.IndexOf((byte)0) != -1)
-                    {
-                        throw new InvalidOperationException();
-                    }
-
-                    try
-                    {
-                        resultString = HeaderValueEncoding.GetString(buffer, span.Length);
-                    }
-                    catch (DecoderFallbackException)
-                    {
-                        throw new InvalidOperationException();
-                    }
-                }
-            }
-
-            return resultString;
-        }
-
-        private static unsafe string GetLatin1StringNonNullCharacters(this Span<byte> span)
-        {
-            if (span.IsEmpty)
-            {
-                return string.Empty;
-            }
-
-            var resultString = new string('\0', span.Length);
-
-            fixed (char* output = resultString)
-            fixed (byte* buffer = span)
-            {
-                // This returns false if there are any null (0 byte) characters in the string.
-                if (!StringUtilities.TryGetLatin1String(buffer, output, span.Length))
-                {
-                    // null characters are considered invalid
-                    throw new InvalidOperationException();
-                }
-            }
-
-            return resultString;
-        }
-
         public static string GetRequestHeaderStringNonNullCharacters(this Span<byte> span, bool useLatin1) =>
-            useLatin1 ? GetLatin1StringNonNullCharacters(span) : GetAsciiOrUTF8StringNonNullCharacters(span);
+            useLatin1 ? span.GetLatin1StringNonNullCharacters() : span.GetAsciiOrUTF8StringNonNullCharacters(HeaderValueEncoding);
 
         public static string GetAsciiStringEscaped(this Span<byte> span, int maxChars)
         {
diff --git a/src/Servers/Kestrel/Core/src/Microsoft.AspNetCore.Server.Kestrel.Core.csproj b/src/Servers/Kestrel/Core/src/Microsoft.AspNetCore.Server.Kestrel.Core.csproj
@@ -15,6 +15,7 @@
     <Compile Include="$(SharedSourceRoot)CertificateGeneration\**\*.cs" />
     <Compile Include="$(SharedSourceRoot)ValueTaskExtensions\**\*.cs" />
     <Compile Include="$(SharedSourceRoot)UrlDecoder\**\*.cs" />
+    <Compile Include="$(SharedSourceRoot)ServerInfrastructure\**\*.cs" />
   </ItemGroup>
 
   <ItemGroup>
diff --git a/src/Shared/HttpSys/RequestProcessing/HeaderEncoding.cs b/src/Shared/HttpSys/RequestProcessing/HeaderEncoding.cs
@@ -1,29 +1,26 @@
 // Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
+using System;
 using System.Text;
+using Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Infrastructure;
 
 namespace Microsoft.AspNetCore.HttpSys.Internal
 {
     internal static class HeaderEncoding
     {
-        // It should just be ASCII or ANSI, but they break badly with un-expected values. We use UTF-8 because it's the same for
-        // ASCII, and because some old client would send UTF8 Host headers and expect UTF8 Location responses
-        // (e.g. IE and HttpWebRequest on intranets).
-        private static Encoding Encoding = new UTF8Encoding(encoderShouldEmitUTF8Identifier: false, throwOnInvalidBytes: false);
+        private static readonly Encoding Encoding = new UTF8Encoding(encoderShouldEmitUTF8Identifier: false, throwOnInvalidBytes: false);
 
-        internal static unsafe string GetString(byte* pBytes, int byteCount)
+        internal static unsafe string GetString(byte* pBytes, int byteCount, bool useLatin1)
         {
-            // net451: return new string(pBytes, 0, byteCount, Encoding);
-
-            var charCount = Encoding.GetCharCount(pBytes, byteCount);
-            var chars = new char[charCount];
-            fixed (char* pChars = chars)
+            if (useLatin1)
+            {
+                return new Span<byte>(pBytes, byteCount).GetLatin1StringNonNullCharacters();
+            }
+            else
             {
-                var count = Encoding.GetChars(pBytes, byteCount, pChars, charCount);
-                System.Diagnostics.Debug.Assert(count == charCount);
+                return new Span<byte>(pBytes, byteCount).GetAsciiOrUTF8StringNonNullCharacters(Encoding);
             }
-            return new string(chars);
         }
 
         internal static byte[] GetBytes(string myString)
diff --git a/src/Shared/HttpSys/RequestProcessing/NativeRequestContext.cs b/src/Shared/HttpSys/RequestProcessing/NativeRequestContext.cs
diff --git a/src/Shared/ServerInfrastructure/StringUtilities.cs b/src/Shared/ServerInfrastructure/StringUtilities.cs
diff --git a/src/Shared/test/Shared.Tests/Microsoft.AspNetCore.Shared.Tests.csproj b/src/Shared/test/Shared.Tests/Microsoft.AspNetCore.Shared.Tests.csproj

Original file line number	Diff line number	Diff line change
`@@ -3,6 +3,7 @@`
`3`	`3`
`4`	`4`	`using System;`
`5`	`5`	`using System.Buffers;`
	`6`	`+using System.Text;`
`6`	`7`	`using System.Threading;`
`7`	`8`	`using System.Threading.Tasks;`
`8`	`9`	`using Microsoft.AspNetCore.Builder;`
`@@ -15,8 +16,8 @@ internal class IISHttpContextOfT<TContext> : IISHttpContext`
`15`	`16`	`{`
`16`	`17`	`private readonly IHttpApplication<TContext> _application;`
`17`	`18`
`18`		`- public IISHttpContextOfT(MemoryPool<byte> memoryPool, IHttpApplication<TContext> application, IntPtr pInProcessHandler, IISServerOptions options, IISHttpServer server, ILogger logger)`
`19`		`- : base(memoryPool, pInProcessHandler, options, server, logger)`
	`19`	`+ public IISHttpContextOfT(MemoryPool<byte> memoryPool, IHttpApplication<TContext> application, IntPtr pInProcessHandler, IISServerOptions options, IISHttpServer server, ILogger logger, bool useLatin1)`
	`20`	`+ : base(memoryPool, pInProcessHandler, options, server, logger, useLatin1)`
`20`	`21`	`{`
`21`	`22`	`_application = application;`
`22`	`23`	`}`
Original file line number	Diff line number	Diff line change
`@@ -214,11 +214,14 @@ private static void OnRequestsDrained(IntPtr serverContext)`
`214`	`214`
`215`	`215`	`private class IISContextFactory<T> : IISContextFactory`
`216`	`216`	`{`
	`217`	`+ private const string Latin1Suppport = "Microsoft.AspNetCore.Server.IIS.Latin1RequestHeaders";`
	`218`	`+`
`217`	`219`	`private readonly IHttpApplication<T> _application;`
`218`	`220`	`private readonly MemoryPool<byte> _memoryPool;`
`219`	`221`	`private readonly IISServerOptions _options;`
`220`	`222`	`private readonly IISHttpServer _server;`
`221`	`223`	`private readonly ILogger _logger;`
	`224`	`+ private readonly bool _useLatin1;`
`222`	`225`
`223`	`226`	`public IISContextFactory(MemoryPool<byte> memoryPool, IHttpApplication<T> application, IISServerOptions options, IISHttpServer server, ILogger logger)`
`224`	`227`	`{`
`@@ -227,11 +230,12 @@ public IISContextFactory(MemoryPool<byte> memoryPool, IHttpApplication<T> applic`
`227`	`230`	`_options = options;`
`228`	`231`	`_server = server;`
`229`	`232`	`_logger = logger;`
	`233`	`+ AppContext.TryGetSwitch(Latin1Suppport, out _useLatin1);`
`230`	`234`	`}`
`231`	`235`
`232`	`236`	`public IISHttpContext CreateHttpContext(IntPtr pInProcessHandler)`
`233`	`237`	`{`
`234`		`- return new IISHttpContextOfT<T>(_memoryPool, _application, pInProcessHandler, _options, _server, _logger);`
	`238`	`+ return new IISHttpContextOfT<T>(_memoryPool, _application, pInProcessHandler, _options, _server, _logger, _useLatin1);`
`235`	`239`	`}`
`236`	`240`	`}`
`237`	`241`	`}`