Skip to content

Commit 7fa6d19

Browse files
halter73halter73
authored
Add option to interpret request headers as Latin1 (#18255)
1 parent a633c66 commit 7fa6d19

17 files changed

+451
-24
lines changed

‎src/Servers/Kestrel/Core/ref/Microsoft.AspNetCore.Server.Kestrel.Core.Manual.cs

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@ public partial class KestrelServerOptions
4040
{
4141
internal System.Security.Cryptography.X509Certificates.X509Certificate2 DefaultCertificate { [System.Runtime.CompilerServices.CompilerGeneratedAttribute]get { throw null; } [System.Runtime.CompilerServices.CompilerGeneratedAttribute]set { } }
4242
internal bool IsDevCertLoaded { [System.Runtime.CompilerServices.CompilerGeneratedAttribute]get { throw null; } [System.Runtime.CompilerServices.CompilerGeneratedAttribute]set { } }
43+
internal bool Latin1RequestHeaders { [System.Runtime.CompilerServices.CompilerGeneratedAttribute]get { throw null; } [System.Runtime.CompilerServices.CompilerGeneratedAttribute]set { } }
4344
internal System.Collections.Generic.List<Microsoft.AspNetCore.Server.Kestrel.Core.ListenOptions> ListenOptions { [System.Runtime.CompilerServices.CompilerGeneratedAttribute]get { throw null; } }
4445
internal void ApplyDefaultCert(Microsoft.AspNetCore.Server.Kestrel.Https.HttpsConnectionAdapterOptions httpsOptions) { }
4546
internal void ApplyEndpointDefaults(Microsoft.AspNetCore.Server.Kestrel.Core.ListenOptions listenOptions) { }
@@ -433,6 +434,7 @@ public ConfigurationReader(Microsoft.Extensions.Configuration.IConfiguration con
433434
public System.Collections.Generic.IDictionary<string, Microsoft.AspNetCore.Server.Kestrel.Core.Internal.CertificateConfig> Certificates { get { throw null; } }
434435
public Microsoft.AspNetCore.Server.Kestrel.Core.Internal.EndpointDefaults EndpointDefaults { get { throw null; } }
435436
public System.Collections.Generic.IEnumerable<Microsoft.AspNetCore.Server.Kestrel.Core.Internal.EndpointConfig> Endpoints { get { throw null; } }
437+
public bool Latin1RequestHeaders { get { throw null; } }
436438
}
437439
internal partial class HttpConnectionContext
438440
{
@@ -879,7 +881,7 @@ public void ThrowRequestTargetRejected(System.Span<byte> target) { }
879881
}
880882
internal sealed partial class HttpRequestHeaders : Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http.HttpHeaders
881883
{
882-
public HttpRequestHeaders(bool reuseHeaderValues = true) { }
884+
public HttpRequestHeaders(bool reuseHeaderValues = true, bool useLatin1 = false) { }
883885
public bool HasConnection { get { throw null; } }
884886
public bool HasTransferEncoding { get { throw null; } }
885887
public Microsoft.Extensions.Primitives.StringValues HeaderAccept { get { throw null; } set { } }
@@ -1614,7 +1616,6 @@ internal static partial class HttpUtilities
16141616
public const string Http2Version = "HTTP/2";
16151617
public const string HttpsUriScheme = "https://";
16161618
public const string HttpUriScheme = "http://";
1617-
public static string GetAsciiOrUTF8StringNonNullCharacters(this System.Span<byte> span) { throw null; }
16181619
public static string GetAsciiStringEscaped(this System.Span<byte> span, int maxChars) { throw null; }
16191620
public static string GetAsciiStringNonNullCharacters(this System.Span<byte> span) { throw null; }
16201621
public static string GetHeaderName(this System.Span<byte> span) { throw null; }
@@ -1624,6 +1625,7 @@ internal static partial class HttpUtilities
16241625
public static Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http.HttpMethod GetKnownMethod(string value) { throw null; }
16251626
[System.Runtime.CompilerServices.MethodImpl(System.Runtime.CompilerServices.MethodImplOptions.AggressiveInlining)]internal unsafe static Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http.HttpVersion GetKnownVersion(byte* location, int length) { throw null; }
16261627
[System.Runtime.CompilerServices.MethodImpl(System.Runtime.CompilerServices.MethodImplOptions.AggressiveInlining)]public static bool GetKnownVersion(this System.Span<byte> span, out Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http.HttpVersion knownVersion, out byte length) { throw null; }
1628+
public static string GetRequestHeaderStringNonNullCharacters(this System.Span<byte> span, bool useLatin1) { throw null; }
16271629
public static bool IsHostHeaderValid(string hostText) { throw null; }
16281630
public static string MethodToString(Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http.HttpMethod method) { throw null; }
16291631
public static string SchemeToString(Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http.HttpScheme scheme) { throw null; }
@@ -1683,6 +1685,7 @@ public StringUtilities() { }
16831685
public static bool BytesOrdinalEqualsStringAndAscii(string previousValue, System.Span<byte> newValue) { throw null; }
16841686
public static string ConcatAsHexSuffix(string str, char separator, uint number) { throw null; }
16851687
public unsafe static bool TryGetAsciiString(byte* input, char* output, int count) { throw null; }
1688+
public unsafe static bool TryGetLatin1String(byte* input, char* output, int count) { throw null; }
16861689
}
16871690
internal partial class TimeoutControl : Microsoft.AspNetCore.Server.Kestrel.Core.Features.IConnectionTimeoutFeature, Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Infrastructure.ITimeoutControl
16881691
{

‎src/Servers/Kestrel/Core/src/Internal/ConfigurationReader.cs

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
// Copyright (c) .NET Foundation. All rights reserved.
1+
// Copyright (c) .NET Foundation. All rights reserved.
22
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
33

44
using System;
@@ -15,11 +15,13 @@ internal class ConfigurationReader
1515
private const string EndpointDefaultsKey = "EndpointDefaults";
1616
private const string EndpointsKey = "Endpoints";
1717
private const string UrlKey = "Url";
18+
private const string Latin1RequestHeadersKey = "Latin1RequestHeaders";
1819

1920
private IConfiguration _configuration;
2021
private IDictionary<string, CertificateConfig> _certificates;
2122
private IList<EndpointConfig> _endpoints;
2223
private EndpointDefaults _endpointDefaults;
24+
private bool? _latin1RequestHeaders;
2325

2426
public ConfigurationReader(IConfiguration configuration)
2527
{
@@ -65,6 +67,19 @@ public IEnumerable<EndpointConfig> Endpoints
6567
}
6668
}
6769

70+
public bool Latin1RequestHeaders
71+
{
72+
get
73+
{
74+
if (_latin1RequestHeaders is null)
75+
{
76+
_latin1RequestHeaders = _configuration.GetValue<bool>(Latin1RequestHeadersKey);
77+
}
78+
79+
return _latin1RequestHeaders.Value;
80+
}
81+
}
82+
6883
private void ReadCertificates()
6984
{
7085
_certificates = new Dictionary<string, CertificateConfig>(0);

‎src/Servers/Kestrel/Core/src/Internal/Http/HttpHeaders.Generated.cs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6105,7 +6105,7 @@ public unsafe void Append(Span<byte> name, Span<byte> value)
61056105
}
61066106

61076107
// We didn't have a previous matching header value, or have already added a header, so get the string for this value.
6108-
var valueStr = value.GetAsciiOrUTF8StringNonNullCharacters();
6108+
var valueStr = value.GetRequestHeaderStringNonNullCharacters(_useLatin1);
61096109
if ((_bits & flag) == 0)
61106110
{
61116111
// We didn't already have a header set, so add a new one.
@@ -6123,7 +6123,7 @@ public unsafe void Append(Span<byte> name, Span<byte> value)
61236123
// The header was not one of the "known" headers.
61246124
// Convert value to string first, because passing two spans causes 8 bytes stack zeroing in
61256125
// this method with rep stosd, which is slower than necessary.
6126-
var valueStr = value.GetAsciiOrUTF8StringNonNullCharacters();
6126+
var valueStr = value.GetRequestHeaderStringNonNullCharacters(_useLatin1);
61276127
AppendUnknownHeaders(name, valueStr);
61286128
}
61296129
}

‎src/Servers/Kestrel/Core/src/Internal/Http/HttpProtocol.cs

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -77,7 +77,11 @@ public HttpProtocol(HttpConnectionContext context)
7777
_context = context;
7878

7979
ServerOptions = ServiceContext.ServerOptions;
80-
HttpRequestHeaders = new HttpRequestHeaders(reuseHeaderValues: !ServerOptions.DisableStringReuse);
80+
81+
HttpRequestHeaders = new HttpRequestHeaders(
82+
reuseHeaderValues: !ServerOptions.DisableStringReuse,
83+
useLatin1: ServerOptions.Latin1RequestHeaders);
84+
8185
HttpResponseControl = this;
8286
}
8387

@@ -513,7 +517,7 @@ public void OnTrailer(Span<byte> name, Span<byte> value)
513517
}
514518

515519
string key = name.GetHeaderName();
516-
var valueStr = value.GetAsciiOrUTF8StringNonNullCharacters();
520+
var valueStr = value.GetRequestHeaderStringNonNullCharacters(ServerOptions.Latin1RequestHeaders);
517521
RequestTrailers.Append(key, valueStr);
518522
}
519523

‎src/Servers/Kestrel/Core/src/Internal/Http/HttpRequestHeaders.cs

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,11 +15,13 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http
1515
internal sealed partial class HttpRequestHeaders : HttpHeaders
1616
{
1717
private readonly bool _reuseHeaderValues;
18+
private readonly bool _useLatin1;
1819
private long _previousBits = 0;
1920

20-
public HttpRequestHeaders(bool reuseHeaderValues = true)
21+
public HttpRequestHeaders(bool reuseHeaderValues = true, bool useLatin1 = false)
2122
{
2223
_reuseHeaderValues = reuseHeaderValues;
24+
_useLatin1 = useLatin1;
2325
}
2426

2527
public void OnHeadersComplete()
@@ -80,7 +82,7 @@ private void AppendContentLength(Span<byte> value)
8082
parsed < 0 ||
8183
consumed != value.Length)
8284
{
83-
BadHttpRequestException.Throw(RequestRejectionReason.InvalidContentLength, value.GetAsciiOrUTF8StringNonNullCharacters());
85+
BadHttpRequestException.Throw(RequestRejectionReason.InvalidContentLength, value.GetRequestHeaderStringNonNullCharacters(_useLatin1));
8486
}
8587

8688
_contentLength = parsed;

‎src/Servers/Kestrel/Core/src/Internal/Infrastructure/HttpUtilities.cs

Lines changed: 30 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -120,7 +120,7 @@ public static unsafe string GetAsciiStringNonNullCharacters(this Span<byte> span
120120
fixed (char* output = asciiString)
121121
fixed (byte* buffer = span)
122122
{
123-
// This version if AsciiUtilities returns null if there are any null (0 byte) characters
123+
// StringUtilities.TryGetAsciiString returns null if there are any null (0 byte) characters
124124
// in the string
125125
if (!StringUtilities.TryGetAsciiString(buffer, output, span.Length))
126126
{
@@ -130,7 +130,7 @@ public static unsafe string GetAsciiStringNonNullCharacters(this Span<byte> span
130130
return asciiString;
131131
}
132132

133-
public static unsafe string GetAsciiOrUTF8StringNonNullCharacters(this Span<byte> span)
133+
private static unsafe string GetAsciiOrUTF8StringNonNullCharacters(this Span<byte> span)
134134
{
135135
if (span.IsEmpty)
136136
{
@@ -142,7 +142,7 @@ public static unsafe string GetAsciiOrUTF8StringNonNullCharacters(this Span<byte
142142
fixed (char* output = resultString)
143143
fixed (byte* buffer = span)
144144
{
145-
// This version if AsciiUtilities returns null if there are any null (0 byte) characters
145+
// StringUtilities.TryGetAsciiString returns null if there are any null (0 byte) characters
146146
// in the string
147147
if (!StringUtilities.TryGetAsciiString(buffer, output, span.Length))
148148
{
@@ -162,9 +162,36 @@ public static unsafe string GetAsciiOrUTF8StringNonNullCharacters(this Span<byte
162162
}
163163
}
164164
}
165+
166+
return resultString;
167+
}
168+
169+
private static unsafe string GetLatin1StringNonNullCharacters(this Span<byte> span)
170+
{
171+
if (span.IsEmpty)
172+
{
173+
return string.Empty;
174+
}
175+
176+
var resultString = new string('\0', span.Length);
177+
178+
fixed (char* output = resultString)
179+
fixed (byte* buffer = span)
180+
{
181+
// This returns false if there are any null (0 byte) characters in the string.
182+
if (!StringUtilities.TryGetLatin1String(buffer, output, span.Length))
183+
{
184+
// null characters are considered invalid
185+
throw new InvalidOperationException();
186+
}
187+
}
188+
165189
return resultString;
166190
}
167191

192+
public static string GetRequestHeaderStringNonNullCharacters(this Span<byte> span, bool useLatin1) =>
193+
useLatin1 ? GetLatin1StringNonNullCharacters(span) : GetAsciiOrUTF8StringNonNullCharacters(span);
194+
168195
public static string GetAsciiStringEscaped(this Span<byte> span, int maxChars)
169196
{
170197
var sb = new StringBuilder();

‎src/Servers/Kestrel/Core/src/Internal/Infrastructure/StringUtilities.cs

Lines changed: 143 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@
22
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
33

44
using System;
5-
using System.Buffers.Binary;
65
using System.Diagnostics;
76
using System.Numerics;
87
using System.Runtime.CompilerServices;
@@ -17,6 +16,9 @@ internal class StringUtilities
1716
[MethodImpl(MethodImplOptions.AggressiveOptimization)]
1817
public static unsafe bool TryGetAsciiString(byte* input, char* output, int count)
1918
{
19+
Debug.Assert(input != null);
20+
Debug.Assert(output != null);
21+
2022
// Calculate end position
2123
var end = input + count;
2224
// Start as valid
@@ -115,6 +117,111 @@ out Unsafe.AsRef<Vector<short>>(output),
115117
return isValid;
116118
}
117119

120+
[MethodImpl(MethodImplOptions.AggressiveOptimization)]
121+
public static unsafe bool TryGetLatin1String(byte* input, char* output, int count)
122+
{
123+
Debug.Assert(input != null);
124+
Debug.Assert(output != null);
125+
126+
// Calculate end position
127+
var end = input + count;
128+
// Start as valid
129+
var isValid = true;
130+
131+
do
132+
{
133+
// If Vector not-accelerated or remaining less than vector size
134+
if (!Vector.IsHardwareAccelerated || input > end - Vector<sbyte>.Count)
135+
{
136+
if (IntPtr.Size == 8) // Use Intrinsic switch for branch elimination
137+
{
138+
// 64-bit: Loop longs by default
139+
while (input <= end - sizeof(long))
140+
{
141+
isValid &= CheckBytesNotNull(((long*)input)[0]);
142+
143+
output[0] = (char)input[0];
144+
output[1] = (char)input[1];
145+
output[2] = (char)input[2];
146+
output[3] = (char)input[3];
147+
output[4] = (char)input[4];
148+
output[5] = (char)input[5];
149+
output[6] = (char)input[6];
150+
output[7] = (char)input[7];
151+
152+
input += sizeof(long);
153+
output += sizeof(long);
154+
}
155+
if (input <= end - sizeof(int))
156+
{
157+
isValid &= CheckBytesNotNull(((int*)input)[0]);
158+
159+
output[0] = (char)input[0];
160+
output[1] = (char)input[1];
161+
output[2] = (char)input[2];
162+
output[3] = (char)input[3];
163+
164+
input += sizeof(int);
165+
output += sizeof(int);
166+
}
167+
}
168+
else
169+
{
170+
// 32-bit: Loop ints by default
171+
while (input <= end - sizeof(int))
172+
{
173+
isValid &= CheckBytesNotNull(((int*)input)[0]);
174+
175+
output[0] = (char)input[0];
176+
output[1] = (char)input[1];
177+
output[2] = (char)input[2];
178+
output[3] = (char)input[3];
179+
180+
input += sizeof(int);
181+
output += sizeof(int);
182+
}
183+
}
184+
if (input <= end - sizeof(short))
185+
{
186+
isValid &= CheckBytesNotNull(((short*)input)[0]);
187+
188+
output[0] = (char)input[0];
189+
output[1] = (char)input[1];
190+
191+
input += sizeof(short);
192+
output += sizeof(short);
193+
}
194+
if (input < end)
195+
{
196+
isValid &= CheckBytesNotNull(((sbyte*)input)[0]);
197+
output[0] = (char)input[0];
198+
}
199+
200+
return isValid;
201+
}
202+
203+
// do/while as entry condition already checked
204+
do
205+
{
206+
// Use byte/ushort instead of signed equivalents to ensure it doesn't fill based on the high bit.
207+
var vector = Unsafe.AsRef<Vector<byte>>(input);
208+
isValid &= CheckBytesNotNull(vector);
209+
Vector.Widen(
210+
vector,
211+
out Unsafe.AsRef<Vector<ushort>>(output),
212+
out Unsafe.AsRef<Vector<ushort>>(output + Vector<ushort>.Count));
213+
214+
input += Vector<byte>.Count;
215+
output += Vector<byte>.Count;
216+
} while (input <= end - Vector<byte>.Count);
217+
218+
// Vector path done, loop back to do non-Vector
219+
// If is a exact multiple of vector size, bail now
220+
} while (input < end);
221+
222+
return isValid;
223+
}
224+
118225
[MethodImpl(MethodImplOptions.AggressiveOptimization)]
119226
public unsafe static bool BytesOrdinalEqualsStringAndAscii(string previousValue, Span<byte> newValue)
120227
{
@@ -421,7 +528,7 @@ private static bool CheckBytesInAsciiRange(Vector<sbyte> check)
421528
// Validate: bytes != 0 && bytes <= 127
422529
// Subtract 1 from all bytes to move 0 to high bits
423530
// bitwise or with self to catch all > 127 bytes
424-
// mask off high bits and check if 0
531+
// mask off non high bits and check if 0
425532

426533
[MethodImpl(MethodImplOptions.AggressiveInlining)] // Needs a push
427534
private static bool CheckBytesInAsciiRange(long check)
@@ -444,5 +551,39 @@ private static bool CheckBytesInAsciiRange(short check)
444551

445552
private static bool CheckBytesInAsciiRange(sbyte check)
446553
=> check > 0;
554+
555+
[MethodImpl(MethodImplOptions.AggressiveInlining)] // Needs a push
556+
private static bool CheckBytesNotNull(Vector<byte> check)
557+
{
558+
// Vectorized byte range check, signed byte != null
559+
return !Vector.EqualsAny(check, Vector<byte>.Zero);
560+
}
561+
562+
// Validate: bytes != 0
563+
// Subtract 1 from all bytes to move 0 to high bits
564+
// bitwise and with ~check so high bits are only set for bytes that were originally 0
565+
// mask off non high bits and check if 0
566+
567+
[MethodImpl(MethodImplOptions.AggressiveInlining)] // Needs a push
568+
private static bool CheckBytesNotNull(long check)
569+
{
570+
const long HighBits = unchecked((long)0x8080808080808080L);
571+
return ((check - 0x0101010101010101L) & ~check & HighBits) == 0;
572+
}
573+
574+
private static bool CheckBytesNotNull(int check)
575+
{
576+
const int HighBits = unchecked((int)0x80808080);
577+
return ((check - 0x01010101) & ~check & HighBits) == 0;
578+
}
579+
580+
private static bool CheckBytesNotNull(short check)
581+
{
582+
const short HighBits = unchecked((short)0x8080);
583+
return ((check - 0x0101) & ~check & HighBits) == 0;
584+
}
585+
586+
private static bool CheckBytesNotNull(sbyte check)
587+
=> check != 0;
447588
}
448589
}

0 commit comments

Comments
 (0)