Skip to content

Commit 86b7de7

Browse files
michael-hawkermichael-hawker
authored
Merge pull request #3378 from Sergio0694/optimization/movsxd-removal
Codegen improvements (removed unnecessary movsxd ops)
2 parents 830e9af + aef54c6 commit 86b7de7

9 files changed

+82
-51
lines changed

‎Microsoft.Toolkit.HighPerformance/Extensions/ArrayExtensions.2D.cs

Lines changed: 7 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -66,13 +66,13 @@ public static ref T DangerousGetReference<T>(this T[,] array)
6666
/// </remarks>
6767
[Pure]
6868
[MethodImpl(MethodImplOptions.AggressiveInlining)]
69-
public static ref T DangerousGetReferenceAt<T>(this T[,] array, int i, int j)
69+
public static unsafe ref T DangerousGetReferenceAt<T>(this T[,] array, int i, int j)
7070
{
7171
#if NETCORE_RUNTIME
7272
var arrayData = Unsafe.As<RawArray2DData>(array);
7373
int offset = (i * arrayData.Width) + j;
7474
ref T r0 = ref Unsafe.As<byte, T>(ref arrayData.Data);
75-
ref T ri = ref Unsafe.Add(ref r0, offset);
75+
ref T ri = ref Unsafe.Add(ref r0, (IntPtr)(void*)(uint)offset);
7676

7777
return ref ri;
7878
#else
@@ -82,10 +82,7 @@ public static ref T DangerousGetReferenceAt<T>(this T[,] array, int i, int j)
8282
return ref array[i, j];
8383
}
8484

85-
unsafe
86-
{
87-
return ref Unsafe.AsRef<T>(null);
88-
}
85+
return ref Unsafe.AsRef<T>(null);
8986
#endif
9087
}
9188

@@ -274,11 +271,11 @@ public static Span<T> AsSpan<T>(this T[,] array)
274271
/// <returns>The number of occurrences of <paramref name="value"/> in <paramref name="array"/>.</returns>
275272
[Pure]
276273
[MethodImpl(MethodImplOptions.AggressiveInlining)]
277-
public static int Count<T>(this T[,] array, T value)
274+
public static unsafe int Count<T>(this T[,] array, T value)
278275
where T : IEquatable<T>
279276
{
280277
ref T r0 = ref array.DangerousGetReference();
281-
IntPtr length = (IntPtr)array.Length;
278+
IntPtr length = (IntPtr)(void*)(uint)array.Length;
282279

283280
return SpanHelper.Count(ref r0, length, value);
284281
}
@@ -293,11 +290,11 @@ public static int Count<T>(this T[,] array, T value)
293290
/// <remarks>The Djb2 hash is fully deterministic and with no random components.</remarks>
294291
[Pure]
295292
[MethodImpl(MethodImplOptions.AggressiveInlining)]
296-
public static int GetDjb2HashCode<T>(this T[,] array)
293+
public static unsafe int GetDjb2HashCode<T>(this T[,] array)
297294
where T : notnull
298295
{
299296
ref T r0 = ref array.DangerousGetReference();
300-
IntPtr length = (IntPtr)array.Length;
297+
IntPtr length = (IntPtr)(void*)(uint)array.Length;
301298

302299
return SpanHelper.GetDjb2HashCode(ref r0, length);
303300
}

‎Microsoft.Toolkit.HighPerformance/Extensions/ArrayExtensions.cs

Lines changed: 7 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -62,12 +62,12 @@ public static ref T DangerousGetReference<T>(this T[] array)
6262
/// <remarks>This method doesn't do any bounds checks, therefore it is responsibility of the caller to ensure the <paramref name="i"/> parameter is valid.</remarks>
6363
[Pure]
6464
[MethodImpl(MethodImplOptions.AggressiveInlining)]
65-
public static ref T DangerousGetReferenceAt<T>(this T[] array, int i)
65+
public static unsafe ref T DangerousGetReferenceAt<T>(this T[] array, int i)
6666
{
6767
#if NETCORE_RUNTIME
6868
var arrayData = Unsafe.As<RawArrayData>(array);
6969
ref T r0 = ref Unsafe.As<byte, T>(ref arrayData.Data);
70-
ref T ri = ref Unsafe.Add(ref r0, i);
70+
ref T ri = ref Unsafe.Add(ref r0, (IntPtr)(void*)(uint)i);
7171

7272
return ref ri;
7373
#else
@@ -76,10 +76,7 @@ public static ref T DangerousGetReferenceAt<T>(this T[] array, int i)
7676
return ref array[i];
7777
}
7878

79-
unsafe
80-
{
81-
return ref Unsafe.AsRef<T>(null);
82-
}
79+
return ref Unsafe.AsRef<T>(null);
8380
#endif
8481
}
8582

@@ -114,11 +111,11 @@ private sealed class RawArrayData
114111
/// <returns>The number of occurrences of <paramref name="value"/> in <paramref name="array"/>.</returns>
115112
[Pure]
116113
[MethodImpl(MethodImplOptions.AggressiveInlining)]
117-
public static int Count<T>(this T[] array, T value)
114+
public static unsafe int Count<T>(this T[] array, T value)
118115
where T : IEquatable<T>
119116
{
120117
ref T r0 = ref array.DangerousGetReference();
121-
IntPtr length = (IntPtr)array.Length;
118+
IntPtr length = (IntPtr)(void*)(uint)array.Length;
122119

123120
return SpanHelper.Count(ref r0, length, value);
124121
}
@@ -185,11 +182,11 @@ public static SpanTokenizer<T> Tokenize<T>(this T[] array, T separator)
185182
/// <remarks>The Djb2 hash is fully deterministic and with no random components.</remarks>
186183
[Pure]
187184
[MethodImpl(MethodImplOptions.AggressiveInlining)]
188-
public static int GetDjb2HashCode<T>(this T[] array)
185+
public static unsafe int GetDjb2HashCode<T>(this T[] array)
189186
where T : notnull
190187
{
191188
ref T r0 = ref array.DangerousGetReference();
192-
IntPtr length = (IntPtr)array.Length;
189+
IntPtr length = (IntPtr)(void*)(uint)array.Length;
193190

194191
return SpanHelper.GetDjb2HashCode(ref r0, length);
195192
}

‎Microsoft.Toolkit.HighPerformance/Extensions/ReadOnlySpanExtensions.cs

Lines changed: 41 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -40,10 +40,40 @@ public static ref T DangerousGetReference<T>(this ReadOnlySpan<T> span)
4040
/// <remarks>This method doesn't do any bounds checks, therefore it is responsibility of the caller to ensure the <paramref name="i"/> parameter is valid.</remarks>
4141
[Pure]
4242
[MethodImpl(MethodImplOptions.AggressiveInlining)]
43-
public static ref T DangerousGetReferenceAt<T>(this ReadOnlySpan<T> span, int i)
43+
public static unsafe ref T DangerousGetReferenceAt<T>(this ReadOnlySpan<T> span, int i)
4444
{
45+
// Here we assume the input index will never be negative, so we do an unsafe cast to
46+
// force the JIT to skip the sign extension when going from int to native int.
47+
// On .NET Core 3.1, if we only use Unsafe.Add(ref r0, i), we get the following:
48+
// =============================
49+
// L0000: mov rax, [rcx]
50+
// L0003: movsxd rdx, edx
51+
// L0006: lea rax, [rax+rdx*4]
52+
// L000a: ret
53+
// =============================
54+
// Note the movsxd (move with sign extension) to expand the index passed in edx to
55+
// the whole rdx register. This is unnecessary and more expensive than just a mov,
56+
// which when done to a large register size automatically zeroes the upper bits.
57+
// With the (IntPtr)(void*)(uint) cast, we get the following codegen instead:
58+
// =============================
59+
// L0000: mov rax, [rcx]
60+
// L0003: mov edx, edx
61+
// L0005: lea rax, [rax+rdx*4]
62+
// L0009: ret
63+
// =============================
64+
// Here we can see how the index is extended to a native integer with just a mov,
65+
// which effectively only zeroes the upper bits of the same register used as source.
66+
// These three casts are a bit verbose, but they do the trick on both 32 bit and 64
67+
// bit architectures, producing optimal code in both cases (they are either completely
68+
// elided on 32 bit systems, or result in the correct register expansion when on 64 bit).
69+
// We first do an unchecked conversion to uint (which is just a reinterpret-cast). We
70+
// then cast to void*, which lets the following IntPtr cast avoid the range check on 32 bit
71+
// (since uint could be out of range there if the original index was negative). The final
72+
// result is a clean mov as shown above. This will eventually be natively supported by the
73+
// JIT compiler (see https://github.com/dotnet/runtime/issues/38794), but doing this here
74+
// still ensures the optimal codegen even on existing runtimes (eg. .NET Core 2.1 and 3.1).
4575
ref T r0 = ref MemoryMarshal.GetReference(span);
46-
ref T ri = ref Unsafe.Add(ref r0, i);
76+
ref T ri = ref Unsafe.Add(ref r0, (IntPtr)(void*)(uint)i);
4777

4878
return ref ri;
4979
}
@@ -87,7 +117,7 @@ public static ref T DangerousGetReferenceAt<T>(this ReadOnlySpan<T> span, int i)
87117
/// </returns>
88118
[Pure]
89119
[MethodImpl(MethodImplOptions.AggressiveInlining)]
90-
public static ref readonly T DangerousGetLookupReferenceAt<T>(this ReadOnlySpan<T> span, int i)
120+
public static unsafe ref readonly T DangerousGetLookupReferenceAt<T>(this ReadOnlySpan<T> span, int i)
91121
{
92122
// Check whether the input is in range by first casting both
93123
// operands to uint and then comparing them, as this allows
@@ -106,12 +136,12 @@ public static ref readonly T DangerousGetLookupReferenceAt<T>(this ReadOnlySpan<
106136
// lookup table can just be assumed to always be false.
107137
bool isInRange = (uint)i < (uint)span.Length;
108138
byte rangeFlag = Unsafe.As<bool, byte>(ref isInRange);
109-
int
110-
negativeFlag = rangeFlag - 1,
139+
uint
140+
negativeFlag = unchecked(rangeFlag - 1u),
111141
mask = ~negativeFlag,
112-
offset = i & mask;
142+
offset = (uint)i & mask;
113143
ref T r0 = ref MemoryMarshal.GetReference(span);
114-
ref T r1 = ref Unsafe.Add(ref r0, offset);
144+
ref T r1 = ref Unsafe.Add(ref r0, (IntPtr)(void*)offset);
115145

116146
return ref r1;
117147
}
@@ -165,11 +195,11 @@ public static unsafe int IndexOf<T>(this ReadOnlySpan<T> span, in T value)
165195
/// <returns>The number of occurrences of <paramref name="value"/> in <paramref name="span"/>.</returns>
166196
[Pure]
167197
[MethodImpl(MethodImplOptions.AggressiveInlining)]
168-
public static int Count<T>(this ReadOnlySpan<T> span, T value)
198+
public static unsafe int Count<T>(this ReadOnlySpan<T> span, T value)
169199
where T : IEquatable<T>
170200
{
171201
ref T r0 = ref MemoryMarshal.GetReference(span);
172-
IntPtr length = (IntPtr)span.Length;
202+
IntPtr length = (IntPtr)(void*)(uint)span.Length;
173203

174204
return SpanHelper.Count(ref r0, length, value);
175205
}
@@ -291,11 +321,11 @@ public static ReadOnlySpanTokenizer<T> Tokenize<T>(this ReadOnlySpan<T> span, T
291321
/// <remarks>The Djb2 hash is fully deterministic and with no random components.</remarks>
292322
[Pure]
293323
[MethodImpl(MethodImplOptions.AggressiveInlining)]
294-
public static int GetDjb2HashCode<T>(this ReadOnlySpan<T> span)
324+
public static unsafe int GetDjb2HashCode<T>(this ReadOnlySpan<T> span)
295325
where T : notnull
296326
{
297327
ref T r0 = ref MemoryMarshal.GetReference(span);
298-
IntPtr length = (IntPtr)span.Length;
328+
IntPtr length = (IntPtr)(void*)(uint)span.Length;
299329

300330
return SpanHelper.GetDjb2HashCode(ref r0, length);
301331
}

‎Microsoft.Toolkit.HighPerformance/Extensions/SpanExtensions.cs

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -40,10 +40,10 @@ public static ref T DangerousGetReference<T>(this Span<T> span)
4040
/// <remarks>This method doesn't do any bounds checks, therefore it is responsibility of the caller to ensure the <paramref name="i"/> parameter is valid.</remarks>
4141
[Pure]
4242
[MethodImpl(MethodImplOptions.AggressiveInlining)]
43-
public static ref T DangerousGetReferenceAt<T>(this Span<T> span, int i)
43+
public static unsafe ref T DangerousGetReferenceAt<T>(this Span<T> span, int i)
4444
{
4545
ref T r0 = ref MemoryMarshal.GetReference(span);
46-
ref T ri = ref Unsafe.Add(ref r0, i);
46+
ref T ri = ref Unsafe.Add(ref r0, (IntPtr)(void*)(uint)i);
4747

4848
return ref ri;
4949
}
@@ -140,11 +140,11 @@ public static unsafe int IndexOf<T>(this Span<T> span, ref T value)
140140
/// <returns>The number of occurrences of <paramref name="value"/> in <paramref name="span"/>.</returns>
141141
[Pure]
142142
[MethodImpl(MethodImplOptions.AggressiveInlining)]
143-
public static int Count<T>(this Span<T> span, T value)
143+
public static unsafe int Count<T>(this Span<T> span, T value)
144144
where T : IEquatable<T>
145145
{
146146
ref T r0 = ref MemoryMarshal.GetReference(span);
147-
IntPtr length = (IntPtr)span.Length;
147+
IntPtr length = (IntPtr)(void*)(uint)span.Length;
148148

149149
return SpanHelper.Count(ref r0, length, value);
150150
}
@@ -211,11 +211,11 @@ public static SpanTokenizer<T> Tokenize<T>(this Span<T> span, T separator)
211211
/// <remarks>The Djb2 hash is fully deterministic and with no random components.</remarks>
212212
[Pure]
213213
[MethodImpl(MethodImplOptions.AggressiveInlining)]
214-
public static int GetDjb2HashCode<T>(this Span<T> span)
214+
public static unsafe int GetDjb2HashCode<T>(this Span<T> span)
215215
where T : notnull
216216
{
217217
ref T r0 = ref MemoryMarshal.GetReference(span);
218-
IntPtr length = (IntPtr)span.Length;
218+
IntPtr length = (IntPtr)(void*)(uint)span.Length;
219219

220220
return SpanHelper.GetDjb2HashCode(ref r0, length);
221221
}

‎Microsoft.Toolkit.HighPerformance/Extensions/StringExtensions.cs

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@ public static ref char DangerousGetReference(this string text)
4848
/// <remarks>This method doesn't do any bounds checks, therefore it is responsibility of the caller to ensure the <paramref name="i"/> parameter is valid.</remarks>
4949
[Pure]
5050
[MethodImpl(MethodImplOptions.AggressiveInlining)]
51-
public static ref char DangerousGetReferenceAt(this string text, int i)
51+
public static unsafe ref char DangerousGetReferenceAt(this string text, int i)
5252
{
5353
#if NETCOREAPP3_1
5454
ref char r0 = ref Unsafe.AsRef(text.GetPinnableReference());
@@ -57,7 +57,7 @@ public static ref char DangerousGetReferenceAt(this string text, int i)
5757
#else
5858
ref char r0 = ref MemoryMarshal.GetReference(text.AsSpan());
5959
#endif
60-
ref char ri = ref Unsafe.Add(ref r0, i);
60+
ref char ri = ref Unsafe.Add(ref r0, (IntPtr)(void*)(uint)i);
6161

6262
return ref ri;
6363
}
@@ -91,10 +91,10 @@ private sealed class RawStringData
9191
/// <returns>The number of occurrences of <paramref name="c"/> in <paramref name="text"/>.</returns>
9292
[Pure]
9393
[MethodImpl(MethodImplOptions.AggressiveInlining)]
94-
public static int Count(this string text, char c)
94+
public static unsafe int Count(this string text, char c)
9595
{
9696
ref char r0 = ref text.DangerousGetReference();
97-
IntPtr length = (IntPtr)text.Length;
97+
IntPtr length = (IntPtr)(void*)(uint)text.Length;
9898

9999
return SpanHelper.Count(ref r0, length, c);
100100
}
@@ -157,10 +157,10 @@ public static ReadOnlySpanTokenizer<char> Tokenize(this string text, char separa
157157
/// <remarks>The Djb2 hash is fully deterministic and with no random components.</remarks>
158158
[Pure]
159159
[MethodImpl(MethodImplOptions.AggressiveInlining)]
160-
public static int GetDjb2HashCode(this string text)
160+
public static unsafe int GetDjb2HashCode(this string text)
161161
{
162162
ref char r0 = ref text.DangerousGetReference();
163-
IntPtr length = (IntPtr)text.Length;
163+
IntPtr length = (IntPtr)(void*)(uint)text.Length;
164164

165165
return SpanHelper.GetDjb2HashCode(ref r0, length);
166166
}

‎Microsoft.Toolkit.HighPerformance/Helpers/HashCode{T}.cs

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,7 @@ public static int Combine(ReadOnlySpan<T> span)
5757
/// <remarks>The returned hash code is not processed through <see cref="HashCode"/> APIs.</remarks>
5858
[Pure]
5959
[MethodImpl(MethodImplOptions.AggressiveInlining)]
60-
internal static int CombineValues(ReadOnlySpan<T> span)
60+
internal static unsafe int CombineValues(ReadOnlySpan<T> span)
6161
{
6262
ref T r0 = ref MemoryMarshal.GetReference(span);
6363

@@ -67,13 +67,19 @@ internal static int CombineValues(ReadOnlySpan<T> span)
6767
// compiler, so this branch will never actually be executed by the code.
6868
if (RuntimeHelpers.IsReferenceOrContainsReferences<T>())
6969
{
70-
return SpanHelper.GetDjb2HashCode(ref r0, (IntPtr)span.Length);
70+
return SpanHelper.GetDjb2HashCode(ref r0, (IntPtr)(void*)(uint)span.Length);
7171
}
7272
#endif
7373

74-
// Get the info for the target memory area to process
74+
// Get the info for the target memory area to process.
75+
// The line below is computing the total byte size for the span,
76+
// and we cast both input factors to uint first to avoid sign extensions
77+
// (they're both guaranteed to always be positive values), and to let the
78+
// JIT avoid the 64 bit computation entirely when running in a 32 bit
79+
// process. In that case it will just compute the byte size as a 32 bit
80+
// multiplication with overflow, which is guaranteed never to happen anyway.
7581
ref byte rb = ref Unsafe.As<T, byte>(ref r0);
76-
IntPtr length = (IntPtr)((long)span.Length * Unsafe.SizeOf<T>());
82+
IntPtr length = (IntPtr)(void*)((uint)span.Length * (uint)Unsafe.SizeOf<T>());
7783

7884
return SpanHelper.GetDjb2LikeByteHash(ref rb, length);
7985
}

‎Microsoft.Toolkit.HighPerformance/Helpers/ParallelHelper.ForEach.IInAction.cs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -136,7 +136,7 @@ public InActionInvoker(
136136
/// </summary>
137137
/// <param name="i">The index of the batch to process</param>
138138
[MethodImpl(MethodImplOptions.AggressiveInlining)]
139-
public void Invoke(int i)
139+
public unsafe void Invoke(int i)
140140
{
141141
int
142142
low = i * this.batchSize,
@@ -147,7 +147,7 @@ public void Invoke(int i)
147147

148148
for (int j = low; j < end; j++)
149149
{
150-
ref TItem rj = ref Unsafe.Add(ref r0, j);
150+
ref TItem rj = ref Unsafe.Add(ref r0, (IntPtr)(void*)(uint)j);
151151

152152
Unsafe.AsRef(this.action).Invoke(rj);
153153
}

‎Microsoft.Toolkit.HighPerformance/Helpers/ParallelHelper.ForEach.IRefAction.cs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -136,7 +136,7 @@ public RefActionInvoker(
136136
/// </summary>
137137
/// <param name="i">The index of the batch to process</param>
138138
[MethodImpl(MethodImplOptions.AggressiveInlining)]
139-
public void Invoke(int i)
139+
public unsafe void Invoke(int i)
140140
{
141141
int
142142
low = i * this.batchSize,
@@ -147,7 +147,7 @@ public void Invoke(int i)
147147

148148
for (int j = low; j < end; j++)
149149
{
150-
ref TItem rj = ref Unsafe.Add(ref r0, j);
150+
ref TItem rj = ref Unsafe.Add(ref r0, (IntPtr)(void*)(uint)j);
151151

152152
Unsafe.AsRef(this.action).Invoke(ref rj);
153153
}

‎Microsoft.Toolkit.HighPerformance/Microsoft.Toolkit.HighPerformance.csproj

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
- MemoryBufferWriter&lt;T&gt;: an IBufferWriter&lt;T&gt;: implementation that can wrap external Memory&lt;T&gt;: instances.
1313
- MemoryOwner&lt;T&gt;: an IMemoryOwner&lt;T&gt; implementation with an embedded length and a fast Span&lt;T&gt; accessor.
1414
- SpanOwner&lt;T&gt;: a stack-only type with the ability to rent a buffer of a specified length and getting a Span&lt;T&gt; from it.
15+
- StringPool: a configurable pool for string instances that be used to minimize allocations when creating multiple strings from char buffers.
1516
- String, array, Span&lt;T&gt;, Memory&lt;T&gt; extensions and more, all focused on high performance.
1617
- HashCode&lt;T&gt;: a SIMD-enabled extension of HashCode to quickly process sequences of values.
1718
- BitHelper: a class with helper methods to perform bit operations on numeric types.

0 commit comments

Comments
 (0)