Merge branch 'master' into feature/memory-cast

Author: Rosuavio

Microsoft.Toolkit.HighPerformance/Box{T}.cs

@@ -78,7 +78,7 @@ public static Box<T> GetFrom(object obj)
                 ThrowInvalidCastExceptionForGetFrom();
             }
 
-            return Unsafe.As<Box<T>>(obj);
+            return Unsafe.As<Box<T>>(obj)!;
         }
 
         /// <summary>
@@ -94,7 +94,7 @@ public static Box<T> GetFrom(object obj)
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         public static Box<T> DangerousGetFrom(object obj)
         {
-            return Unsafe.As<Box<T>>(obj);
+            return Unsafe.As<Box<T>>(obj)!;
         }
 
         /// <summary>
@@ -108,7 +108,7 @@ public static bool TryGetFrom(object obj, [NotNullWhen(true)] out Box<T>? box)
         {
             if (obj.GetType() == typeof(T))
             {
-                box = Unsafe.As<Box<T>>(obj);
+                box = Unsafe.As<Box<T>>(obj)!;
 
                 return true;
             }
@@ -145,7 +145,7 @@ public static implicit operator Box<T>(T value)
             // manually be implemented in the Box<T> type. For instance, boxing a float
             // and calling ToString() on it directly, on its boxed object or on a Box<T>
             // reference retrieved from it will produce the same result in all cases.
-            return Unsafe.As<Box<T>>(value);
+            return Unsafe.As<Box<T>>(value)!;
         }
 
         /// <inheritdoc/>

Microsoft.Toolkit.HighPerformance/Buffers/ArrayPoolBufferWriter{T}.cs

@@ -10,6 +10,7 @@
 using System.Runtime.InteropServices;
 using Microsoft.Toolkit.HighPerformance.Buffers.Views;
 using Microsoft.Toolkit.HighPerformance.Extensions;
+using Microsoft.Toolkit.HighPerformance.Helpers.Internals;
 
 namespace Microsoft.Toolkit.HighPerformance.Buffers
 {
@@ -233,15 +234,15 @@ public void Advance(int count)
         /// <inheritdoc/>
         public Memory<T> GetMemory(int sizeHint = 0)
         {
-            CheckAndResizeBuffer(sizeHint);
+            CheckBufferAndEnsureCapacity(sizeHint);
 
             return this.array.AsMemory(this.index);
         }
 
         /// <inheritdoc/>
         public Span<T> GetSpan(int sizeHint = 0)
         {
-            CheckAndResizeBuffer(sizeHint);
+            CheckBufferAndEnsureCapacity(sizeHint);
 
             return this.array.AsSpan(this.index);
         }
@@ -251,9 +252,11 @@ public Span<T> GetSpan(int sizeHint = 0)
         /// </summary>
         /// <param name="sizeHint">The minimum number of items to ensure space for in <see cref="array"/>.</param>
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        private void CheckAndResizeBuffer(int sizeHint)
+        private void CheckBufferAndEnsureCapacity(int sizeHint)
         {
-            if (this.array is null)
+            T[]? array = this.array;
+
+            if (array is null)
             {
                 ThrowObjectDisposedException();
             }
@@ -268,12 +271,32 @@ private void CheckAndResizeBuffer(int sizeHint)
                 sizeHint = 1;
             }
 
-            if (sizeHint > FreeCapacity)
+            if (sizeHint > array!.Length - this.index)
             {
-                int minimumSize = this.index + sizeHint;
+                ResizeBuffer(sizeHint);
+            }
+        }
 
-                this.pool.Resize(ref this.array, minimumSize);
+        /// <summary>
+        /// Resizes <see cref="array"/> to ensure it can fit the specified number of new items.
+        /// </summary>
+        /// <param name="sizeHint">The minimum number of items to ensure space for in <see cref="array"/>.</param>
+        [MethodImpl(MethodImplOptions.NoInlining)]
+        private void ResizeBuffer(int sizeHint)
+        {
+            int minimumSize = this.index + sizeHint;
+
+            // The ArrayPool<T> class has a maximum threshold of 1024 * 1024 for the maximum length of
+            // pooled arrays, and once this is exceeded it will just allocate a new array every time
+            // of exactly the requested size. In that case, we manually round up the requested size to
+            // the nearest power of two, to ensure that repeated consecutive writes when the array in
+            // use is bigger than that threshold don't end up causing a resize every single time.
+            if (minimumSize > 1024 * 1024)
+            {
+                minimumSize = BitOperations.RoundUpPowerOfTwo(minimumSize);
             }
+
+            this.pool.Resize(ref this.array, minimumSize);
         }
 
         /// <inheritdoc/>

Microsoft.Toolkit.HighPerformance/Buffers/MemoryOwner{T}.cs

@@ -7,6 +7,9 @@
 using System.Diagnostics;
 using System.Diagnostics.Contracts;
 using System.Runtime.CompilerServices;
+#if NETCORE_RUNTIME
+using System.Runtime.InteropServices;
+#endif
 using Microsoft.Toolkit.HighPerformance.Buffers.Views;
 using Microsoft.Toolkit.HighPerformance.Extensions;
 
@@ -180,7 +183,22 @@ public Span<T> Span
                     ThrowObjectDisposedException();
                 }
 
+#if NETCORE_RUNTIME
+                ref T r0 = ref array!.DangerousGetReferenceAt(this.start);
+
+                // On .NET Core runtimes, we can manually create a span from the starting reference to
+                // skip the argument validations, which include an explicit null check, covariance check
+                // for the array and the actual validation for the starting offset and target length. We
+                // only do this on .NET Core as we can leverage the runtime-specific array layout to get
+                // a fast access to the initial element, which makes this trick worth it. Otherwise, on
+                // runtimes where we would need to at least access a static field to retrieve the base
+                // byte offset within an SZ array object, we can get better performance by just using the
+                // default Span<T> constructor and paying the cost of the extra conditional branches,
+                // especially if T is a value type, in which case the covariance check is JIT removed.
+                return MemoryMarshal.CreateSpan(ref r0, this.length);
+#else
                 return new Span<T>(array!, this.start, this.length);
+#endif
             }
         }
 
@@ -208,6 +226,31 @@ public ref T DangerousGetReference()
             return ref array!.DangerousGetReferenceAt(this.start);
         }
 
+        /// <summary>
+        /// Gets an <see cref="ArraySegment{T}"/> instance wrapping the underlying <typeparamref name="T"/> array in use.
+        /// </summary>
+        /// <returns>An <see cref="ArraySegment{T}"/> instance wrapping the underlying <typeparamref name="T"/> array in use.</returns>
+        /// <exception cref="ObjectDisposedException">Thrown when the buffer in use has already been disposed.</exception>
+        /// <remarks>
+        /// This method is meant to be used when working with APIs that only accept an array as input, and should be used with caution.
+        /// In particular, the returned array is rented from an array pool, and it is responsibility of the caller to ensure that it's
+        /// not used after the current <see cref="MemoryOwner{T}"/> instance is disposed. Doing so is considered undefined behavior,
+        /// as the same array might be in use within another <see cref="MemoryOwner{T}"/> instance.
+        /// </remarks>
+        [Pure]
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public ArraySegment<T> DangerousGetArray()
+        {
+            T[]? array = this.array;
+
+            if (array is null)
+            {
+                ThrowObjectDisposedException();
+            }
+
+            return new ArraySegment<T>(array!, this.start, this.length);
+        }
+
         /// <summary>
         /// Slices the buffer currently in use and returns a new <see cref="MemoryOwner{T}"/> instance.
         /// </summary>
@@ -222,7 +265,6 @@ public ref T DangerousGetReference()
         /// size and copy the previous items into the new one, or needing an additional variable/field
         /// to manually handle to track the used range within a given <see cref="MemoryOwner{T}"/> instance.
         /// </remarks>
-        [Pure]
         public MemoryOwner<T> Slice(int start, int length)
         {
             T[]? array = this.array;
@@ -244,6 +286,11 @@ public MemoryOwner<T> Slice(int start, int length)
                 ThrowInvalidLengthException();
             }
 
+            // We're transferring the ownership of the underlying array, so the current
+            // instance no longer needs to be disposed. Because of this, we can manually
+            // suppress the finalizer to reduce the overhead on the garbage collector.
+            GC.SuppressFinalize(this);
+
             return new MemoryOwner<T>(start, length, this.pool, array!);
         }
 

Microsoft.Toolkit.HighPerformance/Buffers/SpanOwner{T}.cs

@@ -7,6 +7,9 @@
 using System.Diagnostics;
 using System.Diagnostics.Contracts;
 using System.Runtime.CompilerServices;
+#if NETCORE_RUNTIME
+using System.Runtime.InteropServices;
+#endif
 using Microsoft.Toolkit.HighPerformance.Buffers.Views;
 using Microsoft.Toolkit.HighPerformance.Extensions;
 
@@ -143,7 +146,16 @@ public int Length
         public Span<T> Span
         {
             [MethodImpl(MethodImplOptions.AggressiveInlining)]
-            get => new Span<T>(this.array, 0, this.length);
+            get
+            {
+#if NETCORE_RUNTIME
+                ref T r0 = ref array!.DangerousGetReference();
+
+                return MemoryMarshal.CreateSpan(ref r0, this.length);
+#else
+                return new Span<T>(this.array, 0, this.length);
+#endif
+            }
         }
 
         /// <summary>
@@ -157,6 +169,23 @@ public ref T DangerousGetReference()
             return ref this.array.DangerousGetReference();
         }
 
+        /// <summary>
+        /// Gets an <see cref="ArraySegment{T}"/> instance wrapping the underlying <typeparamref name="T"/> array in use.
+        /// </summary>
+        /// <returns>An <see cref="ArraySegment{T}"/> instance wrapping the underlying <typeparamref name="T"/> array in use.</returns>
+        /// <remarks>
+        /// This method is meant to be used when working with APIs that only accept an array as input, and should be used with caution.
+        /// In particular, the returned array is rented from an array pool, and it is responsibility of the caller to ensure that it's
+        /// not used after the current <see cref="SpanOwner{T}"/> instance is disposed. Doing so is considered undefined behavior,
+        /// as the same array might be in use within another <see cref="SpanOwner{T}"/> instance.
+        /// </remarks>
+        [Pure]
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public ArraySegment<T> DangerousGetArray()
+        {
+            return new ArraySegment<T>(array!, 0, this.length);
+        }
+
         /// <summary>
         /// Implements the duck-typed <see cref="IDisposable.Dispose"/> method.
         /// </summary>

Microsoft.Toolkit.HighPerformance/Buffers/StringPool.cs

@@ -5,15 +5,13 @@
 using System;
 using System.Diagnostics.CodeAnalysis;
 using System.Diagnostics.Contracts;
-#if NETCOREAPP3_1
-using System.Numerics;
-#endif
 using System.Runtime.CompilerServices;
 using System.Text;
 using Microsoft.Toolkit.HighPerformance.Extensions;
 #if !NETSTANDARD1_4
 using Microsoft.Toolkit.HighPerformance.Helpers;
 #endif
+using BitOperations = Microsoft.Toolkit.HighPerformance.Helpers.Internals.BitOperations;
 
 #nullable enable
 
@@ -79,8 +77,8 @@ static void FindFactors(int size, int factor, out int x, out int y)
                     a = Math.Sqrt((double)size / factor),
                     b = factor * a;
 
-                x = RoundUpPowerOfTwo((int)a);
-                y = RoundUpPowerOfTwo((int)b);
+                x = BitOperations.RoundUpPowerOfTwo((int)a);
+                y = BitOperations.RoundUpPowerOfTwo((int)b);
             }
 
             // We want to find two powers of 2 factors that produce a number
@@ -130,30 +128,6 @@ static void FindFactors(int size, int factor, out int x, out int y)
             Size = p2;
         }
 
-        /// <summary>
-        /// Rounds up an <see cref="int"/> value to a power of 2.
-        /// </summary>
-        /// <param name="x">The input value to round up.</param>
-        /// <returns>The smallest power of two greater than or equal to <paramref name="x"/>.</returns>
-        [Pure]
-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        private static int RoundUpPowerOfTwo(int x)
-        {
-#if NETCOREAPP3_1
-            return 1 << (32 - BitOperations.LeadingZeroCount((uint)(x - 1)));
-#else
-            x--;
-            x |= x >> 1;
-            x |= x >> 2;
-            x |= x >> 4;
-            x |= x >> 8;
-            x |= x >> 16;
-            x++;
-
-            return x;
-#endif
-        }
-
         /// <summary>
         /// Gets the shared <see cref="StringPool"/> instance.
         /// </summary>
@@ -422,11 +396,11 @@ public object SyncRoot
             /// <param name="value">The input <see cref="string"/> instance to cache.</param>
             /// <param name="hashcode">The precomputed hashcode for <paramref name="value"/>.</param>
             [MethodImpl(MethodImplOptions.AggressiveInlining)]
-            public unsafe void Add(string value, int hashcode)
+            public void Add(string value, int hashcode)
             {
                 ref string target = ref TryGet(value.AsSpan(), hashcode);
 
-                if (Unsafe.AreSame(ref target, ref Unsafe.AsRef<string>(null)))
+                if (Unsafe.IsNullRef(ref target))
                 {
                     Insert(value, hashcode);
                 }
@@ -443,11 +417,11 @@ public unsafe void Add(string value, int hashcode)
             /// <param name="hashcode">The precomputed hashcode for <paramref name="value"/>.</param>
             /// <returns>A <see cref="string"/> instance with the contents of <paramref name="value"/>.</returns>
             [MethodImpl(MethodImplOptions.AggressiveInlining)]
-            public unsafe string GetOrAdd(string value, int hashcode)
+            public string GetOrAdd(string value, int hashcode)
             {
                 ref string result = ref TryGet(value.AsSpan(), hashcode);
 
-                if (!Unsafe.AreSame(ref result, ref Unsafe.AsRef<string>(null)))
+                if (!Unsafe.IsNullRef(ref result))
                 {
                     return result;
                 }
@@ -464,11 +438,11 @@ public unsafe string GetOrAdd(string value, int hashcode)
             /// <param name="hashcode">The precomputed hashcode for <paramref name="span"/>.</param>
             /// <returns>A <see cref="string"/> instance with the contents of <paramref name="span"/>, cached if possible.</returns>
             [MethodImpl(MethodImplOptions.AggressiveInlining)]
-            public unsafe string GetOrAdd(ReadOnlySpan<char> span, int hashcode)
+            public string GetOrAdd(ReadOnlySpan<char> span, int hashcode)
             {
                 ref string result = ref TryGet(span, hashcode);
 
-                if (!Unsafe.AreSame(ref result, ref Unsafe.AsRef<string>(null)))
+                if (!Unsafe.IsNullRef(ref result))
                 {
                     return result;
                 }
@@ -488,11 +462,11 @@ public unsafe string GetOrAdd(ReadOnlySpan<char> span, int hashcode)
             /// <param name="value">The resulting cached <see cref="string"/> instance, if present</param>
             /// <returns>Whether or not the target <see cref="string"/> instance was found.</returns>
             [MethodImpl(MethodImplOptions.AggressiveInlining)]
-            public unsafe bool TryGet(ReadOnlySpan<char> span, int hashcode, [NotNullWhen(true)] out string? value)
+            public bool TryGet(ReadOnlySpan<char> span, int hashcode, [NotNullWhen(true)] out string? value)
             {
                 ref string result = ref TryGet(span, hashcode);
 
-                if (!Unsafe.AreSame(ref result, ref Unsafe.AsRef<string>(null)))
+                if (!Unsafe.IsNullRef(ref result))
                 {
                     value = result;
 
@@ -527,7 +501,7 @@ public void Reset()
             private unsafe ref string TryGet(ReadOnlySpan<char> span, int hashcode)
             {
                 ref MapEntry mapEntriesRef = ref this.mapEntries.DangerousGetReference();
-                ref MapEntry entry = ref Unsafe.AsRef<MapEntry>(null);
+                ref MapEntry entry = ref Unsafe.NullRef<MapEntry>();
                 int
                     length = this.buckets.Length,
                     bucketIndex = hashcode & (length - 1);
@@ -547,7 +521,7 @@ private unsafe ref string TryGet(ReadOnlySpan<char> span, int hashcode)
                     }
                 }
 
-                return ref Unsafe.AsRef<string>(null);
+                return ref Unsafe.NullRef<string>();
             }
 
             /// <summary>

Microsoft.Toolkit.HighPerformance/Extensions/ArrayExtensions.1D.cs

@@ -31,7 +31,7 @@ public static partial class ArrayExtensions
         public static ref T DangerousGetReference<T>(this T[] array)
         {
 #if NETCORE_RUNTIME
-            var arrayData = Unsafe.As<RawArrayData>(array);
+            var arrayData = Unsafe.As<RawArrayData>(array)!;
             ref T r0 = ref Unsafe.As<byte, T>(ref arrayData.Data);
 
             return ref r0;
@@ -55,7 +55,7 @@ public static ref T DangerousGetReference<T>(this T[] array)
         public static ref T DangerousGetReferenceAt<T>(this T[] array, int i)
         {
 #if NETCORE_RUNTIME
-            var arrayData = Unsafe.As<RawArrayData>(array);
+            var arrayData = Unsafe.As<RawArrayData>(array)!;
             ref T r0 = ref Unsafe.As<byte, T>(ref arrayData.Data);
             ref T ri = ref Unsafe.Add(ref r0, (nint)(uint)i);