Remove Microsoft.Bcl.Memory 9.0 in Wilson (#3220)

* started merging for Base64UrlEncoder, need to work on decoding

* fixed the decoding for Base64UrlEncoding.cs

* updated base64url encoding.cs and log messages

* update PR according to comments

* updated for .net9.0

* updated bcl memory

---------

Co-authored-by: Mallika Chennupaty <[email protected]>

Author: mdchennu

benchmark/Microsoft.IdentityModel.Benchmarks/Microsoft.IdentityModel.Benchmarks.csproj

@@ -48,9 +48,4 @@
     <ProjectReference Include="..\..\test\Microsoft.IdentityModel.TestUtils\Microsoft.IdentityModel.TestUtils.csproj" />
   </ItemGroup>
 
-  <PropertyGroup Condition="'$(TargetFramework)' == 'net6.0'">
-    <!-- Suprpess the "Microsoft.Bcl.Memory 9.0.0 doesn't support net6.0 and has not been tested with it" warning -->
-    <SuppressTfmSupportBuildWarnings>true</SuppressTfmSupportBuildWarnings>
-  </PropertyGroup>
-
 </Project>

build/common.props

@@ -31,11 +31,6 @@
     <EnforceCodeStyleInBuild>true</EnforceCodeStyleInBuild>
   </PropertyGroup>
 
-  <PropertyGroup Condition="'$(TargetFramework)' == 'net6.0'">
-    <!-- Suprpess the "Microsoft.Bcl.Memory 9.0.0 doesn't support net6.0 and has not been tested with it" warning -->
-    <SuppressTfmSupportBuildWarnings>true</SuppressTfmSupportBuildWarnings>
-  </PropertyGroup>
-
   <PropertyGroup>
     <EnablePackageValidation>false</EnablePackageValidation>
     <PackageValidationBaselineVersion>8.0.0</PackageValidationBaselineVersion>

build/commonTest.props

@@ -51,11 +51,6 @@
     <AdditionalFiles Include="$(MSBuildThisFileDirectory)..\BannedSymbols.txt" />
   </ItemGroup>
 
-  <PropertyGroup Condition="'$(TargetFramework)' == 'net6.0'">
-    <!-- Suprpess the "Microsoft.Bcl.Memory/Microsoft.Extensions.TimeProvider.Testing 9.0.0 doesn't support net6.0 and has not been tested with it" warning -->
-    <SuppressTfmSupportBuildWarnings>true</SuppressTfmSupportBuildWarnings>
-  </PropertyGroup>
-
   <PropertyGroup Condition="'$(TargetFramework)' == 'net462' or '$(TargetFramework)' == 'net472'">
     <!-- Suprpess the "Microsoft.Extensions.TimeProvider.Testing 9.0.0 doesn't support net462 and has not been tested with it" warning -->
     <SuppressTfmSupportBuildWarnings>true</SuppressTfmSupportBuildWarnings>

build/dependencies.props

@@ -12,7 +12,6 @@
     <SystemMemoryVersion>4.5.5</SystemMemoryVersion>
     <SystemSecurityCryptographyCngVersion>4.5.0</SystemSecurityCryptographyCngVersion>
     <SystemTextJson>8.0.5</SystemTextJson>
-    <MicrosoftBclMemory>9.0.0</MicrosoftBclMemory>
   </PropertyGroup>
 
 

src/Microsoft.IdentityModel.Tokens/Base64UrlEncoder.cs

@@ -3,7 +3,9 @@
 
 using System;
 using System.Buffers;
+#if NET9_0_OR_GREATER
 using System.Buffers.Text;
+#endif
 using System.Text;
 using Microsoft.IdentityModel.Logging;
 
@@ -21,6 +23,8 @@ public static class Base64UrlEncoder
         private const char Base64PadCharacter = '=';
         private const char Base64Character62 = '+';
         private const char Base64Character63 = '/';
+        private const char Base64UrlCharacter62 = '-';
+        private const char Base64UrlCharacter63 = '_';
 
         /// <summary>
         /// Performs base64url encoding, which differs from regular base64 encoding as follows:
@@ -98,17 +102,86 @@ public static string Encode(byte[] inArray, int offset, int length)
                         LogHelper.MarkAsNonPII(inArray.Length))));
 #pragma warning restore CA2208 // Instantiate argument exceptions correctly
 
+#if NET9_0_OR_GREATER
             return Base64Url.EncodeToString(inArray.AsSpan().Slice(offset, length));
+#else
+            char[] destination = new char[(inArray.Length + 2) / 3 * 4];
+            int j = Encode(inArray.AsSpan<byte>().Slice(offset, length), destination.AsSpan<char>());
+
+            return new string(destination, 0, j);
+#endif
         }
 
+#if NET9_0_OR_GREATER
         /// <summary>
         /// Populates a <see cref="Span{T}"/> with the base64url encoded representation of a <see cref="ReadOnlySpan{T}"/> of bytes.
         /// </summary>
         /// <param name="inArray">A read-only span of bytes to encode.</param>
         /// <param name="output">The span of characters to write the encoded output.</param>
         /// <returns>The number of characters written to the output span.</returns>
         public static int Encode(ReadOnlySpan<byte> inArray, Span<char> output) => Base64Url.EncodeToChars(inArray, output);
+#else
+        /// <summary>
+        /// Populates a <see cref="Span{T}"/> with the base64url encoded representation of a <see cref="ReadOnlySpan{T}"/> of bytes.
+        /// </summary>
+        /// <param name="inArray">A read-only span of bytes to encode.</param>
+        /// <param name="output">The span of characters to write the encoded output.</param>
+        /// <returns>The number of characters written to the output span.</returns>
+        public static int Encode(ReadOnlySpan<byte> inArray, Span<char> output)
+        {
+            int lengthmod3 = inArray.Length % 3;
+            int limit = (inArray.Length - lengthmod3);
+            ReadOnlySpan<byte> table = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_"u8;
+
+            int i, j = 0;
+
+            // takes 3 bytes from inArray and insert 4 bytes into output
+            for (i = 0; i < limit; i += 3)
+            {
+                byte d0 = inArray[i];
+                byte d1 = inArray[i + 1];
+                byte d2 = inArray[i + 2];
+
+                output[j + 0] = (char)table[d0 >> 2];
+                output[j + 1] = (char)table[((d0 & 0x03) << 4) | (d1 >> 4)];
+                output[j + 2] = (char)table[((d1 & 0x0f) << 2) | (d2 >> 6)];
+                output[j + 3] = (char)table[d2 & 0x3f];
+                j += 4;
+            }
+
+            //Where we left off before
+            i = limit;
 
+            switch (lengthmod3)
+            {
+                case 2:
+                    {
+                        byte d0 = inArray[i];
+                        byte d1 = inArray[i + 1];
+
+                        output[j + 0] = (char)table[d0 >> 2];
+                        output[j + 1] = (char)table[((d0 & 0x03) << 4) | (d1 >> 4)];
+                        output[j + 2] = (char)table[(d1 & 0x0f) << 2];
+                        j += 3;
+                    }
+                    break;
+                case 1:
+                    {
+                        byte d0 = inArray[i];
+
+                        output[j + 0] = (char)table[d0 >> 2];
+                        output[j + 1] = (char)table[(d0 & 0x03) << 4];
+                        j += 2;
+                    }
+                    break;
+
+                    //default or case 0: no further operations are needed.
+            }
+
+            return j;
+        }
+
+#endif
         /// <summary>
         /// Converts the specified base64url encoded string to UTF-8 bytes.
         /// </summary>
@@ -123,6 +196,8 @@ public static byte[] DecodeBytes(string str)
 #if NETCOREAPP
         [SkipLocalsInit]
 #endif
+
+#if NET9_0_OR_GREATER
         internal static byte[] Decode(ReadOnlySpan<char> strSpan)
         {
             int upperBound = Base64Url.GetMaxDecodedLength(strSpan.Length);
@@ -144,33 +219,37 @@ internal static byte[] Decode(ReadOnlySpan<char> strSpan)
                     ArrayPool<byte>.Shared.Return(rented, true);
             }
         }
-
-#if !NET8_0_OR_GREATER
-        private static bool IsOnlyValidBase64Chars(ReadOnlySpan<char> strSpan)
+#else
+        internal static byte[] Decode(ReadOnlySpan<char> strSpan)
         {
-            foreach (char c in strSpan)
-                if (!char.IsDigit(c) && !char.IsLetter(c) && c != Base64Character62 && c != Base64Character63 && c != Base64PadCharacter)
-                    return false;
+            int mod = strSpan.Length % 4;
+            if (mod == 1)
+                throw LogHelper.LogExceptionMessage(new FormatException(LogHelper.FormatInvariant(LogMessages.IDX10400, strSpan.ToString())));
 
-            return true;
+            bool needReplace = strSpan.IndexOfAny(Base64UrlCharacter62, Base64UrlCharacter63) >= 0;
+            int decodedLength = strSpan.Length + (4 - mod) % 4;
+#if NET6_0_OR_GREATER
+            Span<byte> output = new byte[decodedLength];
+            int length = Decode(strSpan, output, needReplace, decodedLength);
+            return output.Slice(0, length).ToArray();
+#else
+            return UnsafeDecode(strSpan, needReplace, decodedLength);
+#endif
         }
-
 #endif
+
 #if NETCOREAPP
         [SkipLocalsInit]
 #endif
+
+#if NET9_0_OR_GREATER
         internal static int Decode(ReadOnlySpan<char> strSpan, Span<byte> output)
         {
             OperationStatus status = Base64Url.DecodeFromChars(strSpan, output, out _, out int bytesWritten);
             if (status == OperationStatus.Done)
                 return bytesWritten;
 
-            if (status == OperationStatus.InvalidData &&
-#if NET8_0_OR_GREATER
-                !Base64.IsValid(strSpan))
-#else
-                !IsOnlyValidBase64Chars(strSpan))
-#endif
+            if (status == OperationStatus.InvalidData && !Base64.IsValid(strSpan))
                 throw LogHelper.LogExceptionMessage(new FormatException(LogHelper.FormatInvariant(LogMessages.IDX10400, strSpan.ToString())));
 
             int mod = strSpan.Length % 4;
@@ -180,8 +259,24 @@ internal static int Decode(ReadOnlySpan<char> strSpan, Span<byte> output)
 
             return Decode(strSpan, output, decodedLength);
         }
+#else
+        internal static void Decode(ReadOnlySpan<char> strSpan, Span<byte> output)
+        {
+            int mod = strSpan.Length % 4;
+            if (mod == 1)
+                throw LogHelper.LogExceptionMessage(new FormatException(LogHelper.FormatInvariant(LogMessages.IDX10400, strSpan.ToString())));
+            bool needReplace = strSpan.IndexOfAny(Base64UrlCharacter62, Base64UrlCharacter63) >= 0;
+            int decodedLength = strSpan.Length + (4 - mod) % 4;
+#if NET6_0_OR_GREATER
+            Decode(strSpan, output, needReplace, decodedLength);
+#else
+            Decode(strSpan, output, needReplace, decodedLength);
+#endif
+        }
 
-#if NETCOREAPP
+#endif
+
+#if NET9_0_OR_GREATER
         [SkipLocalsInit]
         private static int Decode(ReadOnlySpan<char> strSpan, Span<byte> output, int decodedLength)
         {
@@ -254,33 +349,144 @@ private static ReadOnlySpan<char> HandlePadding(ReadOnlySpan<char> source, Span<
 
             return charsSpan;
         }
-#else
-        private static unsafe byte[] UnsafeDecode(ReadOnlySpan<char> strSpan, int decodedLength)
+#elif NET6_0_OR_GREATER
+        [SkipLocalsInit]
+        private static int Decode(ReadOnlySpan<char> strSpan, Span<byte> output, bool needReplace, int decodedLength)
         {
-            if (decodedLength == strSpan.Length)
+            // If the incoming chars don't contain any of the base64url characters that need to be replaced,
+            // and if the incoming chars are of the exact right length, then we'll be able to just pass the
+            // incoming chars directly to DecodeFromUtf8InPlace. Otherwise, rent an array, copy all the
+            // data into it, and do whatever fixups are necessary on that copy, then pass that copy into
+            // DecodeFromUtf8InPlace.
+
+            const int StackAllocThreshold = 512;
+            char[] arrayPoolChars = null;
+            scoped Span<char> charsSpan = default;
+            scoped ReadOnlySpan<char> source = strSpan;
+
+            if (needReplace || decodedLength != source.Length)
+            {
+                charsSpan = decodedLength <= StackAllocThreshold ?
+                    stackalloc char[StackAllocThreshold] :
+                    arrayPoolChars = ArrayPool<char>.Shared.Rent(decodedLength);
+                charsSpan = charsSpan.Slice(0, decodedLength);
+
+                source = HandlePaddingAndReplace(source, charsSpan, needReplace);
+            }
+
+            byte[] arrayPoolBytes = null;
+            Span<byte> bytesSpan = decodedLength <= StackAllocThreshold ?
+                stackalloc byte[StackAllocThreshold] :
+                arrayPoolBytes = ArrayPool<byte>.Shared.Rent(decodedLength);
+
+            int length = Encoding.UTF8.GetBytes(source, bytesSpan);
+            Span<byte> utf8Span = bytesSpan.Slice(0, length);
+            try
             {
-                return Convert.FromBase64CharArray(strSpan.ToArray(), 0, strSpan.Length);
+                OperationStatus status = System.Buffers.Text.Base64.DecodeFromUtf8InPlace(utf8Span, out int bytesWritten);
+                if (status != OperationStatus.Done)
+                    throw LogHelper.LogExceptionMessage(new FormatException(LogHelper.FormatInvariant(LogMessages.IDX10400, strSpan.ToString())));
+
+                utf8Span.Slice(0, bytesWritten).CopyTo(output);
+
+                return bytesWritten;
             }
+            finally
+            {
+                if (arrayPoolBytes is not null)
+                {
+                    bytesSpan.Clear();
+                    ArrayPool<byte>.Shared.Return(arrayPoolBytes);
+                }
 
-            string decodedString = new(char.MinValue, decodedLength);
-            fixed (char* src = strSpan)
-            fixed (char* dest = decodedString)
+                if (arrayPoolChars is not null)
+                {
+                    charsSpan.Clear();
+                    ArrayPool<char>.Shared.Return(arrayPoolChars);
+                }
+            }
+        }
+
+        private static ReadOnlySpan<char> HandlePaddingAndReplace(ReadOnlySpan<char> source, Span<char> charsSpan, bool needReplace)
+        {
+            source.CopyTo(charsSpan);
+            if (source.Length < charsSpan.Length)
             {
-                Buffer.MemoryCopy(src, dest, strSpan.Length * 2, strSpan.Length * 2);
+                charsSpan[source.Length] = Base64PadCharacter;
+                if (source.Length + 1 < charsSpan.Length)
+                {
+                    charsSpan[source.Length + 1] = Base64PadCharacter;
+                }
+            }
 
-                dest[strSpan.Length] = Base64PadCharacter;
-                if (strSpan.Length + 2 == decodedLength)
-                    dest[strSpan.Length + 1] = Base64PadCharacter;
+            if (needReplace)
+            {
+                Span<char> remaining = charsSpan;
+                int pos;
+                while ((pos = remaining.IndexOfAny(Base64UrlCharacter62, Base64UrlCharacter63)) >= 0)
+                {
+                    remaining[pos] = (remaining[pos] == Base64UrlCharacter62) ? Base64Character62 : Base64Character63;
+                    remaining = remaining.Slice(pos + 1);
+                }
             }
 
-            return Convert.FromBase64String(decodedString);
+            return charsSpan;
         }
 
-        private static int Decode(ReadOnlySpan<char> strSpan, Span<byte> output, int decodedLength)
+#else
+        private static unsafe byte[] UnsafeDecode(ReadOnlySpan<char> strSpan, bool needReplace, int decodedLength)
+        {
+            if (needReplace)
+            {
+                string decodedString = new(char.MinValue, decodedLength);
+                fixed (char* dest = decodedString)
+                {
+                    int i = 0;
+                    for (; i < strSpan.Length; i++)
+                    {
+                        if (strSpan[i] == Base64UrlCharacter62)
+                            dest[i] = Base64Character62;
+                        else if (strSpan[i] == Base64UrlCharacter63)
+                            dest[i] = Base64Character63;
+                        else
+                            dest[i] = strSpan[i];
+                    }
+
+                    for (; i < decodedLength; i++)
+                        dest[i] = Base64PadCharacter;
+                }
+
+                return Convert.FromBase64String(decodedString);
+            }
+            else
+            {
+                if (decodedLength == strSpan.Length)
+                {
+                    return Convert.FromBase64CharArray(strSpan.ToArray(), 0, strSpan.Length);
+                }
+                else
+                {
+                    string decodedString = new(char.MinValue, decodedLength);
+                    fixed (char* src = strSpan)
+                    fixed (char* dest = decodedString)
+                    {
+                        Buffer.MemoryCopy(src, dest, strSpan.Length * 2, strSpan.Length * 2);
+
+                        dest[strSpan.Length] = Base64PadCharacter;
+                        if (strSpan.Length + 2 == decodedLength)
+                            dest[strSpan.Length + 1] = Base64PadCharacter;
+                    }
+
+                    return Convert.FromBase64String(decodedString);
+                }
+            }
+        }
+
+        private static void Decode(ReadOnlySpan<char> strSpan, Span<byte> output, bool needReplace, int decodedLength)
         {
-            byte[] result = UnsafeDecode(strSpan, decodedLength);
+            byte[] result = UnsafeDecode(strSpan, needReplace, decodedLength);
             result.CopyTo(output);
-            return result.Length;
+
         }
 #endif