Fix UTF8 BOM tests

Author: mythz

src/ServiceStack.Text/DefaultMemory.cs

@@ -534,7 +534,7 @@ public override ReadOnlyMemory<char> FromUtf8(ReadOnlySpan<byte> source)
         {
             var bytes = source.WithoutBom().ToArray();
             var chars = new char[Encoding.UTF8.GetCharCount(bytes)];
-            var charsWritten = Encoding.UTF8.GetChars(bytes, 0, source.Length, chars, 0);
+            var charsWritten = Encoding.UTF8.GetChars(bytes, 0, bytes.Length, chars, 0);
             return new ReadOnlyMemory<char>(chars, 0, charsWritten);
         }
 
@@ -551,7 +551,7 @@ public override int FromUtf8(ReadOnlySpan<byte> source, Span<char> destination)
         {
             var bytes = source.WithoutBom().ToArray();
             var chars = destination.ToArray();
-            var charsWritten = Encoding.UTF8.GetChars(bytes, 0, source.Length, chars, 0);
+            var charsWritten = Encoding.UTF8.GetChars(bytes, 0, bytes.Length, chars, 0);
             new ReadOnlySpan<char>(chars, 0, charsWritten).CopyTo(destination);
             return charsWritten;
         }

src/ServiceStack.Text/StringExtensions.cs

@@ -305,6 +305,13 @@ public static byte[] ToUtf8Bytes(this double doubleVal)
             return FastToUtf8Bytes(doubleStr);
         }
 
+        public static string WithoutBom(this string value)
+        {
+            return value.Length > 0 && value[0] == 65279 
+                ? value.Substring(1) 
+                : value;
+        }
+
         // from JWT spec
         public static string ToBase64UrlSafe(this byte[] input)
         {

tests/ServiceStack.Text.Tests/SpanMemoryTests.cs

@@ -118,8 +118,10 @@ public void Can_ToUtf8_and_FromUtf8_using_Memory()
                 Assert.That(bytes.ToArray(), Is.EquivalentTo(test.expectedBytes));
 
                 ReadOnlyMemory<char> chars = bytes.FromUtf8();
-                Assert.That(chars.Length, Is.EqualTo(test.expectedString.Length));
-                Assert.That(chars.ToString(), Is.EqualTo(test.expectedString));
+                Assert.That(chars.Length, Is.EqualTo(test.expectedString.Length)
+                    .Or.EqualTo(test.expectedString.WithoutBom().Length));
+                Assert.That(chars.ToString(), Is.EqualTo(test.expectedString)
+                    .Or.EqualTo(test.expectedString.WithoutBom()));
             }
         }
 

tests/ServiceStack.Text.Tests/SpanTests.cs

@@ -111,8 +111,10 @@ public void Can_ToUtf8_and_FromUtf8_using_Span()
                 Assert.That(bytes.ToArray(), Is.EquivalentTo(test.expectedBytes));
 
                 ReadOnlyMemory<char> chars = bytes.FromUtf8();
-                Assert.That(chars.Length, Is.EqualTo(test.expectedString.Length));
-                Assert.That(chars.ToString(), Is.EqualTo(test.expectedString));
+                Assert.That(chars.Length, Is.EqualTo(test.expectedString.Length)
+                    .Or.EqualTo(test.expectedString.WithoutBom().Length));
+                Assert.That(chars.ToString(), Is.EqualTo(test.expectedString)
+                    .Or.EqualTo(test.expectedString.WithoutBom()));
             }
         }
 
@@ -133,8 +135,10 @@ public void Can_ToUtf8_and_FromUtf8_in_place_using_Span()
                 var charsWritten = MemoryProvider.Instance.FromUtf8(bytes.Span, charBuff.Span);
                 chars = charBuff.Slice(0, charsWritten).Span;
 
-                Assert.That(chars.Length, Is.EqualTo(test.expectedString.Length));
-                Assert.That(chars.ToString(), Is.EqualTo(test.expectedString));
+                Assert.That(chars.Length, Is.EqualTo(test.expectedString.Length)
+                    .Or.EqualTo(test.expectedString.WithoutBom().Length));
+                Assert.That(chars.ToString(), Is.EqualTo(test.expectedString)
+                    .Or.EqualTo(test.expectedString.WithoutBom()));
             }
         }