Saving...

Author: lemire

README.md

@@ -4,8 +4,23 @@ This is a fast C# library to process unicode strings.
 
 *It is currently not meant to be usable.*
 
-Our goal is to provide faster methods than 
-https://learn.microsoft.com/en-us/dotnet/api/system.text.encoding?view=net-7.0
+## Motivation
+
+The most important immediate goal would be to speed up the 
+`Utf8Utility.GetPointerToFirstInvalidByte` function.
+
+https://github.com/dotnet/runtime/blob/4d709cd12269fcbb3d0fccfb2515541944475954/src/libraries/System.Private.CoreLib/src/System/Text/Unicode/Utf8Utility.Validation.cs
+
+
+(We may need to speed up `Ascii.GetIndexOfFirstNonAsciiByte` first.)
+
+The question is whether we could do it using this routine:
+
+* John Keiser, Daniel Lemire, [Validating UTF-8 In Less Than One Instruction Per Byte](https://arxiv.org/abs/2010.03090), Software: Practice and Experience 51 (5), 2021
+
+Our generic implementation is available there: https://github.com/simdutf/simdutf/blob/master/src/generic/utf8_validation/utf8_lookup4_algorithm.h
+
+Porting it to C# is no joke, but doable.
 
 ## Requirements
 
@@ -48,4 +63,12 @@ We recommend you use `dotnet format`. E.g.,
 ```
 cd test
 dotnet format
-```
+```
+
+
+## More reading 
+
+
+https://github.com/dotnet/coreclr/pull/21948/files#diff-2a22774bd6bff8e217ecbb3a41afad033ce0ca0f33645e9d8f5bdf7c9e3ac248
+
+https://github.com/dotnet/runtime/issues/41699

benchmark/Benchmark.cs

@@ -5,13 +5,35 @@
 using System.Text;
 using System.Runtime;
 using System.Runtime.InteropServices;
+using System.Buffers;
+
 namespace SimdUnicodeBenchmarks
 {
+
+    // See https://github.com/dotnet/performance/blob/cea924dd0639057c1062444a642a470deef96158/src/benchmarks/micro/libraries/System.Text.Encoding/Perf.Ascii.cs#L38
+    // for a standard benchmark
     public class Checker
     {
         List<char[]> names;
-        List<bool> results;
+        List<bool> results;
+        public static bool RuntimeIsAsciiApproach(ReadOnlySpan<char> s)
+        {
+            // The runtime as of NET 8.0 has a dedicated method for this, but
+            // it is not available prior to that, so let us branch.
+#if NET8_0_OR_GREATER
+            return Ascii.IsValid(s);
+#else
+            foreach (char c in s)
+            {
+                if (c >= 128)
+                {
+                    return false;
+                }
+            }
 
+            return true;
+#endif
+        }
         public static char[] GetRandomASCIIString(uint n)
         {
             var allowedChars = "abcdefghijkmnopqrstuvwxyzABCDEFGHJKLMNOPQRSTUVWXYZ01234567é89";
@@ -72,7 +94,7 @@ public void RuntimeIsAscii()
             int count = 0;
             foreach (char[] name in names)
             {
-                results[count] = (Encoding.ASCII.GetByteCount(name) == name.Length);
+                results[count] = RuntimeIsAsciiApproach(name);
                 count += 1;
             }
         }
@@ -98,9 +120,6 @@ public static void Main(string[] args)
 
             }
             var summary = BenchmarkRunner.Run<Checker>();
-            Console.WriteLine("The RuntimeIsAscii is too fast. The execution time does not depend on the string length.");
-            Console.WriteLine("It is assuredly cheating..");
-
         }
     }
 }

info/FUTUREWORK.md

@@ -0,0 +1,12 @@
+## Motivation (later)
+
+We would like to write faster functions `TranscodeToUtf8` and `TranscodeToUtf16`. Probably, 
+the most difficult and beneficial would be `TranscodeToUtf16`.
+
+
+https://github.com/dotnet/runtime/blob/4d709cd12269fcbb3d0fccfb2515541944475954/src/libraries/System.Private.CoreLib/src/System/Text/Unicode/Utf8Utility.Transcoding.cs#L838
+
+
+
+Our goal is to provide faster methods than 
+https://learn.microsoft.com/en-us/dotnet/api/system.text.encoding?view=net-7.0

test/AsciiTest.cs

@@ -1,13 +1,14 @@
 namespace tests;
+using System.Text;
 
 public class AsciiTest
 {
     [Fact]
     public void Test1()
     {
-        Assert.True(SimdUnicode.Ascii.IsAscii("absads"));
-        Assert.False(SimdUnicode.Ascii.IsAscii("absaé"));
-        Assert.True(SimdUnicode.Ascii.SIMDIsAscii("absads"));
-        Assert.False(SimdUnicode.Ascii.SIMDIsAscii("absaé"));
+        Assert.True(SimdUnicode.Ascii.IsAscii("absads12323123232131231232132132132312321321312321"));
+        Assert.False(SimdUnicode.Ascii.IsAscii("absaé12323123232131231232132132132312321321312321"));
+        Assert.True(SimdUnicode.Ascii.SIMDIsAscii("absads12323123232131231232132132132312321321312321"));
+        Assert.False(SimdUnicode.Ascii.SIMDIsAscii("absaé12323123232131231232132132132312321321312321"));
     }
 }