Optimize surrogate decoding.

Use `char ^ 0xD800 <= 0x3FF` to check if a char code is a lead
surrogate. That avoids doing a later `& 0x3FF` to get rid of the
top bits. Similar for tail surrogate.

This ensures that the `high` function gets values without high
bits.
Also optimize that function to reduce dependency depth and
try to hit `base + (something < small)` expressions that can
optimized into a single x64 address computation.

Gives a ~7% increase on backwards traversal and 38% increase for
forward traversal, based on tool/benchmark.dart compiled with
`dart compile exe`.

Author: lrhn

pkgs/characters/CHANGELOG.md

@@ -1,6 +1,7 @@
-## 1.4.1-wip
+## 1.4.1
 
-- Run `dart format` with the new style.
+* Run `dart format` with the new style.
+* Performance improvement for non-BMP characters.
 
 ## 1.4.0
 

pkgs/characters/lib/src/characters_impl.dart

@@ -509,13 +509,14 @@ class StringCharacterRange implements CharacterRange {
       var index = _end;
       while (index < _string.length) {
         var char = _string.codeUnitAt(index);
+        var surrogateBits = char ^ 0xD800;
         var category = categoryControl;
         var nextIndex = index + 1;
-        if (char & 0xFC00 != 0xD800) {
+        if (surrogateBits >= 0x3FF) {
           category = low(char);
         } else if (nextIndex < _string.length) {
-          var nextChar = _string.codeUnitAt(nextIndex);
-          if (nextChar & 0xFC00 == 0xDC00) {
+          var nextChar = _string.codeUnitAt(nextIndex) ^ 0xDC00;
+          if (nextChar <= 0x3FF) {
             nextIndex += 1;
             category = high(char, nextChar);
           }

pkgs/characters/lib/src/grapheme_clusters/breaks.dart

@@ -76,16 +76,17 @@ class Breaks {
   void step() {
     assert(cursor < end);
     var char = base.codeUnitAt(cursor++);
-    if (char & 0xFC00 != 0xD800) {
+    var surrogate = char ^ 0xD800;
+    if (surrogate >= 0x3FF) {
       state = move(state, low(char));
       return;
     }
     // The category of an unpaired lead surrogate is Control.
     int category;
-    int nextChar;
+    int nextSurrogate;
     if (cursor < end &&
-        (nextChar = base.codeUnitAt(cursor)) & 0xFC00 == 0xDC00) {
-      category = high(char, nextChar);
+        (nextSurrogate = base.codeUnitAt(cursor) ^ 0xDC00) <= 0x3FF) {
+      category = high(surrogate, nextSurrogate);
       cursor++;
     } else {
       category = categoryControl;
@@ -112,27 +113,28 @@ class Breaks {
     }
     var cursorBefore = cursor - 1;
     var prevChar = base.codeUnitAt(cursorBefore);
+    var prevSurrogate = prevChar ^ 0xD800;
     int prevCategory;
-    if (prevChar & 0xF800 != 0xD800) {
+    if (prevSurrogate > 0x7FF) {
       // Not surrogate.
       prevCategory = low(prevChar);
-    } else if (prevChar & 0xFC00 == 0xD800) {
+    } else if (prevSurrogate <= 0x3FF) {
       // Lead surrogate. Check for a following tail surrogate.
-      int tailChar;
+      int tailSurrogate;
       if (cursor < end &&
-          (tailChar = base.codeUnitAt(cursor)) & 0xFC00 == 0xDC00) {
+          (tailSurrogate = base.codeUnitAt(cursor) ^ 0xDC00) <= 0x3FF) {
         cursor += 1;
-        prevCategory = high(prevChar, tailChar);
+        prevCategory = high(prevSurrogate, tailSurrogate);
       } else {
         prevCategory = categoryControl;
       }
     } else {
       // Tail surrogate, check for prior lead surrogate.
-      int leadChar;
+      int leadSurrogate;
       var leadIndex = cursorBefore - 1;
       if (leadIndex >= start &&
-          (leadChar = base.codeUnitAt(leadIndex)) & 0xFC00 == 0xD800) {
-        prevCategory = high(leadChar, prevChar);
+          (leadSurrogate = base.codeUnitAt(leadIndex) ^ 0xD800) <= 0x3FF) {
+        prevCategory = high(leadSurrogate, prevSurrogate);
         cursorBefore = leadIndex;
       } else {
         prevCategory = categoryControl;
@@ -206,18 +208,19 @@ class BackBreaks {
   void step() {
     assert(cursor > start);
     var char = base.codeUnitAt(--cursor);
-    if (char & 0xFC00 != 0xDC00) {
+    var surrogate = char ^ 0xDC00;
+    if (surrogate > 0x3FF) {
       var category = low(char);
       state = moveBack(state, category);
       return;
     }
     // Found tail surrogate, check for prior lead surrogate.
     // The category of an unpaired tail surrogate is Control.
     int category;
-    int prevChar;
+    int prevSurrogate;
     if (cursor >= start &&
-        (prevChar = base.codeUnitAt(--cursor)) & 0xFC00 == 0xD800) {
-      category = high(prevChar, char);
+        (prevSurrogate = base.codeUnitAt(--cursor) ^ 0xD800) <= 0x3FF) {
+      category = high(prevSurrogate, surrogate);
     } else {
       category = categoryControl;
       cursor++;
@@ -342,21 +345,23 @@ int previousBreak(String text, int start, int end, int index) {
   if (start < index && index < end) {
     var cursorBefore = index;
     var nextChar = text.codeUnitAt(index);
+    var nextSurrogate = nextChar ^ 0xD800;
     var category = categoryControl;
-    if (nextChar & 0xF800 != 0xD800) {
+    if (nextSurrogate > 0x7FF) {
       category = low(nextChar);
-    } else if (nextChar & 0xFC00 == 0xD800) {
+    } else if (nextSurrogate <= 0x3FF) {
       var indexAfter = index + 1;
       if (indexAfter < end) {
-        var secondChar = text.codeUnitAt(indexAfter);
-        if (secondChar & 0xFC00 == 0xDC00) {
-          category = high(nextChar, secondChar);
+        var secondSurrogate = text.codeUnitAt(indexAfter) ^ 0xDC00;
+        if (secondSurrogate <= 0x3FF) {
+          category = high(nextChar, secondSurrogate);
         }
       }
     } else {
-      var prevChar = text.codeUnitAt(index - 1);
-      if (prevChar & 0xFC00 == 0xD800) {
-        category = high(prevChar, nextChar);
+      var prevSurrogate = text.codeUnitAt(index - 1) ^ 0xD800;
+      nextSurrogate &= 0x3FF;
+      if (prevSurrogate <= 0x3FF) {
+        category = high(prevSurrogate, nextSurrogate);
         cursorBefore -= 1;
       }
     }

pkgs/characters/lib/src/grapheme_clusters/table.dart

@@ -1139,10 +1139,10 @@ int low(int codeUnit) {
 @pragma('vm:prefer-inline')
 @pragma('wasm:prefer-inline')
 int high(int lead, int tail) {
-  var offset = (((0x3ff & lead) << 10) + (0x3ff & tail)) + (2048 << 8);
-  var chunkStart = _start.codeUnitAt(offset >> 8);
-  var index = chunkStart + (tail & 255);
-  return _data.codeUnitAt(index);
+  var offset = (tail >> 8) + (lead << 2);
+  tail &= 255;
+  var chunkStart = _start.codeUnitAt(2048 + offset);
+  return _data.codeUnitAt(chunkStart + tail);
 }
 
 const _stateMachine = '\x15\x01)))µ\x8d\x01=QeyeyÉ)))ñð\x15\x01)))µ\x8d\x00=Qey'

pkgs/characters/pubspec.yaml

@@ -1,5 +1,5 @@
 name: characters
-version: 1.4.1-wip
+version: 1.4.1
 description: >-
   String replacement with operations that are Unicode/grapheme cluster aware.
 repository: https://github.com/dart-lang/core/tree/main/pkgs/characters

pkgs/characters/test/src/unicode_tests.dart

@@ -31,7 +31,7 @@ String testDescription(List<String> expected) {
 int categoryOf(int codePoint) {
   if (codePoint < 0x10000) return low(codePoint);
   var nonBmpOffset = codePoint - 0x10000;
-  return high(0xD800 + (nonBmpOffset >> 10), 0xDC00 + (nonBmpOffset & 0x3ff));
+  return high(nonBmpOffset >> 10, nonBmpOffset & 0x3ff);
 }
 
 String partCategories(List<String> parts) {

pkgs/characters/tool/bin/generate_tables.dart

@@ -331,16 +331,28 @@ int $name(int lead, int tail) {
 ''';
   }
   var shift = chunkSize.bitLength - 1;
-  var indexVar = chunkSize < 1024 ? 'tail' : 'offset';
-  return '''
+  assert(shift <= 10);
+  if (shift < 10) {
+    return '''
 $preferInline
 int $name(int lead, int tail) {
-  var offset = (((0x3ff & lead) << 10) + (0x3ff & tail)) + ($startOffset << $shift);
-  var chunkStart = $startName.codeUnitAt(offset >> $shift);
-  var index = chunkStart + ($indexVar & ${chunkSize - 1});
-  return $dataName.codeUnitAt(index);
+  var offset = (tail >> $shift) + (lead << ${10 - shift});
+  tail &= ${chunkSize - 1};
+  var chunkStart = $startName.codeUnitAt($startOffset + offset);
+  return $dataName.codeUnitAt(chunkStart + tail);
+}
+''';
+  } else {
+    assert(shift == 10);
+    return '''
+$preferInline
+int $name(int lead, int tail) {
+  var chunkStart = $startName.codeUnitAt($startOffset + lead);
+  return $dataName.codeUnitAt(chunkStart + tail);
 }
 ''';
+  }
+  // Add code if shift > 10 ever becomes optimal for table size.
 }
 
 // -----------------------------------------------------------------------------

pkgs/characters/tool/src/string_literal_writer.dart

@@ -43,12 +43,12 @@ class StringLiteralWriter {
   /// Adds a single UTF-16 code unit.
   void add(int codeUnit) {
     // Always escape: `\n`, `\r`, `'`, `$` and `\`, plus anything the user wants.
-    if (_escape(codeUnit) ||
-        codeUnit == 0x24 ||
-        codeUnit == 0x27 ||
-        codeUnit == 0x5c ||
-        codeUnit == 0x0a ||
-        codeUnit == 0x0d) {
+    if (_escape(codeUnit) || // Anything the user wants encoded.
+        codeUnit == 0x24 /* $ */ ||
+        codeUnit == 0x27 /* ' */ ||
+        codeUnit == 0x5c /* \ */ ||
+        codeUnit == 0x0a /* \n */ ||
+        codeUnit == 0x0d /* \r */) {
       _writeEscape(codeUnit);
       return;
     }
@@ -59,6 +59,9 @@ class StringLiteralWriter {
     buffer.writeCharCode(codeUnit);
   }
 
+  /// Writes an escape for the [codeUnit].
+  ///
+  /// Is only called for characters that need escaping.
   void _writeEscape(int codeUnit) {
     var replacement = _escapeCache[codeUnit];
     if (replacement == null) {
@@ -83,9 +86,8 @@ class StringLiteralWriter {
           replacement = r'\$';
         } else if (codeUnit == "'".codeUnitAt(0)) {
           replacement = r"\'";
-        }
-        if (codeUnit == r''.codeUnitAt(0)) {
-          replacement = r'\';
+        } else if (codeUnit == r'\'.codeUnitAt(0)) {
+          replacement = r'\\';
         } else {
           replacement = r'\x' + codeUnit.toRadixString(16);
         }