Optimizing category lookup, but without packing

Author: cometkim

README.md

@@ -215,7 +215,7 @@ Since [Hermes doesn't support the `Intl.Segmenter` API](https://github.com/faceb
 
 | Name                         | Unicode® | ESM? |   Size    | Size (min) | Size (min+gzip) | Size (min+br) | Size (min+zstd) |
 |------------------------------|----------|------|----------:|-----------:|----------------:|--------------:|----------------:|
-| `unicode-segmenter/grapheme` |   16.0.0 |   ✔️ |    17,150 |     12,602 |           5,344 |         3,982 |           5,053 |
+| `unicode-segmenter/grapheme` |   16.0.0 |   ✔️ |    16,823 |     12,532 |           5,309 |         3,942 |           4,999 |
 | `graphemer`                  |   15.0.0 |   ✖️ ️|   410,435 |     95,104 |          15,752 |        10,660 |          15,911 |
 | `grapheme-splitter`          |   10.0.0 |   ✖️ |   122,254 |     23,682 |           7,852 |         4,802 |           6,753 |
 | `@formatjs/intl-segmenter`*  |   15.0.0 |   ✖️ |   603,510 |    369,673 |          72,273 |        49,530 |          68,027 |
@@ -231,7 +231,7 @@ Since [Hermes doesn't support the `Intl.Segmenter` API](https://github.com/faceb
 
 | Name                         | Bytecode size | Bytecode size (gzip)* |
 |------------------------------|--------------:|----------------------:|
-| `unicode-segmenter/grapheme` |        26,551 |                13,901 |
+| `unicode-segmenter/grapheme` |        26,270 |                13,730 |
 | `graphemer`                  |       134,089 |                31,766 |
 | `grapheme-splitter`          |        63,946 |                19,162 |
 

src/grapheme.js

@@ -198,32 +198,32 @@ const
   SEG0_MIN = 128,
   /** 0x2FFF */
   SEG0_MAX = 12287,
-  /** (0x3000 - 0x80) >> 1 */
-  SEG1_OFF = 6080,
+  /** 0x3000 - 0x80 */
+  SEG1_OFF = 12160,
   /** 0xA000 */
   SEG1_MIN = 40960,
   /** 0xDFFF */
   SEG1_MAX = 57343,
-  /** SEG1_OFF + ((0xE000 - 0xA000) >> 1) */
-  SEG2_OFF = 14272,
+  /** SEG1_OFF + (0xE000 - 0xA000) */
+  SEG2_OFF = 28544,
   /** 0xFE00 */
   SEG2_MIN = 65024;
 
 /**
- * Segmented 4-bit packed lookup table for BMP code points.
+ * Segmented lookup table for BMP code points.
  *
  * Memory optimization: Skip regions that are almost 100% category 0 (Any):
  * - 0x3000-0x9FFF (CJK): 28,672 codepoints, only 12 non-Any -> inlined fast path
  * - 0xE000-0xFDFF (Private Use): 7,680 codepoints, only 1 non-Any -> inlined fast path
  *
  * Cache segments:
- * - Segment 0: 0x0080-0x2FFF (12,160 codepoints -> 6,080 bytes)
- * - Segment 1: 0xA000-0xDFFF (16,384 codepoints -> 8,192 bytes)
- * - Segment 2: 0xFE00-0xFFFF (512 codepoints -> 256 bytes)
+ * - Segment 0: 0x0080-0x2FFF (12,160 bytes)
+ * - Segment 1: 0xA000-0xDFFF (16,384 bytes)
+ * - Segment 2: 0xFE00-0xFFFF (512 bytes)
  *
- * Total: 14,528 bytes (~14KB)
+ * Total: 29,056 bytes (~28KB)
  */
-let bmpLookup = new Uint8Array(14528);
+let bmpLookup = new Uint8Array(29056);
 let bmpCursor = (() => {
   let cursor = 0;
   while (cursor < grapheme_ranges.length) {
@@ -236,14 +236,10 @@ let bmpCursor = (() => {
 
     for (let cp = start; cp <= end && cp <= BMP_MAX; cp++) {
       let idx = -1;
-      if (cp >= SEG0_MIN && cp <= SEG0_MAX) idx = (cp - SEG0_MIN) >> 1;
-      if (cp >= SEG1_MIN && cp <= SEG1_MAX) idx = SEG1_OFF + ((cp - SEG1_MIN) >> 1);
-      if (cp >= SEG2_MIN && cp <= BMP_MAX) idx = SEG2_OFF + ((cp - SEG2_MIN) >> 1);
-      if (idx >= 0) {
-        bmpLookup[idx] = cp & 1
-          ? (bmpLookup[idx] & 0x0F) | (cat << 4)
-          : (bmpLookup[idx] & 0xF0) | cat;
-      }
+      if (cp >= SEG0_MIN && cp <= SEG0_MAX) idx = cp - SEG0_MIN;
+      if (cp >= SEG1_MIN && cp <= SEG1_MAX) idx = SEG1_OFF + (cp - SEG1_MIN);
+      if (cp >= SEG2_MIN && cp <= BMP_MAX) idx = SEG2_OFF + (cp - SEG2_MIN);
+      if (idx >= 0) bmpLookup[idx] = cat;
     }
   }
   return cursor;
@@ -258,13 +254,13 @@ let bmpCursor = (() => {
  * @return {GraphemeCategoryNum}
  */
 function cat(cp) {
-  // Ordered pass by range: 
+  // Ordered pass by range:
   // 1. ASCII fast path
   // 2. Segment 0 cache
-  // 3. CJK fast path 
-  // 4. Segment 1 cache 
-  // 5. PrivateUse fast path 
-  // 6. Segment 2 cache 
+  // 3. CJK fast path
+  // 4. Segment 1 cache
+  // 5. PrivateUse fast path
+  // 6. Segment 2 cache
   // 7. Non-BMP binary search
 
   // ASCII fast path
@@ -274,13 +270,9 @@ function cat(cp) {
     if (cp === 13) return 1;
     return 2;
   }
-
-  let byte = 0, idx = -1;
   // Segment 0
   if (cp <= SEG0_MAX) {
-    idx = (cp - SEG0_MIN) >> 1;
-    byte = bmpLookup[idx];
-    return /** @type {GraphemeCategoryNum} */ (cp & 1 ? byte >> 4 : byte & 0x0F);
+    return /** @type {GraphemeCategoryNum} */ (bmpLookup[cp - SEG0_MIN]);
   }
   // CJK fast path
   if (cp < SEG1_MIN) {
@@ -294,22 +286,18 @@ function cat(cp) {
   }
   // Segment 1
   if (cp <= SEG1_MAX) {
-    idx = SEG1_OFF + ((cp - SEG1_MIN) >> 1);
-    byte = bmpLookup[idx];
-    return /** @type {GraphemeCategoryNum} */ (cp & 1 ? byte >> 4 : byte & 0x0F);
+    return /** @type {GraphemeCategoryNum} */ (bmpLookup[SEG1_OFF + (cp - SEG1_MIN)]);
   }
   // Private Use fast path
   if (cp < SEG2_MIN) {
     return cp === 0xFB1E ? 3 : 0;
   }
   // Segment 2
   if (cp <= BMP_MAX) {
-    idx = SEG2_OFF + ((cp - SEG2_MIN) >> 1);
-    byte = bmpLookup[idx];
-    return /** @type {GraphemeCategoryNum} */ (cp & 1 ? byte >> 4 : byte & 0x0F);
+    return /** @type {GraphemeCategoryNum} */ (bmpLookup[SEG2_OFF + (cp - SEG2_MIN)]);
   }
   // Non-BMP
-  idx = findUnicodeRangeIndex(cp, grapheme_ranges, bmpCursor);
+  let idx = findUnicodeRangeIndex(cp, grapheme_ranges, bmpCursor);
   return idx < 0 ? 0 : grapheme_ranges[idx][2];
 }