Fix: Use lazy encoding for utf-8 encoded string comparison

dev/src/order.ts

@@ -248,19 +248,62 @@ function compareVectors(left: ApiMapValue, right: ApiMapValue): number {
   return compareArrays(leftArray, rightArray);
 }
 
-function stringToUtf8Bytes(str: string): Uint8Array {
-  return new TextEncoder().encode(str);
-}
-
 /*!
  * Compare strings in UTF-8 encoded byte order
  * @private
  * @internal
  */
 export function compareUtf8Strings(left: string, right: string): number {
-  const leftBytes = stringToUtf8Bytes(left);
-  const rightBytes = stringToUtf8Bytes(right);
-  return compareBlobs(Buffer.from(leftBytes), Buffer.from(rightBytes));
+  let i = 0;
+  while (i < left.length && i < right.length) {
+    const leftCodePoint = left.codePointAt(i)!;
+    const rightCodePoint = right.codePointAt(i)!;
+
+    if (leftCodePoint !== rightCodePoint) {
+      if (leftCodePoint < 128 && rightCodePoint < 128) {
+        // ASCII comparison
+        return primitiveComparator(leftCodePoint, rightCodePoint);
+      } else {
+        // Lazy instantiate TextEncoder
+        const encoder = new TextEncoder();
+
+        // UTF-8 encode the character at index i for byte comparison.
+        const leftBytes = encoder.encode(getUtf8SafeSubstring(left, i));
+        const rightBytes = encoder.encode(getUtf8SafeSubstring(right, i));
+        const comp = compareBlobs(
+          Buffer.from(leftBytes),
+          Buffer.from(rightBytes)
+        );
+        if (comp !== 0) {
+          return comp;
+        } else {
+          // EXTREMELY RARE CASE: Code points differ, but their UTF-8 byte
+          // representations are identical. This can happen with malformed input
+          // (invalid surrogate pairs). The backend also actively prevents invalid
+          // surrogates as INVALID_ARGUMENT errors, so we almost never receive
+          // invalid strings from backend.
+          // Fallback to code point comparison for graceful handling.
+          return primitiveComparator(leftCodePoint, rightCodePoint);
+        }
+      }
+    }
+    // Increment by 2 for surrogate pairs, 1 otherwise
+    i += leftCodePoint > 0xffff ? 2 : 1;
+  }
+
+  // Compare lengths if all characters are equal
+  return primitiveComparator(left.length, right.length);
+}
+
+function getUtf8SafeSubstring(str: string, index: number): string {
+  const firstCodePoint = str.codePointAt(index)!;
+  if (firstCodePoint > 0xffff) {
+    // It's a surrogate pair, return the whole pair
+    return str.substring(index, index + 2);
+  } else {
+    // It's a single code point, return it
+    return str.substring(index, index + 1);
+  }
 }
 
 /*!

dev/system-test/firestore.ts

@@ -4086,6 +4086,20 @@ describe('Query class', () => {
     });
 
     describe('sort unicode strings', () => {
+      const expectedDocs = [
+        'b',
+        'a',
+        'h',
+        'i',
+        'c',
+        'f',
+        'e',
+        'd',
+        'g',
+        'k',
+        'j',
+      ];
+
       it('snapshot listener sorts unicode strings same as server', async () => {
         const collection = await testCollectionWithDocs({
           a: {value: 'Łukasiewicz'},
@@ -4095,10 +4109,13 @@ describe('Query class', () => {
           e: {value: 'Ｐ'},
           f: {value: '︒'},
           g: {value: '🐵'},
+          h: {value: '你好'},
+          i: {value: '你顥'},
+          j: {value: '😁'},
+          k: {value: '😀'},
         });
 
         const query = collection.orderBy('value');
-        const expectedDocs = ['b', 'a', 'c', 'f', 'e', 'd', 'g'];
 
         const getSnapshot = await query.get();
         expect(getSnapshot.docs.map(d => d.id)).to.deep.equal(expectedDocs);
@@ -4123,10 +4140,13 @@ describe('Query class', () => {
           e: {value: ['Ｐ']},
           f: {value: ['︒']},
           g: {value: ['🐵']},
+          h: {value: ['你好']},
+          i: {value: ['你顥']},
+          j: {value: ['😁']},
+          k: {value: ['😀']},
         });
 
         const query = collection.orderBy('value');
-        const expectedDocs = ['b', 'a', 'c', 'f', 'e', 'd', 'g'];
 
         const getSnapshot = await query.get();
         expect(getSnapshot.docs.map(d => d.id)).to.deep.equal(expectedDocs);
@@ -4151,10 +4171,13 @@ describe('Query class', () => {
           e: {value: {foo: 'Ｐ'}},
           f: {value: {foo: '︒'}},
           g: {value: {foo: '🐵'}},
+          h: {value: {foo: '你好'}},
+          i: {value: {foo: '你顥'}},
+          j: {value: {foo: '😁'}},
+          k: {value: {foo: '😀'}},
         });
 
         const query = collection.orderBy('value');
-        const expectedDocs = ['b', 'a', 'c', 'f', 'e', 'd', 'g'];
 
         const getSnapshot = await query.get();
         expect(getSnapshot.docs.map(d => d.id)).to.deep.equal(expectedDocs);
@@ -4179,10 +4202,13 @@ describe('Query class', () => {
           e: {value: {Ｐ: true}},
           f: {value: {'︒': true}},
           g: {value: {'🐵': true}},
+          h: {value: {你好: true}},
+          i: {value: {你顥: true}},
+          j: {value: {'😁': true}},
+          k: {value: {'😀': true}},
         });
 
         const query = collection.orderBy('value');
-        const expectedDocs = ['b', 'a', 'c', 'f', 'e', 'd', 'g'];
 
         const getSnapshot = await query.get();
         expect(getSnapshot.docs.map(d => d.id)).to.deep.equal(expectedDocs);
@@ -4207,17 +4233,25 @@ describe('Query class', () => {
           Ｐ: {value: true},
           '︒': {value: true},
           '🐵': {value: true},
+          你好: {value: true},
+          你顥: {value: true},
+          '😁': {value: true},
+          '😀': {value: true},
         });
 
         const query = collection.orderBy(FieldPath.documentId());
         const expectedDocs = [
           'Sierpiński',
           'Łukasiewicz',
+          '你好',
+          '你顥',
           '岩澤',
           '︒',
           'Ｐ',
           '🄟',
           '🐵',
+          '😀',
+          '😁',
         ];
 
         const getSnapshot = await query.get();

dev/test/order.ts

@@ -284,3 +284,219 @@ describe('Order', () => {
     }
   });
 });
+
+class StringPair {
+  constructor(
+    readonly s1: string,
+    readonly s2: string
+  ) {}
+}
+
+class StringPairGenerator {
+  constructor(private stringGenerator: StringGenerator) {}
+
+  next(): StringPair {
+    const prefix = this.stringGenerator.next();
+    const s1 = prefix + this.stringGenerator.next();
+    const s2 = prefix + this.stringGenerator.next();
+    return new StringPair(s1, s2);
+  }
+}
+
+class StringGenerator {
+  private static readonly DEFAULT_SURROGATE_PAIR_PROBABILITY = 0.33;
+  private static readonly DEFAULT_MAX_LENGTH = 20;
+
+  private readonly rnd: Random;
+  private readonly surrogatePairProbability: number;
+  private readonly maxLength: number;
+
+  constructor(seed: number);
+  constructor(rnd: Random, surrogatePairProbability: number, maxLength: number);
+  constructor(
+    seedOrRnd: number | Random,
+    surrogatePairProbability?: number,
+    maxLength?: number
+  ) {
+    if (typeof seedOrRnd === 'number') {
+      this.rnd = new Random(seedOrRnd);
+      this.surrogatePairProbability =
+        StringGenerator.DEFAULT_SURROGATE_PAIR_PROBABILITY;
+      this.maxLength = StringGenerator.DEFAULT_MAX_LENGTH;
+    } else {
+      this.rnd = seedOrRnd;
+      this.surrogatePairProbability = StringGenerator.validateProbability(
+        surrogatePairProbability!
+      );
+      this.maxLength = StringGenerator.validateLength(maxLength!);
+    }
+  }
+
+  private static validateProbability(probability: number): number {
+    if (!Number.isFinite(probability)) {
+      throw new Error(
+        `invalid surrogate pair probability: ${probability} (must be between 0.0 and 1.0, inclusive)`
+      );
+    } else if (probability < 0.0) {
+      throw new Error(
+        `invalid surrogate pair probability: ${probability} (must be greater than or equal to zero)`
+      );
+    } else if (probability > 1.0) {
+      throw new Error(
+        `invalid surrogate pair probability: ${probability} (must be less than or equal to 1)`
+      );
+    }
+    return probability;
+  }
+
+  private static validateLength(length: number): number {
+    if (length < 0) {
+      throw new Error(
+        `invalid maximum string length: ${length} (must be greater than or equal to zero)`
+      );
+    }
+    return length;
+  }
+
+  next(): string {
+    const length = this.rnd.nextInt(this.maxLength + 1);
+    const sb = new StringBuilder();
+    while (sb.length() < length) {
+      const codePoint = this.nextCodePoint();
+      sb.appendCodePoint(codePoint);
+    }
+    return sb.toString();
+  }
+
+  private isNextSurrogatePair(): boolean {
+    return StringGenerator.nextBoolean(this.rnd, this.surrogatePairProbability);
+  }
+
+  private static nextBoolean(rnd: Random, probability: number): boolean {
+    if (probability === 0.0) {
+      return false;
+    } else if (probability === 1.0) {
+      return true;
+    } else {
+      return rnd.nextFloat() < probability;
+    }
+  }
+
+  private nextCodePoint(): number {
+    if (this.isNextSurrogatePair()) {
+      return this.nextSurrogateCodePoint();
+    } else {
+      return this.nextNonSurrogateCodePoint();
+    }
+  }
+
+  private nextSurrogateCodePoint(): number {
+    const highSurrogateMin = 0xd800;
+    const highSurrogateMax = 0xdbff;
+    const lowSurrogateMin = 0xdc00;
+    const lowSurrogateMax = 0xdfff;
+
+    const highSurrogate = this.nextCodePointRange(
+      highSurrogateMin,
+      highSurrogateMax
+    );
+    const lowSurrogate = this.nextCodePointRange(
+      lowSurrogateMin,
+      lowSurrogateMax
+    );
+
+    return (highSurrogate - 0xd800) * 0x400 + (lowSurrogate - 0xdc00) + 0x10000;
+  }
+
+  private nextNonSurrogateCodePoint(): number {
+    let codePoint;
+    do {
+      codePoint = this.nextCodePointRange(0, 0xffff); // BMP range
+    } while (codePoint >= 0xd800 && codePoint <= 0xdfff); // Exclude surrogate range
+
+    return codePoint;
+  }
+
+  private nextCodePointRange(min: number, max: number): number {
+    const rangeSize = max - min + 1;
+    const offset = this.rnd.nextInt(rangeSize);
+    return min + offset;
+  }
+}
+
+class Random {
+  private seed: number;
+
+  constructor(seed: number) {
+    this.seed = seed;
+  }
+
+  nextInt(max: number): number {
+    this.seed = (this.seed * 9301 + 49297) % 233280;
+    const rnd = this.seed / 233280;
+    return Math.floor(rnd * max);
+  }
+
+  nextFloat(): number {
+    this.seed = (this.seed * 9301 + 49297) % 233280;
+    return this.seed / 233280;
+  }
+}
+
+class StringBuilder {
+  private buffer: string[] = [];
+
+  append(str: string): StringBuilder {
+    this.buffer.push(str);
+    return this;
+  }
+
+  appendCodePoint(codePoint: number): StringBuilder {
+    this.buffer.push(String.fromCodePoint(codePoint));
+    return this;
+  }
+
+  toString(): string {
+    return this.buffer.join('');
+  }
+
+  length(): number {
+    return this.buffer.join('').length;
+  }
+}
+
+describe('CompareUtf8Strings', () => {
+  it('compareUtf8Strings should return correct results', () => {
+    const errors = [];
+    const seed = Math.floor(Math.random() * Number.MAX_SAFE_INTEGER);
+    let passCount = 0;
+    const stringGenerator = new StringGenerator(new Random(seed), 0.33, 20);
+    const stringPairGenerator = new StringPairGenerator(stringGenerator);
+
+    for (let i = 0; i < 1000000 && errors.length < 10; i++) {
+      const {s1, s2} = stringPairGenerator.next();
+
+      const actual = order.compareUtf8Strings(s1, s2);
+      const expected = Buffer.from(s1, 'utf8').compare(Buffer.from(s2, 'utf8'));
+
+      if (actual === expected) {
+        passCount++;
+      } else {
+        errors.push(
+          `compareUtf8Strings(s1="${s1}", s2="${s2}") returned ${actual}, ` +
+            `but expected ${expected} (i=${i}, s1.length=${s1.length}, s2.length=${s2.length})`
+        );
+      }
+    }
+
+    if (errors.length > 0) {
+      console.error(
+        `${errors.length} test cases failed, ${passCount} test cases passed, seed=${seed};`
+      );
+      errors.forEach((error, index) =>
+        console.error(`errors[${index}]: ${error}`)
+      );
+      throw new Error('Test failed');
+    }
+  }).timeout(20000);
+});