Massively improved the performance

Author: matt-aitken

apps/webapp/app/utils/detectBadJsonStrings.ts

@@ -1,48 +1,49 @@
 export function detectBadJsonStrings(jsonString: string): boolean {
-  // Single regex with global flag to find all matches with their positions
-  const regex = /\\ud[89ab][0-9a-f]{2}|\\ud[cd][0-9a-f]{2}/g;
-  const matches: Array<{ index: number; isHigh: boolean }> = [];
+  // Fast path: skip everything if no \u
+  let idx = jsonString.indexOf("\\u");
+  if (idx === -1) return false;
 
-  let match;
-  while ((match = regex.exec(jsonString)) !== null) {
-    const isHigh =
-      match[0].startsWith("\\ud8") ||
-      match[0].startsWith("\\ud9") ||
-      match[0].startsWith("\\uda") ||
-      match[0].startsWith("\\udb");
-    matches.push({ index: match.index, isHigh });
-  }
-
-  if (matches.length === 0) {
-    return false; // No Unicode escapes found
-  }
-
-  // Check for incomplete pairs
-  const highSurrogates = new Set<number>();
-  const lowSurrogates = new Set<number>();
-
-  for (const { index, isHigh } of matches) {
-    if (isHigh) {
-      highSurrogates.add(index);
-    } else {
-      lowSurrogates.add(index);
-    }
-  }
-
-  // Check for unmatched surrogates
-  for (const highIndex of highSurrogates) {
-    const expectedLowIndex = highIndex + 6; // Length of high surrogate
-    if (!lowSurrogates.has(expectedLowIndex)) {
-      return true; // Incomplete high surrogate
-    }
-  }
-
-  for (const lowIndex of lowSurrogates) {
-    const expectedHighIndex = lowIndex - 6; // Length of low surrogate
-    if (!highSurrogates.has(expectedHighIndex)) {
-      return true; // Incomplete low surrogate
+  // Only check the area around each \u
+  while (idx !== -1 && idx < jsonString.length - 5) {
+    if (jsonString[idx + 1] === "u" && jsonString[idx + 2] === "d") {
+      const third = jsonString[idx + 3];
+      // High surrogate
+      if (
+        /[89ab]/.test(third) &&
+        /[0-9a-f]/.test(jsonString[idx + 4]) &&
+        /[0-9a-f]/.test(jsonString[idx + 5])
+      ) {
+        // Check for low surrogate after
+        if (
+          jsonString.substr(idx + 6, 2) !== "\\u" ||
+          jsonString[idx + 8] !== "d" ||
+          !/[cd]/.test(jsonString[idx + 9]) ||
+          !/[0-9a-f]/.test(jsonString[idx + 10]) ||
+          !/[0-9a-f]/.test(jsonString[idx + 11])
+        ) {
+          return true; // Incomplete high surrogate
+        }
+      }
+      // Low surrogate
+      if (
+        (third === "c" || third === "d") &&
+        /[0-9a-f]/.test(jsonString[idx + 4]) &&
+        /[0-9a-f]/.test(jsonString[idx + 5])
+      ) {
+        // Check for high surrogate before
+        if (
+          idx < 6 ||
+          jsonString.substr(idx - 6, 2) !== "\\u" ||
+          jsonString[idx - 4] !== "d" ||
+          !/[89ab]/.test(jsonString[idx - 3]) ||
+          !/[0-9a-f]/.test(jsonString[idx - 2]) ||
+          !/[0-9a-f]/.test(jsonString[idx - 1])
+        ) {
+          return true; // Incomplete low surrogate
+        }
+      }
     }
+    idx = jsonString.indexOf("\\u", idx + 1);
   }
-
   return false;
 }

apps/webapp/test/detectbadJsonStrings.test.ts

@@ -97,7 +97,7 @@ describe("detectBadJsonStrings", () => {
     expect(goodOverhead / iterations).toBeLessThan(0.01); // Less than 10 microseconds per call
 
     // Bad JSON can be slower due to regex matching, but still reasonable
-    expect(badOverhead / iterations).toBeLessThan(0.02); // Less than 20 microseconds per call
+    expect(badOverhead / iterations).toBeLessThan(0.01); // Less than 20 microseconds per call
 
     // Total overhead for 100k calls should be reasonable
     expect(goodOverhead).toBeLessThan(1000); // Less than 1 second for 100k calls
@@ -127,6 +127,52 @@ describe("detectBadJsonStrings", () => {
       expect(time / iterations).toBeLessThan(size / 1000); // Roughly linear scaling
     }
   });
+
+  it("should show significant performance improvement with quick rejection", () => {
+    const longText = `hello world `.repeat(1_000);
+    const goodJson = `{"title": "hello", "text": "${longText}"}`;
+    const badJson = `{"title": "hello\\ud835", "text": "${longText}"}`;
+    const noUnicodeJson = `{"title": "hello", "text": "${longText}"}`;
+
+    const iterations = 100_000;
+
+    // Warm up
+    for (let i = 0; i < 1000; i++) {
+      detectBadJsonStrings(goodJson);
+      detectBadJsonStrings(badJson);
+      detectBadJsonStrings(noUnicodeJson);
+    }
+
+    // Test strings with no Unicode escapes (99.9% case)
+    const noUnicodeStart = performance.now();
+    for (let i = 0; i < iterations; i++) {
+      detectBadJsonStrings(noUnicodeJson);
+    }
+    const noUnicodeTime = performance.now() - noUnicodeStart;
+
+    // Test strings with Unicode escapes (0.1% case)
+    const withUnicodeStart = performance.now();
+    for (let i = 0; i < iterations; i++) {
+      detectBadJsonStrings(badJson);
+    }
+    const withUnicodeTime = performance.now() - withUnicodeStart;
+
+    console.log(
+      `No Unicode escapes (${iterations} iterations): ${noUnicodeTime.toFixed(2)}ms (${(
+        noUnicodeTime / iterations
+      ).toFixed(4)}ms per call)`
+    );
+    console.log(
+      `With Unicode escapes (${iterations} iterations): ${withUnicodeTime.toFixed(2)}ms (${(
+        withUnicodeTime / iterations
+      ).toFixed(4)}ms per call)`
+    );
+    console.log(
+      `Performance ratio: ${(withUnicodeTime / noUnicodeTime).toFixed(
+        2
+      )}x slower for Unicode strings`
+    );
+  });
 });
 
 function processPacket(data: string): { data?: string; dataType?: string } {