Calculate text string length correctly for code points outside BMP (#132593) (#132598)

Strings parsed with the optimized UTF8 parsing have their length calculated during parsing. This length should be the same as the length if the string is parsed with the non-optimized path. Specifically, characters outside the basic multilingual plane require 2 chars per code point in the UTF16 encoding.

(cherry picked from commit fa6e905)

Author: parkertimmins

docs/changelog/132593.yaml

@@ -0,0 +1,5 @@
+pr: 132593
+summary: Strings outside BMP have 2 chars per code points
+area: Mapping
+type: bug
+issues: []

libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/json/ESUTF8StreamJsonParser.java

@@ -112,7 +112,8 @@ protected Text _finishAndReturnText() throws IOException {
                         return null;
                     }
                     ptr += bytesToSkip;
-                    ++stringLength;
+                    // Code points that require 4 bytes in UTF-8 will use 2 chars in UTF-16.
+                    stringLength += (bytesToSkip == 4 ? 2 : 1);
                 }
                 default -> {
                     return null;

libs/x-content/impl/src/test/java/org/elasticsearch/xcontent/provider/json/ESUTF8StreamJsonParserTests.java

@@ -76,6 +76,18 @@ public void testGetValueAsText() throws IOException {
             assertThat(parser.getValueAsString(), Matchers.equalTo("bår"));
         });
 
+        testParseJson("{\"foo\": \"\uD83D\uDE0A\"}", parser -> {
+            assertThat(parser.nextToken(), Matchers.equalTo(JsonToken.START_OBJECT));
+            assertThat(parser.nextFieldName(), Matchers.equalTo("foo"));
+            assertThat(parser.nextValue(), Matchers.equalTo(JsonToken.VALUE_STRING));
+
+            var text = parser.getValueAsText();
+            assertThat(text, Matchers.notNullValue());
+            var bytes = text.bytes();
+            assertTextRef(bytes, "\uD83D\uDE0A");
+            assertThat(text.stringLength(), Matchers.equalTo(2));
+        });
+
         testParseJson("{\"foo\": \"bår\"}", parser -> {
             assertThat(parser.nextToken(), Matchers.equalTo(JsonToken.START_OBJECT));
             assertThat(parser.nextFieldName(), Matchers.equalTo("foo"));
@@ -143,19 +155,37 @@ private record TestInput(String input, String result, boolean supportsOptimized)
         new TestInput("\\/", "/", true),
         new TestInput("\\\\", "\\", true) };
 
-    private int randomCodepoint(boolean includeAscii) {
+    private int randomCodepointIncludeAscii() {
         while (true) {
             char val = Character.toChars(randomInt(0xFFFF))[0];
-            if (val <= 0x7f && includeAscii == false) {
-                continue;
-            }
             if (val >= Character.MIN_SURROGATE && val <= Character.MAX_SURROGATE) {
                 continue;
             }
             return val;
         }
     }
 
+    private int randomCodepointIncludeOutsideBMP(int remainingLength) {
+        while (true) {
+            int codePoint = randomInt(0x10FFFF);
+            char[] val = Character.toChars(codePoint);
+            // Don't include ascii
+            if (val.length == 1 && val[0] <= 0x7F) {
+                continue;
+            }
+            boolean surrogate = val[0] >= Character.MIN_SURROGATE && val[0] <= Character.MAX_SURROGATE;
+            // Single surrogate is invalid
+            if (val.length == 1 && surrogate) {
+                continue;
+            }
+            // Not enough remaining space for a surrogate pair
+            if (remainingLength < 2 && surrogate) {
+                continue;
+            }
+            return codePoint;
+        }
+    }
+
     private TestInput buildRandomInput(int length) {
         StringBuilder input = new StringBuilder(length);
         StringBuilder result = new StringBuilder(length);
@@ -171,13 +201,14 @@ private TestInput buildRandomInput(int length) {
                         doesSupportOptimized = doesSupportOptimized && escape.supportsOptimized();
                     }
                     case 1 -> {
-                        int value = randomCodepoint(true);
+                        int value = randomCodepointIncludeAscii();
                         input.append(String.format(Locale.ENGLISH, "\\u%04x", value));
                         result.append(Character.toChars(value));
                         doesSupportOptimized = false;
                     }
                     default -> {
-                        var value = Character.toChars(randomCodepoint(false));
+                        var remainingLength = length - i;
+                        var value = Character.toChars(randomCodepointIncludeOutsideBMP(remainingLength));
                         input.append(value);
                         result.append(value);
                     }
@@ -222,7 +253,9 @@ public void testGetValueRandomized() throws IOException {
 
                 String currVal = inputs[i].result();
                 if (inputs[i].supportsOptimized()) {
-                    assertTextRef(parser.getValueAsText().bytes(), currVal);
+                    var text = parser.getValueAsText();
+                    assertTextRef(text.bytes(), currVal);
+                    assertThat(text.stringLength(), Matchers.equalTo(currVal.length()));
                 } else {
                     assertThat(parser.getValueAsText(), Matchers.nullValue());
                     assertThat(parser.getValueAsString(), Matchers.equalTo(currVal));