Skip to content

Bug fix, last optimised value in ESUTF8StreamJsonParser kept old value. #135184

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 21 commits into from
Sep 29, 2025
Merged

Bug fix, last optimised value in ESUTF8StreamJsonParser kept old value. #135184

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
21 commits
Select commit Hold shift + click to select a range
5851594
Reset the last optimised value when stringEnd gets reset too.
gmarouli Sep 22, 2025
8552375
Update docs/changelog/135184.yaml
gmarouli Sep 22, 2025
add871b
Revert "Reset the last optimised value when stringEnd gets reset too."
gmarouli Sep 23, 2025
c9844d1
Use qualified import
gmarouli Sep 23, 2025
0048c84
Add unit test that captures the issue.
gmarouli Sep 23, 2025
57d4c8f
[CI] Update transport version definitions
Sep 23, 2025
3402bc6
Reset the last value properly
gmarouli Sep 23, 2025
4b71f8c
[CI] Update transport version definitions
Sep 23, 2025
222a251
Add comments
gmarouli Sep 23, 2025
3be8275
Update docs/changelog/135184.yaml
gmarouli Sep 24, 2025
34ddd56
Merge branch 'main' into bug-fix-utf8-optimiser
gmarouli Sep 24, 2025
482c33e
Add getText() also in ESCborParserTests
gmarouli Sep 24, 2025
964ef6d
Merge branch 'main' into bug-fix-utf8-optimiser
gmarouli Sep 25, 2025
9a39f67
Add randomised test that compares an optimised and a baseline parseAr…
gmarouli Sep 25, 2025
1a00f09
Revert adding to importing org.hamcrest.Matchers
gmarouli Sep 25, 2025
5caf1f6
Revert adding to importing org.hamcrest.Matchers and remove not neede…
gmarouli Sep 25, 2025
f3e75d1
Delete docs/changelog/135184.yaml
gmarouli Sep 25, 2025
8f5f178
Use `mapOrdered` to simplify the assertion
gmarouli Sep 29, 2025
3cf40d3
Add yaml test
gmarouli Sep 29, 2025
23f6a11
Make `text()` also call `optimisedText()` some times
gmarouli Sep 29, 2025
850c4ac
Merge branch 'main' into bug-fix-utf8-optimiser
gmarouli Sep 29, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions docs/changelog/135184.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
pr: 135184
summary: "Bug fix, last optimised value in `ESUTF8StreamJsonParser` kept old value"
area: Infra/Core
type: bug
issues: []
15 changes: 7 additions & 8 deletions ...t/impl/src/main/java/org/elasticsearch/xcontent/provider/json/ESUTF8StreamJsonParser.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -148,33 +148,32 @@ protected Text _finishAndReturnText() throws IOException {

@Override
public JsonToken nextToken() throws IOException {
maybeResetCurrentTokenState();
stringEnd = -1;
resetCurrentTokenState();
return super.nextToken();
}

@Override
public boolean nextFieldName(SerializableString str) throws IOException {
maybeResetCurrentTokenState();
stringEnd = -1;
resetCurrentTokenState();
return super.nextFieldName(str);
}

@Override
public String nextFieldName() throws IOException {
maybeResetCurrentTokenState();
stringEnd = -1;
resetCurrentTokenState();
return super.nextFieldName();
}

/**
* Resets the current token state before moving to the next.
*/
private void maybeResetCurrentTokenState() {
private void resetCurrentTokenState() {
if (_currToken == JsonToken.VALUE_STRING && _tokenIncomplete && stringEnd > 0) {
_inputPtr = stringEnd;
_tokenIncomplete = false;
lastOptimisedValue = null;
}
// When resetting the `stringEnd`, we need to reset the lastOptimised value.
stringEnd = -1;
lastOptimisedValue = null;
}
}
239 changes: 126 additions & 113 deletions ...l/src/test/java/org/elasticsearch/xcontent/provider/json/ESUTF8StreamJsonParserTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,124 +38,137 @@ private void assertTextRef(XContentString.UTF8Bytes textRef, String expectedValu
}

public void testGetValueAsText() throws IOException {
testParseJson("{\"foo\": \"bar\"}", parser -> {
// testParseJson("{\"foo\": \"bar\"}", parser -> {
// assertThat(parser.nextToken(), Matchers.equalTo(JsonToken.START_OBJECT));
// assertThat(parser.nextFieldName(), Matchers.equalTo("foo"));
// assertThat(parser.nextValue(), Matchers.equalTo(JsonToken.VALUE_STRING));
//
// var text = parser.getValueAsText();
// assertThat(text, Matchers.notNullValue());
//
// var bytes = text.bytes();
// assertThat(bytes.offset(), Matchers.equalTo(9));
// assertThat(bytes.offset() + bytes.length(), Matchers.equalTo(12));
// assertTextRef(bytes, "bar");
//
// assertThat(parser.getValueAsString(), Matchers.equalTo("bar"));
// assertThat(parser.getValueAsText(), Matchers.nullValue());
//
// assertThat(parser.nextToken(), Matchers.equalTo(JsonToken.END_OBJECT));
// });
//
// testParseJson("{\"foo\": [\"bar\\\"baz\\\"\", \"foobar\"]}", parser -> {
// assertThat(parser.nextToken(), Matchers.equalTo(JsonToken.START_OBJECT));
// assertThat(parser.nextFieldName(), Matchers.equalTo("foo"));
//
// assertThat(parser.nextValue(), Matchers.equalTo(JsonToken.START_ARRAY));
// assertThat(parser.nextValue(), Matchers.equalTo(JsonToken.VALUE_STRING));
//
// var firstText = parser.getValueAsText();
// assertThat(firstText, Matchers.notNullValue());
// assertTextRef(firstText.bytes(), "bar\"baz\"");
// // Retrieve the value for a second time to ensure the last value is available
// firstText = parser.getValueAsText();
// assertThat(firstText, Matchers.notNullValue());
// assertTextRef(firstText.bytes(), "bar\"baz\"");
//
// // Ensure values lastOptimisedValue is reset
// assertThat(parser.nextValue(), Matchers.equalTo(JsonToken.VALUE_STRING));
// var secondTest = parser.getValueAsText();
// assertThat(secondTest, Matchers.notNullValue());
// assertTextRef(secondTest.bytes(), "foobar");
// secondTest = parser.getValueAsText();
// assertThat(secondTest, Matchers.notNullValue());
// assertTextRef(secondTest.bytes(), "foobar");
// assertThat(parser.nextValue(), Matchers.equalTo(JsonToken.END_ARRAY));
// });
//
// testParseJson("{\"foo\": \"b\\u00e5r\"}", parser -> {
// assertThat(parser.nextToken(), Matchers.equalTo(JsonToken.START_OBJECT));
// assertThat(parser.nextFieldName(), Matchers.equalTo("foo"));
// assertThat(parser.nextValue(), Matchers.equalTo(JsonToken.VALUE_STRING));
//
// assertThat(parser.getValueAsText(), Matchers.nullValue());
// assertThat(parser.getValueAsString(), Matchers.equalTo("bår"));
// });
//
// testParseJson("{\"foo\": \"\uD83D\uDE0A\"}", parser -> {
// assertThat(parser.nextToken(), Matchers.equalTo(JsonToken.START_OBJECT));
// assertThat(parser.nextFieldName(), Matchers.equalTo("foo"));
// assertThat(parser.nextValue(), Matchers.equalTo(JsonToken.VALUE_STRING));
//
// var text = parser.getValueAsText();
// assertThat(text, Matchers.notNullValue());
// var bytes = text.bytes();
// assertTextRef(bytes, "\uD83D\uDE0A");
// assertThat(text.stringLength(), Matchers.equalTo(2));
// });
//
// testParseJson("{\"foo\": \"bår\"}", parser -> {
// assertThat(parser.nextToken(), Matchers.equalTo(JsonToken.START_OBJECT));
// assertThat(parser.nextFieldName(), Matchers.equalTo("foo"));
// assertThat(parser.nextValue(), Matchers.equalTo(JsonToken.VALUE_STRING));
//
// var text = parser.getValueAsText();
// assertThat(text, Matchers.notNullValue());
//
// var bytes = text.bytes();
// assertThat(bytes.offset(), Matchers.equalTo(9));
// assertThat(bytes.offset() + bytes.length(), Matchers.equalTo(13));
// assertTextRef(bytes, "bår");
//
// assertThat(parser.getValueAsString(), Matchers.equalTo("bår"));
//
// assertThat(parser.nextToken(), Matchers.equalTo(JsonToken.END_OBJECT));
// });
//
// testParseJson("{\"foo\": [\"lorem\", \"ipsum\", \"dolor\"]}", parser -> {
// assertThat(parser.nextToken(), Matchers.equalTo(JsonToken.START_OBJECT));
// assertThat(parser.nextFieldName(), Matchers.equalTo("foo"));
// assertThat(parser.nextValue(), Matchers.equalTo(JsonToken.START_ARRAY));
//
// assertThat(parser.nextValue(), Matchers.equalTo(JsonToken.VALUE_STRING));
// {
// var textRef = parser.getValueAsText().bytes();
// assertThat(textRef, Matchers.notNullValue());
// assertThat(textRef.offset(), Matchers.equalTo(10));
// assertThat(textRef.offset() + textRef.length(), Matchers.equalTo(15));
// assertTextRef(textRef, "lorem");
// }
//
// assertThat(parser.nextValue(), Matchers.equalTo(JsonToken.VALUE_STRING));
// {
// var textRef = parser.getValueAsText().bytes();
// assertThat(textRef, Matchers.notNullValue());
// assertThat(textRef.offset(), Matchers.equalTo(19));
// assertThat(textRef.offset() + textRef.length(), Matchers.equalTo(24));
// assertTextRef(textRef, "ipsum");
// }
//
// assertThat(parser.nextValue(), Matchers.equalTo(JsonToken.VALUE_STRING));
// {
// var textRef = parser.getValueAsText().bytes();
// assertThat(textRef, Matchers.notNullValue());
// assertThat(textRef.offset(), Matchers.equalTo(28));
// assertThat(textRef.offset() + textRef.length(), Matchers.equalTo(33));
// assertTextRef(textRef, "dolor");
// }
//
// assertThat(parser.nextToken(), Matchers.equalTo(JsonToken.END_ARRAY));
// assertThat(parser.nextToken(), Matchers.equalTo(JsonToken.END_OBJECT));
// });

testParseJson("{\"꧐꧌ꦍ\": {\"』〜〚。〥《〰〹〳〕\": [\"184.123.133.116\"]}}", parser -> {
assertThat(parser.nextToken(), Matchers.equalTo(JsonToken.START_OBJECT));
assertThat(parser.nextFieldName(), Matchers.equalTo("foo"));
assertThat(parser.nextValue(), Matchers.equalTo(JsonToken.VALUE_STRING));

var text = parser.getValueAsText();
assertThat(text, Matchers.notNullValue());

var bytes = text.bytes();
assertThat(bytes.offset(), Matchers.equalTo(9));
assertThat(bytes.offset() + bytes.length(), Matchers.equalTo(12));
assertTextRef(bytes, "bar");

assertThat(parser.getValueAsString(), Matchers.equalTo("bar"));
assertThat(parser.getValueAsText(), Matchers.nullValue());

assertThat(parser.nextToken(), Matchers.equalTo(JsonToken.END_OBJECT));
});

testParseJson("{\"foo\": [\"bar\\\"baz\\\"\", \"foobar\"]}", parser -> {
assertThat(parser.nextToken(), Matchers.equalTo(JsonToken.START_OBJECT));
assertThat(parser.nextFieldName(), Matchers.equalTo("foo"));

assertThat(parser.nextFieldName(), Matchers.equalTo("꧐꧌ꦍ"));
assertThat(parser.nextValue(), Matchers.equalTo(JsonToken.START_OBJECT));
assertThat(parser.nextFieldName(), Matchers.equalTo("』〜〚。〥《〰〹〳〕"));
assertThat(parser.nextValue(), Matchers.equalTo(JsonToken.START_ARRAY));
assertThat(parser.nextValue(), Matchers.equalTo(JsonToken.VALUE_STRING));

var firstText = parser.getValueAsText();
assertThat(firstText, Matchers.notNullValue());
assertTextRef(firstText.bytes(), "bar\"baz\"");
// Retrieve the value for a second time to ensure the last value is available
firstText = parser.getValueAsText();
assertThat(firstText, Matchers.notNullValue());
assertTextRef(firstText.bytes(), "bar\"baz\"");

// Ensure values lastOptimisedValue is reset
assertThat(parser.nextValue(), Matchers.equalTo(JsonToken.VALUE_STRING));
var secondTest = parser.getValueAsText();
assertThat(secondTest, Matchers.notNullValue());
assertTextRef(secondTest.bytes(), "foobar");
secondTest = parser.getValueAsText();
assertThat(secondTest, Matchers.notNullValue());
assertTextRef(secondTest.bytes(), "foobar");
assertThat(parser.nextValue(), Matchers.equalTo(JsonToken.END_ARRAY));
});

testParseJson("{\"foo\": \"b\\u00e5r\"}", parser -> {
assertThat(parser.nextToken(), Matchers.equalTo(JsonToken.START_OBJECT));
assertThat(parser.nextFieldName(), Matchers.equalTo("foo"));
assertThat(parser.nextValue(), Matchers.equalTo(JsonToken.VALUE_STRING));

assertThat(parser.getValueAsText(), Matchers.nullValue());
assertThat(parser.getValueAsString(), Matchers.equalTo("bår"));
});

testParseJson("{\"foo\": \"\uD83D\uDE0A\"}", parser -> {
assertThat(parser.nextToken(), Matchers.equalTo(JsonToken.START_OBJECT));
assertThat(parser.nextFieldName(), Matchers.equalTo("foo"));
assertThat(parser.nextValue(), Matchers.equalTo(JsonToken.VALUE_STRING));

var text = parser.getValueAsText();
assertThat(text, Matchers.notNullValue());
var bytes = text.bytes();
assertTextRef(bytes, "\uD83D\uDE0A");
assertThat(text.stringLength(), Matchers.equalTo(2));
});

testParseJson("{\"foo\": \"bår\"}", parser -> {
assertThat(parser.nextToken(), Matchers.equalTo(JsonToken.START_OBJECT));
assertThat(parser.nextFieldName(), Matchers.equalTo("foo"));
assertThat(parser.nextValue(), Matchers.equalTo(JsonToken.VALUE_STRING));

var text = parser.getValueAsText();
assertThat(text, Matchers.notNullValue());

var bytes = text.bytes();
assertThat(bytes.offset(), Matchers.equalTo(9));
assertThat(bytes.offset() + bytes.length(), Matchers.equalTo(13));
assertTextRef(bytes, "bår");

assertThat(parser.getValueAsString(), Matchers.equalTo("bår"));

assertThat(parser.nextToken(), Matchers.equalTo(JsonToken.END_OBJECT));
});

testParseJson("{\"foo\": [\"lorem\", \"ipsum\", \"dolor\"]}", parser -> {
assertThat(parser.nextToken(), Matchers.equalTo(JsonToken.START_OBJECT));
assertThat(parser.nextFieldName(), Matchers.equalTo("foo"));
assertThat(parser.nextValue(), Matchers.equalTo(JsonToken.START_ARRAY));

assertThat(parser.nextValue(), Matchers.equalTo(JsonToken.VALUE_STRING));
{
var textRef = parser.getValueAsText().bytes();
assertThat(textRef, Matchers.notNullValue());
assertThat(textRef.offset(), Matchers.equalTo(10));
assertThat(textRef.offset() + textRef.length(), Matchers.equalTo(15));
assertTextRef(textRef, "lorem");
}

assertThat(parser.nextValue(), Matchers.equalTo(JsonToken.VALUE_STRING));
{
var textRef = parser.getValueAsText().bytes();
assertThat(textRef, Matchers.notNullValue());
assertThat(textRef.offset(), Matchers.equalTo(19));
assertThat(textRef.offset() + textRef.length(), Matchers.equalTo(24));
assertTextRef(textRef, "ipsum");
}

assertThat(parser.nextValue(), Matchers.equalTo(JsonToken.VALUE_STRING));
{
var textRef = parser.getValueAsText().bytes();
assertThat(textRef, Matchers.notNullValue());
assertThat(textRef.offset(), Matchers.equalTo(28));
assertThat(textRef.offset() + textRef.length(), Matchers.equalTo(33));
assertTextRef(textRef, "dolor");
}

assertThat(parser.getValueAsString(), Matchers.equalTo("184.123.133.116"));
assertThat(parser.nextToken(), Matchers.equalTo(JsonToken.END_ARRAY));
assertThat(parser.nextToken(), Matchers.equalTo(JsonToken.END_OBJECT));
assertThat(parser.nextToken(), Matchers.equalTo(JsonToken.END_OBJECT));
});
}

Expand Down