From 0a264845a9b3657c09cd0a3ae50bfdcc6d0169b0 Mon Sep 17 00:00:00 2001 From: Jordan Powers Date: Fri, 25 Jul 2025 15:32:07 -0700 Subject: [PATCH 1/4] Add unicode test to IgnoredSourceFieldMapperTests --- .../mapper/IgnoredSourceFieldMapperTests.java | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java index 11e2305838705..ef41d20438858 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java @@ -10,12 +10,14 @@ package org.elasticsearch.index.mapper; import org.apache.lucene.index.DirectoryReader; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.CheckedConsumer; import org.elasticsearch.core.Nullable; import org.elasticsearch.search.lookup.SourceFilter; import org.elasticsearch.test.FieldMaskingReader; import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.json.JsonXContent; import org.hamcrest.Matchers; import org.junit.Before; @@ -123,6 +125,24 @@ public void testIgnoredString() throws IOException { ); } + public void testIgnoredStringFullUnicode() throws IOException { + String value = randomUnicodeOfCodepointLengthBetween(5, 20); + String fieldName = randomUnicodeOfCodepointLength(5); + + String expected = Strings.toString(JsonXContent.contentBuilder().startObject().field(fieldName, value).endObject()); + + assertEquals(expected, getSyntheticSourceWithFieldLimit(b -> b.field(fieldName, value))); + assertEquals( + expected, + getSyntheticSourceWithFieldLimit(new SourceFilter(new String[] { fieldName }, null), b -> b.field(fieldName, value)) + ); + assertEquals( + "{}", + getSyntheticSourceWithFieldLimit(new SourceFilter(null, new String[] { fieldName }), b -> b.field(fieldName, value)) + ); + + } + public void testIgnoredInt() throws IOException { int value = randomInt(); assertEquals("{\"my_value\":" + value + "}", getSyntheticSourceWithFieldLimit(b -> b.field("my_value", value))); From 4383d50292b5b39befe445c5c1e2260ce2714d93 Mon Sep 17 00:00:00 2001 From: Jordan Powers Date: Fri, 25 Jul 2025 15:32:56 -0700 Subject: [PATCH 2/4] Use proper byte length in IgnoredSourceFieldMapper#encode --- .../elasticsearch/index/mapper/IgnoredSourceFieldMapper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapper.java index d8d8200baac31..cd7f0d1c956dc 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapper.java @@ -170,7 +170,7 @@ static byte[] encode(NameValue values) { byte[] nameBytes = values.name.getBytes(StandardCharsets.UTF_8); byte[] bytes = new byte[4 + nameBytes.length + values.value.length]; - ByteUtils.writeIntLE(values.name.length() + PARENT_OFFSET_IN_NAME_OFFSET * values.parentOffset, bytes, 0); + ByteUtils.writeIntLE(nameBytes.length + PARENT_OFFSET_IN_NAME_OFFSET * values.parentOffset, bytes, 0); System.arraycopy(nameBytes, 0, bytes, 4, nameBytes.length); System.arraycopy(values.value.bytes, values.value.offset, bytes, 4 + nameBytes.length, values.value.length); return bytes; From d389317cc3964f0a758552c61de7677566351f6e Mon Sep 17 00:00:00 2001 From: Jordan Powers Date: Fri, 25 Jul 2025 15:45:32 -0700 Subject: [PATCH 3/4] Update docs/changelog/131950.yaml --- docs/changelog/131950.yaml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 docs/changelog/131950.yaml diff --git a/docs/changelog/131950.yaml b/docs/changelog/131950.yaml new file mode 100644 index 0000000000000..2df89d0f23b0d --- /dev/null +++ b/docs/changelog/131950.yaml @@ -0,0 +1,5 @@ +pr: 131950 +summary: Fix encoding of non-ascii field names in ignored source +area: Mapping +type: bug +issues: [] From c635256b98c7aa40c1ddb51bcf2dd024c1380edf Mon Sep 17 00:00:00 2001 From: Jordan Powers Date: Fri, 25 Jul 2025 16:14:18 -0700 Subject: [PATCH 4/4] Remove sourcefilter tests --- .../index/mapper/IgnoredSourceFieldMapperTests.java | 9 --------- 1 file changed, 9 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java index ef41d20438858..423811c9e22a2 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java @@ -132,15 +132,6 @@ public void testIgnoredStringFullUnicode() throws IOException { String expected = Strings.toString(JsonXContent.contentBuilder().startObject().field(fieldName, value).endObject()); assertEquals(expected, getSyntheticSourceWithFieldLimit(b -> b.field(fieldName, value))); - assertEquals( - expected, - getSyntheticSourceWithFieldLimit(new SourceFilter(new String[] { fieldName }, null), b -> b.field(fieldName, value)) - ); - assertEquals( - "{}", - getSyntheticSourceWithFieldLimit(new SourceFilter(null, new String[] { fieldName }), b -> b.field(fieldName, value)) - ); - } public void testIgnoredInt() throws IOException {