From a19422ce19d1ad9738dceca88e27d603576a9d42 Mon Sep 17 00:00:00 2001 From: Parker Timmins Date: Tue, 26 Aug 2025 18:41:55 -0500 Subject: [PATCH 1/3] Allow empty trailing field names in path of flattened object --- .../FlattenedFieldSyntheticWriterHelper.java | 4 +- .../flattened/FlattenedFieldMapperTests.java | 21 ++++++++ ...ttenedFieldSyntheticWriterHelperTests.java | 53 +++++++++++++++++++ .../scalar/string/ContainsEvaluator.java | 11 ++++ 4 files changed, 88 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldSyntheticWriterHelper.java b/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldSyntheticWriterHelper.java index 0cbddf3044c75..5493bcddb4a84 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldSyntheticWriterHelper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldSyntheticWriterHelper.java @@ -116,7 +116,9 @@ private KeyValue(final String value, final Prefix prefix, final String leaf) { KeyValue(final BytesRef keyValue) { this( - FlattenedFieldParser.extractKey(keyValue).utf8ToString().split(PATH_SEPARATOR_PATTERN), + // Splitting with a negative limit includes trailing empty strings. + // This is needed in case the provide path has trailing path separators. + FlattenedFieldParser.extractKey(keyValue).utf8ToString().split(PATH_SEPARATOR_PATTERN, -1), FlattenedFieldParser.extractValue(keyValue).utf8ToString() ); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapperTests.java index 7f695b1581fbf..0628ca47308ea 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapperTests.java @@ -960,6 +960,27 @@ public void testSyntheticSourceWithMatchesInNestedPath() throws IOException { {"field":{"a":{"b":{"c":"1"}},"b":{"b":{"d":"2"}}}}""")); } + public void testMultipleDotsInPath() throws IOException { + DocumentMapper mapper = createSytheticSourceMapperService( + mapping(b -> { b.startObject("field").field("type", "flattened").endObject(); }) + ).documentMapper(); + + var syntheticSource = syntheticSource(mapper, b -> { + b.startObject("field"); + { + b.startObject("."); + { + b.field(".", "bar"); + } + b.endObject(); + } + b.endObject(); + }); + // This behavior is weird to say the least. But this is the only reasonable way to interpret the meaning of the path `...` + assertThat(syntheticSource, equalTo(""" + {"field":{"":{"":{"":{"":"bar"}}}}}""")); + } + @Override protected boolean supportsCopyTo() { return false; diff --git a/server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldSyntheticWriterHelperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldSyntheticWriterHelperTests.java index 4ca5da1c42d40..2eabaef1b87f8 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldSyntheticWriterHelperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldSyntheticWriterHelperTests.java @@ -21,6 +21,7 @@ import java.nio.charset.StandardCharsets; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -246,6 +247,58 @@ public void testScalarObjectMismatchInNestedObject() throws IOException { assertEquals("{\"a\":{\"b\":{\"c\":\"10\",\"c.d\":\"20\"}}}", baos.toString(StandardCharsets.UTF_8)); } + public void testSingleDotPath() throws IOException { + // GIVEN + final SortedSetDocValues dv = mock(SortedSetDocValues.class); + final FlattenedFieldSyntheticWriterHelper writer = new FlattenedFieldSyntheticWriterHelper(new SortedSetSortedKeyedValues(dv)); + final ByteArrayOutputStream baos = new ByteArrayOutputStream(); + final XContentBuilder builder = new XContentBuilder(XContentType.JSON.xContent(), baos); + final List bytes = Stream.of("." + '\0' + "10") + .map(x -> x.getBytes(StandardCharsets.UTF_8)) + .toList(); + when(dv.getValueCount()).thenReturn(Long.valueOf(bytes.size())); + when(dv.docValueCount()).thenReturn(bytes.size()); + for (int i = 0; i < bytes.size(); i++) { + when(dv.nextOrd()).thenReturn((long) i); + when(dv.lookupOrd(ArgumentMatchers.eq((long) i))).thenReturn(new BytesRef(bytes.get(i), 0, bytes.get(i).length)); + } + + // WHEN + builder.startObject(); + writer.write(builder); + builder.endObject(); + builder.flush(); + + // THEN + assertEquals("{\"\":{\"\":\"10\"}}", baos.toString(StandardCharsets.UTF_8)); + } + + public void testTrailingDotsPath() throws IOException { + // GIVEN + final SortedSetDocValues dv = mock(SortedSetDocValues.class); + final FlattenedFieldSyntheticWriterHelper writer = new FlattenedFieldSyntheticWriterHelper(new SortedSetSortedKeyedValues(dv)); + final ByteArrayOutputStream baos = new ByteArrayOutputStream(); + final XContentBuilder builder = new XContentBuilder(XContentType.JSON.xContent(), baos); + final List bytes = Stream.of("cat.." + '\0' + "10") + .map(x -> x.getBytes(StandardCharsets.UTF_8)) + .toList(); + when(dv.getValueCount()).thenReturn(Long.valueOf(bytes.size())); + when(dv.docValueCount()).thenReturn(bytes.size()); + for (int i = 0; i < bytes.size(); i++) { + when(dv.nextOrd()).thenReturn((long) i); + when(dv.lookupOrd(ArgumentMatchers.eq((long) i))).thenReturn(new BytesRef(bytes.get(i), 0, bytes.get(i).length)); + } + + // WHEN + builder.startObject(); + writer.write(builder); + builder.endObject(); + builder.flush(); + + // THEN + assertEquals("{\"cat\":{\"\":{\"\":\"10\"}}}", baos.toString(StandardCharsets.UTF_8)); + } + private class SortedSetSortedKeyedValues implements FlattenedFieldSyntheticWriterHelper.SortedKeyedValues { private final SortedSetDocValues dv; private int seen = 0; diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/ContainsEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/ContainsEvaluator.java index cd4d40dabaec2..2a922687127d2 100644 --- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/ContainsEvaluator.java +++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/ContainsEvaluator.java @@ -8,6 +8,7 @@ import java.lang.Override; import java.lang.String; import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.RamUsageEstimator; import org.elasticsearch.compute.data.Block; import org.elasticsearch.compute.data.BooleanBlock; import org.elasticsearch.compute.data.BooleanVector; @@ -25,6 +26,8 @@ * This class is generated. Edit {@code EvaluatorImplementer} instead. */ public final class ContainsEvaluator implements EvalOperator.ExpressionEvaluator { + private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(ContainsEvaluator.class); + private final Source source; private final EvalOperator.ExpressionEvaluator str; @@ -60,6 +63,14 @@ public Block eval(Page page) { } } + @Override + public long baseRamBytesUsed() { + long baseRamBytesUsed = BASE_RAM_BYTES_USED; + baseRamBytesUsed += str.baseRamBytesUsed(); + baseRamBytesUsed += substr.baseRamBytesUsed(); + return baseRamBytesUsed; + } + public BooleanBlock eval(int positionCount, BytesRefBlock strBlock, BytesRefBlock substrBlock) { try(BooleanBlock.Builder result = driverContext.blockFactory().newBooleanBlockBuilder(positionCount)) { BytesRef strScratch = new BytesRef(); From bd57124da7b9faec3f59abb2f01dbd3e403c4043 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Wed, 27 Aug 2025 00:06:35 +0000 Subject: [PATCH 2/3] [CI] Auto commit changes from spotless --- .../FlattenedFieldSyntheticWriterHelperTests.java | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldSyntheticWriterHelperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldSyntheticWriterHelperTests.java index 2eabaef1b87f8..f59640194f8cd 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldSyntheticWriterHelperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldSyntheticWriterHelperTests.java @@ -253,9 +253,7 @@ public void testSingleDotPath() throws IOException { final FlattenedFieldSyntheticWriterHelper writer = new FlattenedFieldSyntheticWriterHelper(new SortedSetSortedKeyedValues(dv)); final ByteArrayOutputStream baos = new ByteArrayOutputStream(); final XContentBuilder builder = new XContentBuilder(XContentType.JSON.xContent(), baos); - final List bytes = Stream.of("." + '\0' + "10") - .map(x -> x.getBytes(StandardCharsets.UTF_8)) - .toList(); + final List bytes = Stream.of("." + '\0' + "10").map(x -> x.getBytes(StandardCharsets.UTF_8)).toList(); when(dv.getValueCount()).thenReturn(Long.valueOf(bytes.size())); when(dv.docValueCount()).thenReturn(bytes.size()); for (int i = 0; i < bytes.size(); i++) { @@ -279,9 +277,7 @@ public void testTrailingDotsPath() throws IOException { final FlattenedFieldSyntheticWriterHelper writer = new FlattenedFieldSyntheticWriterHelper(new SortedSetSortedKeyedValues(dv)); final ByteArrayOutputStream baos = new ByteArrayOutputStream(); final XContentBuilder builder = new XContentBuilder(XContentType.JSON.xContent(), baos); - final List bytes = Stream.of("cat.." + '\0' + "10") - .map(x -> x.getBytes(StandardCharsets.UTF_8)) - .toList(); + final List bytes = Stream.of("cat.." + '\0' + "10").map(x -> x.getBytes(StandardCharsets.UTF_8)).toList(); when(dv.getValueCount()).thenReturn(Long.valueOf(bytes.size())); when(dv.docValueCount()).thenReturn(bytes.size()); for (int i = 0; i < bytes.size(); i++) { From 05188c5c92b334e1eec196aadac81427c44ebc09 Mon Sep 17 00:00:00 2001 From: Parker Timmins Date: Tue, 26 Aug 2025 19:30:32 -0500 Subject: [PATCH 3/3] Update docs/changelog/133611.yaml --- docs/changelog/133611.yaml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 docs/changelog/133611.yaml diff --git a/docs/changelog/133611.yaml b/docs/changelog/133611.yaml new file mode 100644 index 0000000000000..9b86f75d9d276 --- /dev/null +++ b/docs/changelog/133611.yaml @@ -0,0 +1,6 @@ +pr: 133611 +summary: Allow trailing empty string field names in paths of flattened field +area: Mapping +type: bug +issues: + - 130139