From 8fa6388b81e61dae22e9e8f90df4ac6927c12ce2 Mon Sep 17 00:00:00 2001 From: Oleksandr Kolomiiets Date: Tue, 11 Feb 2025 09:45:37 -0800 Subject: [PATCH 1/4] Use FallbackSyntheticSourceBlockLoader for number fields --- .../mapper/FallbackSyntheticSourceBlockLoaderBenchmark.java | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 benchmarks/src/main/java/org/elasticsearch/benchmark/index/mapper/FallbackSyntheticSourceBlockLoaderBenchmark.java diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/index/mapper/FallbackSyntheticSourceBlockLoaderBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/index/mapper/FallbackSyntheticSourceBlockLoaderBenchmark.java new file mode 100644 index 0000000000000..ac6ac86ef65f8 --- /dev/null +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/index/mapper/FallbackSyntheticSourceBlockLoaderBenchmark.java @@ -0,0 +1,4 @@ +package org.elasticsearch.benchmark.index.mapper; + +public class FallbackSyntheticSourceBlockLoaderBenchmark { +} From ac960b9010239543a0f7abe4a41faa3020de9448 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Tue, 11 Feb 2025 17:56:13 +0000 Subject: [PATCH 2/4] [CI] Auto commit changes from spotless --- .../mapper/FallbackSyntheticSourceBlockLoaderBenchmark.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/index/mapper/FallbackSyntheticSourceBlockLoaderBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/index/mapper/FallbackSyntheticSourceBlockLoaderBenchmark.java index ac6ac86ef65f8..88c5ea65060b2 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/index/mapper/FallbackSyntheticSourceBlockLoaderBenchmark.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/index/mapper/FallbackSyntheticSourceBlockLoaderBenchmark.java @@ -1,4 +1,3 @@ package org.elasticsearch.benchmark.index.mapper; -public class FallbackSyntheticSourceBlockLoaderBenchmark { -} +public class FallbackSyntheticSourceBlockLoaderBenchmark {} From 3191be57e18add495c0b1e52f5f2c59ac0c2ff54 Mon Sep 17 00:00:00 2001 From: Oleksandr Kolomiiets Date: Wed, 12 Feb 2025 11:15:19 -0800 Subject: [PATCH 3/4] address feedback --- .../mapper/FallbackSyntheticSourceBlockLoaderBenchmark.java | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 benchmarks/src/main/java/org/elasticsearch/benchmark/index/mapper/FallbackSyntheticSourceBlockLoaderBenchmark.java diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/index/mapper/FallbackSyntheticSourceBlockLoaderBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/index/mapper/FallbackSyntheticSourceBlockLoaderBenchmark.java deleted file mode 100644 index 88c5ea65060b2..0000000000000 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/index/mapper/FallbackSyntheticSourceBlockLoaderBenchmark.java +++ /dev/null @@ -1,3 +0,0 @@ -package org.elasticsearch.benchmark.index.mapper; - -public class FallbackSyntheticSourceBlockLoaderBenchmark {} From 1847aca8482b87b9927179cc46f6eddc1dcd1c26 Mon Sep 17 00:00:00 2001 From: Oleksandr Kolomiiets Date: Thu, 13 Feb 2025 11:21:49 -0800 Subject: [PATCH 4/4] Fix handling of object arrays FallbackSyntheticSourceBlockLoader --- .../FallbackSyntheticSourceBlockLoader.java | 44 +++++++++++++++---- .../index/mapper/BlockLoaderTestCase.java | 25 ++++++++--- 2 files changed, 56 insertions(+), 13 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/FallbackSyntheticSourceBlockLoader.java b/server/src/main/java/org/elasticsearch/index/mapper/FallbackSyntheticSourceBlockLoader.java index 28ea37ef73e33..64f4d1bec04c7 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/FallbackSyntheticSourceBlockLoader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/FallbackSyntheticSourceBlockLoader.java @@ -23,6 +23,7 @@ import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.Stack; /** * Block loader for fields that use fallback synthetic source implementation. @@ -191,18 +192,45 @@ private void parseFieldFromParent(IgnoredSourceFieldMapper.NameValue nameValue, .createParser(filterParserConfig, nameValue.value().bytes, nameValue.value().offset + 1, nameValue.value().length - 1) ) { parser.nextToken(); - var fieldNameInParser = new StringBuilder(nameValue.name()); - while (true) { - if (parser.currentToken() == XContentParser.Token.FIELD_NAME) { - fieldNameInParser.append('.').append(parser.currentName()); - if (fieldNameInParser.toString().equals(fieldName)) { - parser.nextToken(); - break; + var fieldNames = new Stack() { + { + push(nameValue.name()); + } + }; + + while (parser.currentToken() != null) { + // We are descending into an object/array hierarchy of arbitrary depth + // until we find the field that we need. + while (true) { + if (parser.currentToken() == XContentParser.Token.FIELD_NAME) { + fieldNames.push(parser.currentName()); + var nameInParser = String.join(".", fieldNames); + if (nameInParser.equals(fieldName)) { + parser.nextToken(); + break; + } + } else { + assert parser.currentToken() == XContentParser.Token.START_OBJECT + || parser.currentToken() == XContentParser.Token.START_ARRAY; } + + parser.nextToken(); } + parseWithReader(parser, blockValues); parser.nextToken(); + + // We are coming back up in object/array hierarchy. + // If arrays are present we will explore all array items by going back down again. + while (parser.currentToken() == XContentParser.Token.END_OBJECT + || parser.currentToken() == XContentParser.Token.END_ARRAY) { + // When exiting an object arrays we'll see END_OBJECT followed by END_ARRAY, but we only need to pop the object name + // once. + if (parser.currentToken() == XContentParser.Token.END_OBJECT) { + fieldNames.pop(); + } + parser.nextToken(); + } } - parseWithReader(parser, blockValues); } } diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/BlockLoaderTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/BlockLoaderTestCase.java index a7595cf52297b..ab6fd109ed375 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/BlockLoaderTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/BlockLoaderTestCase.java @@ -19,6 +19,7 @@ import org.elasticsearch.logsdb.datageneration.DataGeneratorSpecification; import org.elasticsearch.logsdb.datageneration.DocumentGenerator; import org.elasticsearch.logsdb.datageneration.FieldType; +import org.elasticsearch.logsdb.datageneration.Mapping; import org.elasticsearch.logsdb.datageneration.MappingGenerator; import org.elasticsearch.logsdb.datageneration.Template; import org.elasticsearch.logsdb.datageneration.datasource.DataSourceHandler; @@ -72,9 +73,13 @@ public DataSourceResponse.ObjectMappingParametersGenerator handle( public void testBlockLoader() throws IOException { var template = new Template(Map.of(fieldName, new Template.Leaf(fieldName, fieldType))); - runTest(template, fieldName); + var syntheticSource = randomBoolean(); + var mapping = mappingGenerator.generate(template); + + runTest(template, mapping, syntheticSource, fieldName); } + @SuppressWarnings("unchecked") public void testBlockLoaderForFieldInObject() throws IOException { int depth = randomIntBetween(0, 3); @@ -94,14 +99,24 @@ public void testBlockLoaderForFieldInObject() throws IOException { fullFieldName.append('.').append(fieldName); currentLevel.put(fieldName, new Template.Leaf(fieldName, fieldType)); var template = new Template(top); - runTest(template, fullFieldName.toString()); - } - private void runTest(Template template, String fieldName) throws IOException { + var syntheticSource = randomBoolean(); + var mapping = mappingGenerator.generate(template); + + if (syntheticSource && randomBoolean()) { + // force fallback synthetic source in the hierarchy + var docMapping = (Map) mapping.raw().get("_doc"); + var topLevelMapping = (Map) ((Map) docMapping.get("properties")).get("top"); + topLevelMapping.put("synthetic_source_keep", "all"); + } + + runTest(template, mapping, syntheticSource, fullFieldName.toString()); + } + + private void runTest(Template template, Mapping mapping, boolean syntheticSource, String fieldName) throws IOException { var mappingXContent = XContentBuilder.builder(XContentType.JSON.xContent()).map(mapping.raw()); - var syntheticSource = randomBoolean(); var mapperService = syntheticSource ? createSytheticSourceMapperService(mappingXContent) : createMapperService(mappingXContent); var document = documentGenerator.generate(template, mapping);