From 77277c2b1dc542ecfa1a0d68aa79671b065f8292 Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Fri, 27 Sep 2024 17:02:50 +0300 Subject: [PATCH 01/21] Add object param for keeping synthetic source --- .../indices.create/20_synthetic_source.yml | 2 +- .../21_synthetic_source_stored.yml | 20 +++--- .../index/mapper/DocumentParser.java | 40 +++++------- .../index/mapper/NestedObjectMapper.java | 29 ++++++--- .../index/mapper/ObjectMapper.java | 61 +++++++++++-------- .../index/mapper/PassThroughObjectMapper.java | 16 ++++- .../index/mapper/RootObjectMapper.java | 10 +-- .../index/mapper/DocumentParserTests.java | 2 +- .../FieldAliasMapperValidationTests.java | 2 +- .../index/mapper/FieldTypeLookupTests.java | 2 + .../mapper/IgnoredSourceFieldMapperTests.java | 32 +++++----- .../index/mapper/MappingLookupTests.java | 2 +- .../index/mapper/NestedObjectMapperTests.java | 4 +- .../index/mapper/ObjectMapperTests.java | 12 ++-- .../mapper/PassThroughObjectMapperTests.java | 2 + .../index/mapper/MapperTestCase.java | 6 +- .../traces-otel@mappings.yaml | 2 +- ...cument_level_security_synthetic_source.yml | 2 +- 18 files changed, 136 insertions(+), 110 deletions(-) diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml index 41d9fcc30a880..80481fa51ee28 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml @@ -912,7 +912,7 @@ subobjects auto: id: type: keyword stored: - store_array_source: true + synthetic_source_keep: arrays properties: span: properties: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/21_synthetic_source_stored.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/21_synthetic_source_stored.yml index 7d7be765631e5..f80af6aa145ee 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/21_synthetic_source_stored.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/21_synthetic_source_stored.yml @@ -25,7 +25,7 @@ object param - object array: id: type: keyword stored: - store_array_source: true + synthetic_source_keep: arrays properties: span: properties: @@ -77,10 +77,10 @@ object param - object array within array: mode: synthetic properties: stored: - store_array_source: true + synthetic_source_keep: arrays properties: path: - store_array_source: true + synthetic_source_keep: arrays properties: to: properties: @@ -120,7 +120,7 @@ object param - no object array: mode: synthetic properties: stored: - store_array_source: true + synthetic_source_keep: arrays properties: span: properties: @@ -164,7 +164,7 @@ object param - field ordering in object array: a: type: keyword b: - store_array_source: true + synthetic_source_keep: arrays properties: aa: type: keyword @@ -173,7 +173,7 @@ object param - field ordering in object array: c: type: keyword d: - store_array_source: true + synthetic_source_keep: arrays properties: aa: type: keyword @@ -215,7 +215,7 @@ object param - nested object array next to other fields: b: properties: c: - store_array_source: true + synthetic_source_keep: arrays properties: aa: type: keyword @@ -272,7 +272,7 @@ object param - nested object with stored array: type: nested nested_array_stored: type: nested - store_array_source: true + synthetic_source_keep: arrays - do: bulk: @@ -322,7 +322,7 @@ index param - nested array within array: to: properties: some: - store_array_source: true + synthetic_source_keep: arrays properties: id: type: integer @@ -369,7 +369,7 @@ stored field under object with store_array_source: name: type: keyword obj: - store_array_source: true + synthetic_source_keep: arrays properties: foo: type: keyword diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java index 7f9b59d427656..4d098779c5083 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java @@ -429,20 +429,21 @@ static void parseObjectOrNested(DocumentParserContext context) throws IOExceptio throwOnConcreteValue(context.parent(), currentFieldName, context); } + if (context.canAddIgnoredField() && getSourceKeepMode(context, context.parent().sourceKeepMode()) == Mapper.SourceKeepMode.ALL) { + context = context.addIgnoredFieldFromContext( + new IgnoredSourceFieldMapper.NameValue( + context.parent().fullPath(), + context.parent().fullPath().lastIndexOf(context.parent().leafName()), + null, + context.doc() + ) + ); + token = context.parser().currentToken(); + parser = context.parser(); + } + if (context.parent().isNested()) { // Handle a nested object that doesn't contain an array. Arrays are handled in #parseNonDynamicArray. - if (context.parent().storeArraySource() && context.canAddIgnoredField()) { - context = context.addIgnoredFieldFromContext( - new IgnoredSourceFieldMapper.NameValue( - context.parent().fullPath(), - context.parent().fullPath().lastIndexOf(context.parent().leafName()), - null, - context.doc() - ) - ); - token = context.parser().currentToken(); - parser = context.parser(); - } context = context.createNestedContext((NestedObjectMapper) context.parent()); } @@ -809,9 +810,8 @@ private static void parseNonDynamicArray( // Check if we need to record the array source. This only applies to synthetic source. if (context.canAddIgnoredField()) { boolean objectRequiresStoringSource = mapper instanceof ObjectMapper objectMapper - && (objectMapper.storeArraySource() - || (context.sourceKeepModeFromIndexSettings() == Mapper.SourceKeepMode.ARRAYS - && objectMapper instanceof NestedObjectMapper == false) + && (objectMapper instanceof NestedObjectMapper == false + && (getSourceKeepMode(context, objectMapper.sourceKeepMode()) != Mapper.SourceKeepMode.NONE) || objectMapper.dynamic == ObjectMapper.Dynamic.RUNTIME); boolean fieldWithFallbackSyntheticSource = mapper instanceof FieldMapper fieldMapper && fieldMapper.syntheticSourceMode() == FieldMapper.SyntheticSourceMode.FALLBACK; @@ -1129,15 +1129,7 @@ protected SyntheticSourceSupport syntheticSourceSupport() { private static class NoOpObjectMapper extends ObjectMapper { NoOpObjectMapper(String name, String fullPath) { - super( - name, - fullPath, - Explicit.IMPLICIT_TRUE, - Optional.empty(), - Explicit.IMPLICIT_FALSE, - Dynamic.RUNTIME, - Collections.emptyMap() - ); + super(name, fullPath, Explicit.IMPLICIT_TRUE, Optional.empty(), Optional.empty(), Dynamic.RUNTIME, Collections.emptyMap()); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/NestedObjectMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/NestedObjectMapper.java index fc5f28dd51c9d..5661f19b2d3b9 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/NestedObjectMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/NestedObjectMapper.java @@ -98,6 +98,17 @@ public NestedObjectMapper build(MapperBuilderContext context) { } else { nestedTypePath = fullPath; } + if (sourceKeepMode.orElse(SourceKeepMode.NONE) == SourceKeepMode.ARRAYS) { + throw new MapperException( + "parameter [ " + + Mapper.SYNTHETIC_SOURCE_KEEP_PARAM + + " ] can't be set to [" + + SourceKeepMode.ARRAYS + + "] for nested object [" + + fullPath + + "]" + ); + } final Query nestedTypeFilter = NestedPathFieldMapper.filter(indexCreatedVersion, nestedTypePath); NestedMapperBuilderContext nestedContext = new NestedMapperBuilderContext( context.buildFullName(leafName()), @@ -115,7 +126,7 @@ public NestedObjectMapper build(MapperBuilderContext context) { buildMappers(nestedContext), enabled, dynamic, - storeArraySource, + sourceKeepMode, includeInParent, includeInRoot, parentTypeFilter, @@ -213,7 +224,7 @@ public MapperBuilderContext createChildContext(String name, Dynamic dynamic) { Map mappers, Explicit enabled, ObjectMapper.Dynamic dynamic, - Explicit storeArraySource, + Optional sourceKeepMode, Explicit includeInParent, Explicit includeInRoot, Query parentTypeFilter, @@ -222,7 +233,7 @@ public MapperBuilderContext createChildContext(String name, Dynamic dynamic) { Function bitsetProducer, IndexSettings indexSettings ) { - super(name, fullPath, enabled, Optional.empty(), storeArraySource, dynamic, mappers); + super(name, fullPath, enabled, Optional.empty(), sourceKeepMode, dynamic, mappers); this.parentTypeFilter = parentTypeFilter; this.nestedTypePath = nestedTypePath; this.nestedTypeFilter = nestedTypeFilter; @@ -283,7 +294,7 @@ NestedObjectMapper withoutMappers() { Map.of(), enabled, dynamic, - storeArraySource, + sourceKeepMode, includeInParent, includeInRoot, parentTypeFilter, @@ -310,8 +321,8 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (isEnabled() != Defaults.ENABLED) { builder.field("enabled", enabled.value()); } - if (storeArraySource != Defaults.STORE_ARRAY_SOURCE) { - builder.field(STORE_ARRAY_SOURCE_PARAM, storeArraySource.value()); + if (sourceKeepMode.isPresent()) { + builder.field("synthetic_source_keep", sourceKeepMode.get()); } serializeMappers(builder, params); return builder.endObject(); @@ -359,7 +370,7 @@ public ObjectMapper merge(Mapper mergeWith, MapperMergeContext parentMergeContex mergeResult.mappers(), mergeResult.enabled(), mergeResult.dynamic(), - mergeResult.trackArraySource(), + mergeResult.sourceKeepMode(), incInParent, incInRoot, parentTypeFilter, @@ -393,8 +404,8 @@ protected MapperMergeContext createChildContext(MapperMergeContext mapperMergeCo @Override public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() { - if (storeArraySource()) { - // IgnoredSourceFieldMapper integration takes care of writing the source for nested objects that enabled store_array_source. + if (sourceKeepMode.orElse(SourceKeepMode.NONE) == SourceKeepMode.ALL) { + // IgnoredSourceFieldMapper integration takes care of writing the source for the nested object. return SourceLoader.SyntheticFieldLoader.NOTHING; } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java index b9b611d8c62f9..6e5c8299be658 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java @@ -89,7 +89,6 @@ public String toString() { public static class Defaults { public static final boolean ENABLED = true; public static final Optional SUBOBJECTS = Optional.empty(); - public static final Explicit STORE_ARRAY_SOURCE = Explicit.IMPLICIT_FALSE; public static final Dynamic DYNAMIC = Dynamic.TRUE; } @@ -127,7 +126,7 @@ static Dynamic getRootDynamic(MappingLookup mappingLookup) { public static class Builder extends Mapper.Builder { protected Optional subobjects; protected Explicit enabled = Explicit.IMPLICIT_TRUE; - protected Explicit storeArraySource = Defaults.STORE_ARRAY_SOURCE; + protected Optional sourceKeepMode = Optional.empty(); protected Dynamic dynamic; protected final List mappersBuilders = new ArrayList<>(); @@ -141,8 +140,8 @@ public Builder enabled(boolean enabled) { return this; } - public Builder storeArraySource(boolean value) { - this.storeArraySource = Explicit.explicitBoolean(value); + public Builder sourceKeepMode(SourceKeepMode sourceKeepMode) { + this.sourceKeepMode = Optional.of(sourceKeepMode); return this; } @@ -288,7 +287,7 @@ public ObjectMapper build(MapperBuilderContext context) { context.buildFullName(leafName()), enabled, subobjects, - storeArraySource, + sourceKeepMode, dynamic, buildMappers(context.createChildContext(leafName(), dynamic)) ); @@ -350,7 +349,10 @@ protected static boolean parseObjectOrDocumentTypeProperties( builder.enabled(XContentMapValues.nodeBooleanValue(fieldNode, fieldName + ".enabled")); return true; } else if (fieldName.equals(STORE_ARRAY_SOURCE_PARAM)) { - builder.storeArraySource(XContentMapValues.nodeBooleanValue(fieldNode, fieldName + ".store_array_source")); + builder.sourceKeepMode(SourceKeepMode.ARRAYS); + return true; + } else if (fieldName.equals(Mapper.SYNTHETIC_SOURCE_KEEP_PARAM)) { + builder.sourceKeepMode(SourceKeepMode.from(fieldNode.toString())); return true; } else if (fieldName.equals("properties")) { if (fieldNode instanceof Collection && ((Collection) fieldNode).isEmpty()) { @@ -477,7 +479,7 @@ private static void validateFieldName(String fieldName, IndexVersion indexCreate protected final Explicit enabled; protected final Optional subobjects; - protected final Explicit storeArraySource; + protected final Optional sourceKeepMode; protected final Dynamic dynamic; protected final Map mappers; @@ -487,7 +489,7 @@ private static void validateFieldName(String fieldName, IndexVersion indexCreate String fullPath, Explicit enabled, Optional subobjects, - Explicit storeArraySource, + Optional sourceKeepMode, Dynamic dynamic, Map mappers ) { @@ -497,7 +499,7 @@ private static void validateFieldName(String fieldName, IndexVersion indexCreate this.fullPath = internFieldName(fullPath); this.enabled = enabled; this.subobjects = subobjects; - this.storeArraySource = storeArraySource; + this.sourceKeepMode = sourceKeepMode; this.dynamic = dynamic; if (mappers == null) { this.mappers = Map.of(); @@ -525,7 +527,7 @@ public Builder newBuilder(IndexVersion indexVersionCreated) { * This is typically used in the context of a mapper merge when there's not enough budget to add the entire object. */ ObjectMapper withoutMappers() { - return new ObjectMapper(leafName(), fullPath, enabled, subobjects, storeArraySource, dynamic, Map.of()); + return new ObjectMapper(leafName(), fullPath, enabled, subobjects, sourceKeepMode, dynamic, Map.of()); } @Override @@ -563,8 +565,8 @@ public final Subobjects subobjects() { return subobjects.orElse(Subobjects.ENABLED); } - public final boolean storeArraySource() { - return storeArraySource.value(); + public final Optional sourceKeepMode() { + return sourceKeepMode; } @Override @@ -593,7 +595,7 @@ public ObjectMapper merge(Mapper mergeWith, MapperMergeContext parentMergeContex fullPath, mergeResult.enabled, mergeResult.subObjects, - mergeResult.trackArraySource, + mergeResult.sourceKeepMode, mergeResult.dynamic, mergeResult.mappers ); @@ -602,7 +604,7 @@ public ObjectMapper merge(Mapper mergeWith, MapperMergeContext parentMergeContex protected record MergeResult( Explicit enabled, Optional subObjects, - Explicit trackArraySource, + Optional sourceKeepMode, Dynamic dynamic, Map mappers ) { @@ -636,26 +638,31 @@ static MergeResult build(ObjectMapper existing, ObjectMapper mergeWithObject, Ma } else { subObjects = existing.subobjects; } - final Explicit trackArraySource; - if (mergeWithObject.storeArraySource.explicit()) { + final Optional sourceKeepMode; + if (mergeWithObject.sourceKeepMode.isPresent()) { if (reason == MergeReason.INDEX_TEMPLATE) { - trackArraySource = mergeWithObject.storeArraySource; - } else if (existing.storeArraySource != mergeWithObject.storeArraySource) { + sourceKeepMode = mergeWithObject.sourceKeepMode; + } else if (existing.sourceKeepMode != mergeWithObject.sourceKeepMode) { throw new MapperException( - "the [store_array_source] parameter can't be updated for the object mapping [" + existing.fullPath() + "]" + "the [ " + + Mapper.SYNTHETIC_SOURCE_KEEP_PARAM + + " ] parameter can't be updated for the object mapping [" + + existing.fullPath() + + "]" ); } else { - trackArraySource = existing.storeArraySource; + sourceKeepMode = existing.sourceKeepMode; } } else { - trackArraySource = existing.storeArraySource; + sourceKeepMode = existing.sourceKeepMode; } + MapperMergeContext objectMergeContext = existing.createChildContext(parentMergeContext, existing.leafName()); Map mergedMappers = buildMergedMappers(existing, mergeWithObject, objectMergeContext, subObjects); return new MergeResult( enabled, subObjects, - trackArraySource, + sourceKeepMode, mergeWithObject.dynamic != null ? mergeWithObject.dynamic : existing.dynamic, mergedMappers ); @@ -780,7 +787,7 @@ private void asFlattenedFieldMappers( flattenedMappers.add( path.pathAsText("").isEmpty() ? this - : new ObjectMapper(path.pathAsText(leafName()), fullPath, enabled, subobjects, storeArraySource, dynamic, mappers) + : new ObjectMapper(path.pathAsText(leafName()), fullPath, enabled, subobjects, sourceKeepMode, dynamic, mappers) ); return; } @@ -807,8 +814,8 @@ Optional checkFlattenable(MapperBuilderContext context) { + ")" ); } - if (storeArraySource()) { - return Optional.of("the value of [store_array_source] is [true]"); + if (sourceKeepMode.isPresent()) { + return Optional.of("the value of [" + Mapper.SYNTHETIC_SOURCE_KEEP_PARAM + "] is [ " + sourceKeepMode.get() + " ]"); } if (isEnabled() == false) { return Optional.of("the value of [enabled] is [false]"); @@ -840,8 +847,8 @@ void toXContent(XContentBuilder builder, Params params, ToXContent custom) throw if (subobjects.isPresent()) { builder.field("subobjects", subobjects.get().printedValue); } - if (storeArraySource != Defaults.STORE_ARRAY_SOURCE) { - builder.field(STORE_ARRAY_SOURCE_PARAM, storeArraySource.value()); + if (sourceKeepMode.isPresent()) { + builder.field("synthetic_source_keep", sourceKeepMode.get()); } if (custom != null) { custom.toXContent(builder, params); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/PassThroughObjectMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/PassThroughObjectMapper.java index 9ef36b99a57c5..80f845d626a2f 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/PassThroughObjectMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/PassThroughObjectMapper.java @@ -82,6 +82,7 @@ public PassThroughObjectMapper build(MapperBuilderContext context) { leafName(), context.buildFullName(leafName()), enabled, + sourceKeepMode, dynamic, buildMappers(context.createChildContext(leafName(), timeSeriesDimensionSubFields.value(), dynamic)), timeSeriesDimensionSubFields, @@ -99,13 +100,14 @@ public PassThroughObjectMapper build(MapperBuilderContext context) { String name, String fullPath, Explicit enabled, + Optional sourceKeepMode, Dynamic dynamic, Map mappers, Explicit timeSeriesDimensionSubFields, int priority ) { // Subobjects are not currently supported. - super(name, fullPath, enabled, Optional.of(Subobjects.DISABLED), Explicit.IMPLICIT_FALSE, dynamic, mappers); + super(name, fullPath, enabled, Optional.of(Subobjects.DISABLED), sourceKeepMode, dynamic, mappers); this.timeSeriesDimensionSubFields = timeSeriesDimensionSubFields; this.priority = priority; if (priority < 0) { @@ -115,7 +117,16 @@ public PassThroughObjectMapper build(MapperBuilderContext context) { @Override PassThroughObjectMapper withoutMappers() { - return new PassThroughObjectMapper(leafName(), fullPath(), enabled, dynamic, Map.of(), timeSeriesDimensionSubFields, priority); + return new PassThroughObjectMapper( + leafName(), + fullPath(), + enabled, + sourceKeepMode, + dynamic, + Map.of(), + timeSeriesDimensionSubFields, + priority + ); } @Override @@ -158,6 +169,7 @@ public PassThroughObjectMapper merge(Mapper mergeWith, MapperMergeContext parent leafName(), fullPath(), mergeResult.enabled(), + mergeResult.sourceKeepMode(), mergeResult.dynamic(), mergeResult.mappers(), containsDimensions, diff --git a/server/src/main/java/org/elasticsearch/index/mapper/RootObjectMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/RootObjectMapper.java index 878f9c92fa552..db3604d29b22d 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/RootObjectMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/RootObjectMapper.java @@ -113,7 +113,7 @@ public RootObjectMapper build(MapperBuilderContext context) { leafName(), enabled, subobjects, - storeArraySource, + sourceKeepMode, dynamic, buildMappers(context.createChildContext(null, dynamic)), new HashMap<>(runtimeFields), @@ -135,7 +135,7 @@ public RootObjectMapper build(MapperBuilderContext context) { String name, Explicit enabled, Optional subobjects, - Explicit trackArraySource, + Optional sourceKeepMode, Dynamic dynamic, Map mappers, Map runtimeFields, @@ -144,7 +144,7 @@ public RootObjectMapper build(MapperBuilderContext context) { Explicit dateDetection, Explicit numericDetection ) { - super(name, name, enabled, subobjects, trackArraySource, dynamic, mappers); + super(name, name, enabled, subobjects, sourceKeepMode, dynamic, mappers); this.runtimeFields = runtimeFields; this.dynamicTemplates = dynamicTemplates; this.dynamicDateTimeFormatters = dynamicDateTimeFormatters; @@ -166,7 +166,7 @@ RootObjectMapper withoutMappers() { leafName(), enabled, subobjects, - storeArraySource, + sourceKeepMode, dynamic, Map.of(), Map.of(), @@ -282,7 +282,7 @@ public RootObjectMapper merge(Mapper mergeWith, MapperMergeContext parentMergeCo leafName(), mergeResult.enabled(), mergeResult.subObjects(), - mergeResult.trackArraySource(), + mergeResult.sourceKeepMode(), mergeResult.dynamic(), mergeResult.mappers(), Map.copyOf(runtimeFields), diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java index 93f546eb288b9..54f58ce116a8b 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java @@ -2319,7 +2319,7 @@ public void testSubobjectsAutoFlattened() throws Exception { b.startObject("complex.attribute").field("type", "flattened").endObject(); b.startObject("path").field("type", "object"); { - b.field("store_array_source", "true").field("subobjects", "auto"); + b.field("synthetic_source_keep", "arrays").field("subobjects", "auto"); b.startObject("properties"); { b.startObject("nested.attribute").field("type", "keyword").endObject(); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/FieldAliasMapperValidationTests.java b/server/src/test/java/org/elasticsearch/index/mapper/FieldAliasMapperValidationTests.java index d48c5550631cd..e385177b87147 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/FieldAliasMapperValidationTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/FieldAliasMapperValidationTests.java @@ -179,7 +179,7 @@ private static ObjectMapper createObjectMapper(String name) { name, Explicit.IMPLICIT_TRUE, Optional.empty(), - Explicit.IMPLICIT_FALSE, + Optional.empty(), ObjectMapper.Dynamic.FALSE, emptyMap() ); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/FieldTypeLookupTests.java b/server/src/test/java/org/elasticsearch/index/mapper/FieldTypeLookupTests.java index 18c4f393bc696..ae793bc3b329e 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/FieldTypeLookupTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/FieldTypeLookupTests.java @@ -20,6 +20,7 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import static java.util.Collections.emptyList; @@ -433,6 +434,7 @@ private PassThroughObjectMapper createPassThroughMapper(String name, Map { b.startObject("path"); { - b.field("type", "object").field("store_array_source", true); + b.field("type", "object").field("synthetic_source_keep", "arrays"); b.startObject("properties"); { - b.startObject("to").field("type", "object").field("store_array_source", true); + b.startObject("to").field("type", "object").field("synthetic_source_keep", "arrays"); { b.startObject("properties"); { @@ -893,7 +893,7 @@ public void testObjectArrayAndValue() throws IOException { { b.startObject("stored"); { - b.field("type", "object").field("store_array_source", true); + b.field("type", "object").field("synthetic_source_keep", "arrays"); b.startObject("properties").startObject("leaf").field("type", "integer").endObject().endObject(); } b.endObject(); @@ -977,7 +977,7 @@ public void testStoredArrayWithinHigherLevelArray() throws IOException { b.field("type", "object"); b.startObject("properties"); { - b.startObject("to").field("type", "object").field("store_array_source", true); + b.startObject("to").field("type", "object").field("synthetic_source_keep", "arrays"); { b.startObject("properties"); { @@ -1056,7 +1056,7 @@ public void testFallbackFieldWithinHigherLevelArray() throws IOException { public void testFieldOrdering() throws IOException { DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> { b.startObject("A").field("type", "integer").endObject(); - b.startObject("B").field("type", "object").field("store_array_source", true); + b.startObject("B").field("type", "object").field("synthetic_source_keep", "arrays"); { b.startObject("properties"); { @@ -1067,7 +1067,7 @@ public void testFieldOrdering() throws IOException { } b.endObject(); b.startObject("C").field("type", "integer").endObject(); - b.startObject("D").field("type", "object").field("store_array_source", true); + b.startObject("D").field("type", "object").field("synthetic_source_keep", "arrays"); { b.startObject("properties"); { @@ -1105,7 +1105,7 @@ public void testNestedObjectWithField() throws IOException { DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> { b.startObject("path").field("type", "nested"); { - b.field("store_array_source", true); + b.field("synthetic_source_keep", "all"); b.startObject("properties"); { b.startObject("foo").field("type", "keyword").endObject(); @@ -1127,7 +1127,7 @@ public void testNestedObjectWithArray() throws IOException { DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> { b.startObject("path").field("type", "nested"); { - b.field("store_array_source", true); + b.field("synthetic_source_keep", "all"); b.startObject("properties"); { b.startObject("foo").field("type", "keyword").endObject(); @@ -1160,7 +1160,7 @@ public void testNestedSubobjectWithField() throws IOException { b.startObject("int_value").field("type", "integer").endObject(); b.startObject("to").field("type", "nested"); { - b.field("store_array_source", true); + b.field("synthetic_source_keep", "all"); b.startObject("properties"); { b.startObject("foo").field("type", "keyword").endObject(); @@ -1201,7 +1201,7 @@ public void testNestedSubobjectWithArray() throws IOException { b.startObject("int_value").field("type", "integer").endObject(); b.startObject("to").field("type", "nested"); { - b.field("store_array_source", true); + b.field("synthetic_source_keep", "all"); b.startObject("properties"); { b.startObject("foo").field("type", "keyword").endObject(); @@ -1241,7 +1241,7 @@ public void testNestedSubobjectWithArray() throws IOException { public void testNestedObjectIncludeInRoot() throws IOException { DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> { - b.startObject("path").field("type", "nested").field("store_array_source", true).field("include_in_root", true); + b.startObject("path").field("type", "nested").field("synthetic_source_keep", "all").field("include_in_root", true); { b.startObject("properties"); { @@ -1502,7 +1502,7 @@ public void testStoredNestedSubObjectWithNameOverlappingParentName() throws IOEx b.startObject("path"); b.startObject("properties"); { - b.startObject("at").field("type", "nested").field("store_array_source", "true").endObject(); + b.startObject("at").field("type", "nested").field("synthetic_source_keep", "all").endObject(); } b.endObject(); b.endObject(); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/MappingLookupTests.java b/server/src/test/java/org/elasticsearch/index/mapper/MappingLookupTests.java index 1381df07789b5..fd44e68df19a8 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/MappingLookupTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/MappingLookupTests.java @@ -84,7 +84,7 @@ public void testSubfieldOverride() { "object", Explicit.EXPLICIT_TRUE, Optional.empty(), - Explicit.IMPLICIT_FALSE, + Optional.empty(), ObjectMapper.Dynamic.TRUE, Collections.singletonMap("object.subfield", fieldMapper) ); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/NestedObjectMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/NestedObjectMapperTests.java index 0a954115e77f6..be1469e25f24d 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/NestedObjectMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/NestedObjectMapperTests.java @@ -1571,14 +1571,14 @@ public void testNestedMapperFilters() throws Exception { public void testStoreArraySourceinSyntheticSourceMode() throws IOException { DocumentMapper mapper = createDocumentMapper(syntheticSourceMapping(b -> { - b.startObject("o").field("type", "nested").field(ObjectMapper.STORE_ARRAY_SOURCE_PARAM, true).endObject(); + b.startObject("o").field("type", "nested").field("synthetic_source_keep", "all").endObject(); })); assertNotNull(mapper.mapping().getRoot().getMapper("o")); } public void testStoreArraySourceNoopInNonSyntheticSourceMode() throws IOException { DocumentMapper mapper = createDocumentMapper(mapping(b -> { - b.startObject("o").field("type", "nested").field(ObjectMapper.STORE_ARRAY_SOURCE_PARAM, true).endObject(); + b.startObject("o").field("type", "nested").field("synthetic_source_keep", "all").endObject(); })); assertNotNull(mapper.mapping().getRoot().getMapper("o")); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperTests.java index 4bc91b793d049..92ee11eb80b94 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperTests.java @@ -167,7 +167,7 @@ public void testMergeEnabledForIndexTemplates() throws IOException { assertNotNull(objectMapper); assertFalse(objectMapper.isEnabled()); assertEquals(ObjectMapper.Subobjects.ENABLED, objectMapper.subobjects()); - assertFalse(objectMapper.storeArraySource()); + assertTrue(objectMapper.sourceKeepMode().isEmpty()); // Setting 'enabled' to true is allowed, and updates the mapping. update = Strings.toString( @@ -189,7 +189,7 @@ public void testMergeEnabledForIndexTemplates() throws IOException { assertNotNull(objectMapper); assertTrue(objectMapper.isEnabled()); assertEquals(ObjectMapper.Subobjects.AUTO, objectMapper.subobjects()); - assertTrue(objectMapper.storeArraySource()); + assertEquals(Mapper.SourceKeepMode.ARRAYS, objectMapper.sourceKeepMode().orElse(Mapper.SourceKeepMode.NONE)); } public void testFieldReplacementForIndexTemplates() throws IOException { @@ -548,7 +548,7 @@ public void testSubobjectsAutoWithInnerNonFlattenableObject() throws IOException { b.startObject("time"); { - b.field(ObjectMapper.STORE_ARRAY_SOURCE_PARAM, true); + b.field("synthetic_source_keep", "arrays"); b.startObject("properties"); { b.startObject("max").field("type", "long").endObject(); @@ -715,14 +715,14 @@ public void testSyntheticSourceDocValuesFieldWithout() throws IOException { public void testStoreArraySourceinSyntheticSourceMode() throws IOException { DocumentMapper mapper = createDocumentMapper(syntheticSourceMapping(b -> { - b.startObject("o").field("type", "object").field(ObjectMapper.STORE_ARRAY_SOURCE_PARAM, true).endObject(); + b.startObject("o").field("type", "object").field("synthetic_source_keep", "arrays").endObject(); })); assertNotNull(mapper.mapping().getRoot().getMapper("o")); } public void testStoreArraySourceNoopInNonSyntheticSourceMode() throws IOException { DocumentMapper mapper = createDocumentMapper(mapping(b -> { - b.startObject("o").field("type", "object").field(ObjectMapper.STORE_ARRAY_SOURCE_PARAM, true).endObject(); + b.startObject("o").field("type", "object").field("synthetic_source_keep", "arrays").endObject(); })); assertNotNull(mapper.mapping().getRoot().getMapper("o")); } @@ -764,7 +764,7 @@ private ObjectMapper createObjectMapperWithAllParametersSet(CheckedConsumer { b.startObject("field"); - b.field(Mapper.SYNTHETIC_SOURCE_KEEP_PARAM, "none"); + b.field("synthetic_source_keep", "none"); example.mapping().accept(b); b.endObject(); })); @@ -1548,7 +1548,7 @@ public void testSyntheticSourceKeepAll() throws IOException { SyntheticSourceExample example = syntheticSourceSupportForKeepTests(shouldUseIgnoreMalformed()).example(1); DocumentMapper mapperAll = createDocumentMapper(syntheticSourceMapping(b -> { b.startObject("field"); - b.field(Mapper.SYNTHETIC_SOURCE_KEEP_PARAM, "all"); + b.field("synthetic_source_keep", "all"); example.mapping().accept(b); b.endObject(); })); @@ -1565,7 +1565,7 @@ public void testSyntheticSourceKeepArrays() throws IOException { SyntheticSourceExample example = syntheticSourceSupportForKeepTests(shouldUseIgnoreMalformed()).example(1); DocumentMapper mapperAll = createDocumentMapper(syntheticSourceMapping(b -> { b.startObject("field"); - b.field(Mapper.SYNTHETIC_SOURCE_KEEP_PARAM, randomFrom("arrays", "all")); // Both options keep array source. + b.field("synthetic_source_keep", randomFrom("arrays", "all")); // Both options keep array source. example.mapping().accept(b); b.endObject(); })); diff --git a/x-pack/plugin/otel-data/src/main/resources/component-templates/traces-otel@mappings.yaml b/x-pack/plugin/otel-data/src/main/resources/component-templates/traces-otel@mappings.yaml index a4c62efeed7a4..0e77bc208eed4 100644 --- a/x-pack/plugin/otel-data/src/main/resources/component-templates/traces-otel@mappings.yaml +++ b/x-pack/plugin/otel-data/src/main/resources/component-templates/traces-otel@mappings.yaml @@ -44,7 +44,7 @@ template: dropped_events_count: type: long links: - store_array_source: true + synthetic_source_keep: arrays properties: trace_id: type: keyword diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/security/authz_api_keys/40_document_level_security_synthetic_source.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/security/authz_api_keys/40_document_level_security_synthetic_source.yml index 769b9d848ba35..52abe0a3d83d7 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/security/authz_api_keys/40_document_level_security_synthetic_source.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/security/authz_api_keys/40_document_level_security_synthetic_source.yml @@ -186,7 +186,7 @@ Filter on object with stored source: type: keyword obj: type: object - store_array_source: true + synthetic_source_keep: arrays properties: secret: type: keyword From 468d250a87871775615841816d4933f244ad9455 Mon Sep 17 00:00:00 2001 From: Kostas Krikellas <131142368+kkrik-es@users.noreply.github.com> Date: Fri, 27 Sep 2024 17:04:25 +0300 Subject: [PATCH 02/21] Update docs/changelog/113690.yaml --- docs/changelog/113690.yaml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 docs/changelog/113690.yaml diff --git a/docs/changelog/113690.yaml b/docs/changelog/113690.yaml new file mode 100644 index 0000000000000..bd5f1245f471e --- /dev/null +++ b/docs/changelog/113690.yaml @@ -0,0 +1,5 @@ +pr: 113690 +summary: Add object param for keeping synthetic source +area: Mapping +type: enhancement +issues: [] From 3b30147422b4cbbd90980f7c89f3e5959bdf4f0d Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Sat, 28 Sep 2024 13:23:58 +0300 Subject: [PATCH 03/21] fix merging --- .../test/indices.create/21_synthetic_source_stored.yml | 2 +- .../org/elasticsearch/index/mapper/DocumentParser.java | 7 ++++--- .../org/elasticsearch/index/mapper/NestedObjectMapper.java | 2 +- .../java/org/elasticsearch/index/mapper/ObjectMapper.java | 2 +- 4 files changed, 7 insertions(+), 6 deletions(-) diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/21_synthetic_source_stored.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/21_synthetic_source_stored.yml index f80af6aa145ee..73f09836d98cf 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/21_synthetic_source_stored.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/21_synthetic_source_stored.yml @@ -272,7 +272,7 @@ object param - nested object with stored array: type: nested nested_array_stored: type: nested - synthetic_source_keep: arrays + synthetic_source_keep: all - do: bulk: diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java index 4d098779c5083..19fec3eeb0a72 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java @@ -810,9 +810,10 @@ private static void parseNonDynamicArray( // Check if we need to record the array source. This only applies to synthetic source. if (context.canAddIgnoredField()) { boolean objectRequiresStoringSource = mapper instanceof ObjectMapper objectMapper - && (objectMapper instanceof NestedObjectMapper == false - && (getSourceKeepMode(context, objectMapper.sourceKeepMode()) != Mapper.SourceKeepMode.NONE) - || objectMapper.dynamic == ObjectMapper.Dynamic.RUNTIME); + && (objectMapper.dynamic == ObjectMapper.Dynamic.RUNTIME + || (getSourceKeepMode(context, objectMapper.sourceKeepMode()) == Mapper.SourceKeepMode.ALL + || getSourceKeepMode(context, objectMapper.sourceKeepMode()) == Mapper.SourceKeepMode.ARRAYS + && objectMapper instanceof NestedObjectMapper == false)); boolean fieldWithFallbackSyntheticSource = mapper instanceof FieldMapper fieldMapper && fieldMapper.syntheticSourceMode() == FieldMapper.SyntheticSourceMode.FALLBACK; boolean fieldWithStoredArraySource = mapper instanceof FieldMapper fieldMapper diff --git a/server/src/main/java/org/elasticsearch/index/mapper/NestedObjectMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/NestedObjectMapper.java index 5661f19b2d3b9..d0e0dcb6b97ba 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/NestedObjectMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/NestedObjectMapper.java @@ -322,7 +322,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field("enabled", enabled.value()); } if (sourceKeepMode.isPresent()) { - builder.field("synthetic_source_keep", sourceKeepMode.get()); + builder.field(Mapper.SYNTHETIC_SOURCE_KEEP_PARAM, sourceKeepMode.get()); } serializeMappers(builder, params); return builder.endObject(); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java index 6e5c8299be658..7d59e24f56154 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java @@ -642,7 +642,7 @@ static MergeResult build(ObjectMapper existing, ObjectMapper mergeWithObject, Ma if (mergeWithObject.sourceKeepMode.isPresent()) { if (reason == MergeReason.INDEX_TEMPLATE) { sourceKeepMode = mergeWithObject.sourceKeepMode; - } else if (existing.sourceKeepMode != mergeWithObject.sourceKeepMode) { + } else if (existing.sourceKeepMode.isEmpty() || existing.sourceKeepMode.get() != mergeWithObject.sourceKeepMode.get()) { throw new MapperException( "the [ " + Mapper.SYNTHETIC_SOURCE_KEEP_PARAM From 16c98651b533c659a95edd4591868475f75a1f16 Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Sun, 29 Sep 2024 17:37:30 +0300 Subject: [PATCH 04/21] add tests --- .../21_synthetic_source_stored.yml | 75 ++++++++++++++++- .../mapper/IgnoredSourceFieldMapperTests.java | 81 ++++++++++++++++++- .../DefaultMappingParametersHandler.java | 6 ++ 3 files changed, 158 insertions(+), 4 deletions(-) diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/21_synthetic_source_stored.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/21_synthetic_source_stored.yml index 73f09836d98cf..2b580418456d2 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/21_synthetic_source_stored.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/21_synthetic_source_stored.yml @@ -1,3 +1,67 @@ +--- +object param - store complex object: + - requires: + cluster_features: ["mapper.synthetic_source_keep"] + reason: requires tracking ignored source + + - do: + indices.create: + index: test + body: + mappings: + _source: + mode: synthetic + properties: + id: + type: integer + stored: + synthetic_source_keep: all + properties: + object_array: + properties: + trace: + type: keyword + nested: + type: nested + kw: + type: keyword + + - do: + bulk: + index: test + refresh: true + body: + - '{ "create": { } }' + - '{ "id": 1, "stored": { "object_array": [ {"trace": "B"}, {"trace": "A"} ], "nested": [ {"foo": 20}, {"foo": 10} ], "kw": 100 } }' + - '{ "create": { } }' + - '{ "id": 2, "stored": { "object_array": { "trace": ["D", "C"] }, "nested": { "bar": [ 40, 30] }, "kw": 200, "baz": "2000" } }' + - '{ "create": { } }' + - '{ "id": 3, "stored": [ { "object_array": { "trace": "E" } }, { "nested": { "bar": [ 60, 50] } }, { "kw": 300 } ] }' + + - do: + search: + index: test + sort: id + + - match: { hits.hits.0._source.id: 1 } + - match: { hits.hits.0._source.stored.object_array.0.trace: B } + - match: { hits.hits.0._source.stored.object_array.1.trace: A } + - match: { hits.hits.0._source.stored.nested.0.foo: 20 } + - match: { hits.hits.0._source.stored.nested.1.foo: 10 } + - match: { hits.hits.0._source.stored.kw: 100 } + + - match: { hits.hits.1._source.id: 2 } + - match: { hits.hits.1._source.stored.object_array.trace: [D, C] } + - match: { hits.hits.1._source.stored.nested.bar: [40, 30] } + - match: { hits.hits.1._source.stored.kw: 200 } + - match: { hits.hits.1._source.stored.baz: "2000" } + + - match: { hits.hits.2._source.id: 3 } + - match: { hits.hits.2._source.stored.0.object_array.trace: E } + - match: { hits.hits.2._source.stored.1.nested.bar: [ 60, 50 ] } + - match: { hits.hits.2._source.stored.2.kw: 300 } + + --- object param - object array: - requires: @@ -772,6 +836,9 @@ index param - root arrays: properties: id: type: keyword + obj_default: + type: object + synthetic_source_keep: none - do: bulk: @@ -782,6 +849,8 @@ index param - root arrays: - '{ "id": 1, "leaf": [30, 20, 10], "leaf_default": [30, 20, 10], "obj": [ { "trace": { "id": "a" }, "span": { "id": "1" } }, { "trace": { "id": "b" }, "span": { "id": "1" } } ] }' - '{ "create": { } }' - '{ "id": 2, "leaf": [130, 120, 110], "leaf_default": [130, 120, 110], "obj": [ { "trace": { "id": "aa" }, "span": { "id": "2" } }, { "trace": { "id": "bb" }, "span": { "id": "2" } } ] }' + - '{ "create": { } }' + - '{ "id": 3, "obj_default": [ { "trace": { "id": "bb" }, "span": { "id": "2" } }, { "trace": { "id": "aa" }, "span": { "id": "2" } } ] }' - do: search: @@ -799,13 +868,17 @@ index param - root arrays: - match: { hits.hits.1._source.id: 2 } - match: { hits.hits.1._source.leaf: [ 130, 120, 110 ] } - - match: { hits.hits.0._source.leaf_default: [10, 20, 30] } + - match: { hits.hits.1._source.leaf_default: [110, 120, 130] } - length: { hits.hits.1._source.obj: 2 } - match: { hits.hits.1._source.obj.0.trace.id: aa } - match: { hits.hits.1._source.obj.0.span.id: "2" } - match: { hits.hits.1._source.obj.1.trace.id: bb } - match: { hits.hits.1._source.obj.1.span.id: "2" } + - match: { hits.hits.2._source.id: 3 } + - match: { hits.hits.2._source.obj_default.trace.id: [aa, bb] } + - match: { hits.hits.2._source.obj_default.span.id: "2" } + --- index param - dynamic root arrays: diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java index 4904bfbfe5f6e..511ab1186f1b3 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java @@ -578,13 +578,39 @@ public void testIndexStoredArraySourceRootObjectArray() throws IOException { })).documentMapper(); var syntheticSource = syntheticSource(documentMapper, b -> { b.startArray("path"); + b.startObject().field("int_value", 20).endObject(); b.startObject().field("int_value", 10).endObject(); + b.endArray(); + b.field("bool_value", true); + }); + assertEquals(""" + {"bool_value":true,"path":[{"int_value":20},{"int_value":10}]}""", syntheticSource); + } + + public void testIndexStoredArraySourceRootObjectArrayWithBypass() throws IOException { + DocumentMapper documentMapper = createMapperServiceWithStoredArraySource(syntheticSourceMapping(b -> { + b.startObject("path"); + { + b.field("type", "object"); + b.field("synthetic_source_keep", "none"); + b.startObject("properties"); + { + b.startObject("int_value").field("type", "integer").endObject(); + } + b.endObject(); + } + b.endObject(); + b.startObject("bool_value").field("type", "boolean").endObject(); + })).documentMapper(); + var syntheticSource = syntheticSource(documentMapper, b -> { + b.startArray("path"); b.startObject().field("int_value", 20).endObject(); + b.startObject().field("int_value", 10).endObject(); b.endArray(); b.field("bool_value", true); }); assertEquals(""" - {"bool_value":true,"path":[{"int_value":10},{"int_value":20}]}""", syntheticSource); + {"bool_value":true,"path":{"int_value":[10,20]}}""", syntheticSource); } public void testIndexStoredArraySourceNestedValueArray() throws IOException { @@ -622,6 +648,12 @@ public void testIndexStoredArraySourceNestedValueArrayDisabled() throws IOExcept { b.startObject("int_value").field("type", "integer").field(Mapper.SYNTHETIC_SOURCE_KEEP_PARAM, "none").endObject(); b.startObject("bool_value").field("type", "boolean").endObject(); + b.startObject("obj").field("type", "object").field(Mapper.SYNTHETIC_SOURCE_KEEP_PARAM, "none"); + b.startObject("properties"); + { + b.startObject("foo").field("type", "integer").endObject(); + } + b.endObject().endObject(); } b.endObject(); } @@ -632,11 +664,19 @@ public void testIndexStoredArraySourceNestedValueArrayDisabled() throws IOExcept { b.array("int_value", new int[] { 30, 20, 10 }); b.field("bool_value", true); + b.startArray("obj"); + { + b.startObject().field("foo", 2).endObject(); + b.startObject().field("foo", 1).endObject(); + } + b.endArray(); } b.endObject(); }); assertEquals(""" - {"path":{"bool_value":true,"int_value":[10,20,30]}}""", syntheticSource); + {"path":{"bool_value":true,"int_value":[10,20,30],"obj":{"foo":[1,2]}}} + + """, syntheticSource); } public void testFieldStoredArraySourceNestedValueArray() throws IOException { @@ -902,7 +942,6 @@ public void testObjectArrayAndValue() throws IOException { } b.endObject(); })).documentMapper(); - // { "path": [ { "stored":[ { "leaf": 10 } ] }, { "stored": { "leaf": 20 } } ] } var syntheticSource = syntheticSource(documentMapper, b -> { b.startArray("path"); { @@ -1023,6 +1062,42 @@ public void testStoredArrayWithinHigherLevelArray() throws IOException { {"path":{"to":[{"name":"A"},{"name":"B"},{"name":"C"},{"name":"D"}]}}""", booleanValue), syntheticSource); } + public void testObjectWithKeepAll() throws IOException { + DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> { + b.startObject("path"); + { + b.field("type", "object").field("synthetic_source_keep", "all"); + b.startObject("properties"); + { + b.startObject("a").field("type", "object").endObject(); + b.startObject("b").field("type", "integer").endObject(); + } + b.endObject(); + } + b.endObject(); + b.startObject("id").field("type", "integer").endObject(); + })).documentMapper(); + var syntheticSource = syntheticSource(documentMapper, b -> { + b.startObject("path"); + { + b.startArray("a"); + { + b.startObject().field("foo", 30).endObject(); + b.startObject().field("foo", 20).endObject(); + b.startObject().field("foo", 10).endObject(); + b.startObject().field("bar", 20).endObject(); + b.startObject().field("bar", 10).endObject(); + } + b.endArray(); + b.array("b", 4, 1, 3, 2); + } + b.endObject(); + b.field("id", 10); + }); + assertEquals(""" + {"id":10,"path":{"a":[{"foo":30},{"foo":20},{"foo":10},{"bar":20},{"bar":10}],"b":[4,1,3,2]}}""", syntheticSource); + } + public void testFallbackFieldWithinHigherLevelArray() throws IOException { DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> { b.startObject("path"); diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultMappingParametersHandler.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultMappingParametersHandler.java index 89850cd56bbd0..5a780dcc9b4a1 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultMappingParametersHandler.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultMappingParametersHandler.java @@ -83,6 +83,9 @@ public DataSourceResponse.ObjectMappingParametersGenerator handle(DataSourceRequ if (ESTestCase.randomBoolean()) { parameters.put("dynamic", ESTestCase.randomFrom("true", "false", "strict")); } + if (ESTestCase.randomBoolean()) { + parameters.put(Mapper.SYNTHETIC_SOURCE_KEEP_PARAM, "all"); // [arrays] doesn't apply to nested objects + } return parameters; }); @@ -96,6 +99,9 @@ public DataSourceResponse.ObjectMappingParametersGenerator handle(DataSourceRequ if (ESTestCase.randomBoolean()) { parameters.put("enabled", ESTestCase.randomFrom("true", "false")); } + if (ESTestCase.randomBoolean()) { + parameters.put(Mapper.SYNTHETIC_SOURCE_KEEP_PARAM, ESTestCase.randomFrom("none", "arrays", "all")); + } return parameters; }); From 51a5f712e98d3f5dd3041e40e9ba906d43cdd449 Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Mon, 30 Sep 2024 14:26:06 +0300 Subject: [PATCH 05/21] merge --- .../index/mapper/IgnoredSourceFieldMapperTests.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java index 5d3d5968ebeaf..280cc1de494cd 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java @@ -674,9 +674,7 @@ public void testIndexStoredArraySourceNestedValueArrayDisabled() throws IOExcept b.endObject(); }); assertEquals(""" - {"path":{"bool_value":true,"int_value":[10,20,30],"obj":{"foo":[1,2]}}} - - """, syntheticSource); + {"path":{"bool_value":true,"int_value":[10,20,30],"obj":{"foo":[1,2]}}}""", syntheticSource); } public void testFieldStoredArraySourceNestedValueArray() throws IOException { From 6dcdf44cb3a1a0c3d6b29b43ee531332a31e9c41 Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Mon, 30 Sep 2024 16:43:04 +0300 Subject: [PATCH 06/21] fix randomized tests --- .../index/mapper/RootObjectMapper.java | 5 +++++ .../index/mapper/RootObjectMapperTests.java | 13 +++++++++++++ .../datasource/DataSourceRequest.java | 7 ++++++- .../datasource/DefaultMappingParametersHandler.java | 2 +- .../fields/NestedFieldDataGenerator.java | 2 +- .../fields/ObjectFieldDataGenerator.java | 2 +- .../fields/TopLevelObjectFieldDataGenerator.java | 2 +- 7 files changed, 28 insertions(+), 5 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/RootObjectMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/RootObjectMapper.java index db3604d29b22d..ce983e8a327c9 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/RootObjectMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/RootObjectMapper.java @@ -150,6 +150,11 @@ public RootObjectMapper build(MapperBuilderContext context) { this.dynamicDateTimeFormatters = dynamicDateTimeFormatters; this.dateDetection = dateDetection; this.numericDetection = numericDetection; + if (sourceKeepMode.orElse(SourceKeepMode.NONE) == SourceKeepMode.ALL) { + throw new MapperParsingException( + "root object can't be configured with [" + Mapper.SYNTHETIC_SOURCE_KEEP_PARAM + ":" + SourceKeepMode.ALL + "]" + ); + } } @Override diff --git a/server/src/test/java/org/elasticsearch/index/mapper/RootObjectMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/RootObjectMapperTests.java index c4e36b681e98e..562ed50637e3e 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/RootObjectMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/RootObjectMapperTests.java @@ -362,6 +362,19 @@ public void testEmptyType() throws Exception { assertThat(e.getMessage(), containsString("type cannot be an empty string")); } + public void testSyntheticSourceKeepAllThrows() throws IOException { + String mapping = Strings.toString( + XContentFactory.jsonBuilder() + .startObject() + .startObject(MapperService.SINGLE_MAPPING_NAME) + .field("synthetic_source_keep", "all") + .endObject() + .endObject() + ); + Exception e = expectThrows(MapperParsingException.class, () -> createMapperService(mapping)); + assertThat(e.getMessage(), containsString("root object can't be configured with [synthetic_source_keep:all]")); + } + public void testWithoutMappers() throws IOException { RootObjectMapper shallowRoot = createRootObjectMapperWithAllParametersSet(b -> {}, b -> {}); RootObjectMapper root = createRootObjectMapperWithAllParametersSet(b -> { diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceRequest.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceRequest.java index 81e120511a40f..067d1b96e965e 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceRequest.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceRequest.java @@ -12,6 +12,7 @@ import org.elasticsearch.logsdb.datageneration.DataGeneratorSpecification; import org.elasticsearch.logsdb.datageneration.FieldType; import org.elasticsearch.logsdb.datageneration.fields.DynamicMapping; +import org.elasticsearch.test.ESTestCase; import java.util.Set; @@ -115,11 +116,15 @@ public DataSourceResponse.LeafMappingParametersGenerator accept(DataSourceHandle } } - record ObjectMappingParametersGenerator(boolean isNested) + record ObjectMappingParametersGenerator(boolean isRoot, boolean isNested) implements DataSourceRequest { public DataSourceResponse.ObjectMappingParametersGenerator accept(DataSourceHandler handler) { return handler.handle(this); } + + public String syntheticSourceKeepValue() { + return isRoot() ? ESTestCase.randomFrom("none", "arrays") : ESTestCase.randomFrom("none", "arrays", "all"); + } } } diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultMappingParametersHandler.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultMappingParametersHandler.java index 5a780dcc9b4a1..69f839d461b40 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultMappingParametersHandler.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultMappingParametersHandler.java @@ -100,7 +100,7 @@ public DataSourceResponse.ObjectMappingParametersGenerator handle(DataSourceRequ parameters.put("enabled", ESTestCase.randomFrom("true", "false")); } if (ESTestCase.randomBoolean()) { - parameters.put(Mapper.SYNTHETIC_SOURCE_KEEP_PARAM, ESTestCase.randomFrom("none", "arrays", "all")); + parameters.put(Mapper.SYNTHETIC_SOURCE_KEEP_PARAM, request.syntheticSourceKeepValue()); } return parameters; diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/NestedFieldDataGenerator.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/NestedFieldDataGenerator.java index b5cd4f78aff95..ba168b221f572 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/NestedFieldDataGenerator.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/NestedFieldDataGenerator.java @@ -28,7 +28,7 @@ public class NestedFieldDataGenerator implements FieldDataGenerator { this.mappingParameters = context.specification() .dataSource() - .get(new DataSourceRequest.ObjectMappingParametersGenerator(true)) + .get(new DataSourceRequest.ObjectMappingParametersGenerator(false, true)) .mappingGenerator() .get(); var dynamicMapping = context.determineDynamicMapping(mappingParameters); diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/ObjectFieldDataGenerator.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/ObjectFieldDataGenerator.java index 27c27e31702f7..084310ac967fc 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/ObjectFieldDataGenerator.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/ObjectFieldDataGenerator.java @@ -28,7 +28,7 @@ public class ObjectFieldDataGenerator implements FieldDataGenerator { this.mappingParameters = context.specification() .dataSource() - .get(new DataSourceRequest.ObjectMappingParametersGenerator(false)) + .get(new DataSourceRequest.ObjectMappingParametersGenerator(false, false)) .mappingGenerator() .get(); var dynamicMapping = context.determineDynamicMapping(mappingParameters); diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/TopLevelObjectFieldDataGenerator.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/TopLevelObjectFieldDataGenerator.java index e85d18a1dac12..2c7aa65d8c6d1 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/TopLevelObjectFieldDataGenerator.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/TopLevelObjectFieldDataGenerator.java @@ -37,7 +37,7 @@ public TopLevelObjectFieldDataGenerator(DataGeneratorSpecification specification this.mappingParameters = Map.of(); } else { this.mappingParameters = new HashMap<>( - specification.dataSource().get(new DataSourceRequest.ObjectMappingParametersGenerator(false)).mappingGenerator().get() + specification.dataSource().get(new DataSourceRequest.ObjectMappingParametersGenerator(true, false)).mappingGenerator().get() ); // Top-level object can't be disabled because @timestamp is a required field in data streams. this.mappingParameters.remove("enabled"); From b8b85c73b88d2a2481060ffcffc237a1da03d6be Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Tue, 1 Oct 2024 12:20:51 +0300 Subject: [PATCH 07/21] add documentation --- .../mapping/fields/synthetic-source.asciidoc | 136 ++++++++++++++++-- .../index/mapper/ObjectMapper.java | 1 + 2 files changed, 127 insertions(+), 10 deletions(-) diff --git a/docs/reference/mapping/fields/synthetic-source.asciidoc b/docs/reference/mapping/fields/synthetic-source.asciidoc index ccea38cf602da..eb2ccae7b7fc7 100644 --- a/docs/reference/mapping/fields/synthetic-source.asciidoc +++ b/docs/reference/mapping/fields/synthetic-source.asciidoc @@ -32,18 +32,25 @@ space. Additional latency can be avoided by not loading `_source` field in queri [[synthetic-source-fields]] ===== Supported fields -Synthetic `_source` is supported by all field types. Depending on implementation details, field types have different properties when used with synthetic `_source`. +Synthetic `_source` is supported by all field types. Depending on implementation details, field types have different +properties when used with synthetic `_source`. -<> construct synthetic `_source` using existing data, most commonly <> and <>. For these field types, no additional space is needed to store the contents of `_source` field. Due to the storage layout of <>, the generated `_source` field undergoes <> compared to original document. +<> construct synthetic `_source` using existing data, most +commonly <> and <>. For these field types, no additional space +is needed to store the contents of `_source` field. Due to the storage layout of <>, the +generated `_source` field undergoes <> compared to original document. -For all other field types, the original value of the field is stored as is, in the same way as the `_source` field in non-synthetic mode. In this case there are no modifications and field data in `_source` is the same as in the original document. Similarly, malformed values of fields that use <> or <> need to be stored as is. This approach is less storage efficient since data needed for `_source` reconstruction is stored in addition to other data required to index the field (like `doc_values`). +For all other field types, the original value of the field is stored as is, in the same way as the `_source` field in +non-synthetic mode. In this case there are no modifications and field data in `_source` is the same as in the original +document. Similarly, malformed values of fields that use <> or +<> need to be stored as is. This approach is less storage efficient since data needed for +`_source` reconstruction is stored in addition to other data required to index the field (like `doc_values`). [[synthetic-source-restrictions]] ===== Synthetic `_source` restrictions -Synthetic `_source` cannot be used together with field mappings that use <>. - -Some field types have additional restrictions. These restrictions are documented in the **synthetic `_source`** section of the field type's <>. +Some field types have additional restrictions. These restrictions are documented in the **synthetic `_source`** section +of the field type's <>. [[synthetic-source-modifications]] ===== Synthetic `_source` modifications @@ -144,6 +151,42 @@ Will become: ---- // TEST[s/^/{"_source":/ s/\n$/}/] +This impacts how source contents can be referenced in <>. For instance, referencing +a script in its original source form will return null: + +[source,js] +---- +"script": { "source": """ emit(params._source['foo.bar.baz']) """ } +---- +// NOTCONSOLE + +Instead, source references need to be in line with the mapping structure: + +[source,js] +---- +"script": { "source": """ emit(params._source['foo']['bar']['baz']) """ } +---- +// NOTCONSOLE + +or simply + +[source,js] +---- +"script": { "source": """ emit(params._source.foo.bar.baz) """ } +---- +// NOTCONSOLE + +The following <> are preferable as, in addition to being agnostic to the +mapping structure, they make use of docvalues if available and fall back to synthetic source only when needed. This +reduces source synthesizing, a slow and costly operation. + +[source,js] +---- +"script": { "source": """ emit(field('foo.bar.baz').get(null)) """ } +"script": { "source": """ emit($('foo.bar.baz', null)) """ } +---- +// NOTCONSOLE + [[synthetic-source-modifications-alphabetical]] ====== Alphabetical sorting Synthetic `_source` fields are sorted alphabetically. The @@ -155,18 +198,91 @@ that ordering. [[synthetic-source-modifications-ranges]] ====== Representation of ranges -Range field values (e.g. `long_range`) are always represented as inclusive on both sides with bounds adjusted accordingly. See <>. +Range field values (e.g. `long_range`) are always represented as inclusive on both sides with bounds adjusted +accordingly. See <>. [[synthetic-source-precision-loss-for-point-types]] ====== Reduced precision of `geo_point` values -Values of `geo_point` fields are represented in synthetic `_source` with reduced precision. See <>. +Values of `geo_point` fields are represented in synthetic `_source` with reduced precision. See +<>. + +[[synthetic-source-keep]] +====== Keeping the origical source + +It is possible to record the original source of an object or field, at extra storage cost, using param +`synthetic_source_keep`. The default value is `none`; setting it to `arrays` leads to storing the original source for +arrays of the corresponding field or object, while setting it to `all` records both singleton instances and arrays of +the object or value it is applied to. When applied to objects, the source of all sub-objects and sub-fields gets +captured. For instance: + +[source,console,id=create-index-with-synthetic-source-keep] +---- +PUT idx_keep +{ + "mappings": { + "_source": { + "mode": "synthetic" + }, + "properties": { + "path": { + "type": "object", + "synthetic_source_keep": "all" + }, + "ids": { + "type": "integer", + "synthetic_source_keep": "arrays" + } + } + } +} +---- +// TEST + +[source,console,id=synthetic-source-keep] +---- +PUT idx_keep/_doc/1 +{ + "path": { + "to": [ + { "foo": [3, 2, 1] }, + { "foo": [30, 20, 10] } + ], + "bar": "baz" + }, + "ids": [ 200, 100, 300, 100 ] +} +---- +// TEST[s/$/\nGET idx_keep\/_doc\/1?filter_path=_source\n/] + +returns the original source, with no array deduplication and sorting: + +[source,console-result] +---- +{ + "path": { + "to": [ + { "foo": [3, 2, 1] }, + { "foo": [30, 20, 10] } + ], + "bar": "baz" + }, + "ids": [ 200, 100, 300, 100 ] +} +---- +// TEST[s/^/{"_source":/ s/\n$/}/] +The option for capturing the source of arrays can be applied at index level, by setting +`index.mapping.synthetic_source_keep` to `arrays`. This applies to all objects and fields in the index, except for +the ones with explicit overrides of `synthetic_source_keep` set to `none`. In this case, the storage overhead grows +with the number and sizes of arrays present in source of each document, naturally. [[synthetic-source-fields-native-list]] ===== Field types that support synthetic source with no storage overhead -The following field types support synthetic source using data from <> or <>, and require no additional storage space to construct the `_source` field. +The following field types support synthetic source using data from <> or +>, and require no additional storage space to construct the `_source` field. -NOTE: If you enable the <> or <> settings, then additional storage is required to store ignored field values for these types. +NOTE: If you enable the <> or <> settings, then +additional storage is required to store ignored field values for these types. ** <> ** {plugins}/mapper-annotated-text-usage.html#annotated-text-synthetic-source[`annotated-text`] diff --git a/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java index 12a837b01d493..0b9727aa66c8a 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java @@ -45,6 +45,7 @@ public class ObjectMapper extends Mapper { public static final String CONTENT_TYPE = "object"; static final String STORE_ARRAY_SOURCE_PARAM = "store_array_source"; static final NodeFeature SUBOBJECTS_AUTO = new NodeFeature("mapper.subobjects_auto"); + // No-op. All uses of this feature were reverted but node features can't be removed. static final NodeFeature SUBOBJECTS_AUTO_FIXES = new NodeFeature("mapper.subobjects_auto_fixes"); /** From ee047bc5ccd4cd6ae5e1a983cd434e1b36287373 Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Wed, 2 Oct 2024 15:47:54 +0300 Subject: [PATCH 08/21] dedup id in docs --- docs/reference/mapping/fields/synthetic-source.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/reference/mapping/fields/synthetic-source.asciidoc b/docs/reference/mapping/fields/synthetic-source.asciidoc index eb2ccae7b7fc7..5c4556d8d2ddc 100644 --- a/docs/reference/mapping/fields/synthetic-source.asciidoc +++ b/docs/reference/mapping/fields/synthetic-source.asciidoc @@ -238,7 +238,7 @@ PUT idx_keep ---- // TEST -[source,console,id=synthetic-source-keep] +[source,console,id=synthetic-source-keep-example] ---- PUT idx_keep/_doc/1 { From c214eb48b1874bd9d6b46abc8adaf6578a1649b3 Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Thu, 3 Oct 2024 14:41:56 +0300 Subject: [PATCH 09/21] update documentation --- .../mapping/fields/synthetic-source.asciidoc | 20 +++++++++++++------ .../index/mapper/DocumentParser.java | 15 ++++---------- 2 files changed, 18 insertions(+), 17 deletions(-) diff --git a/docs/reference/mapping/fields/synthetic-source.asciidoc b/docs/reference/mapping/fields/synthetic-source.asciidoc index 5c4556d8d2ddc..7549850025c7d 100644 --- a/docs/reference/mapping/fields/synthetic-source.asciidoc +++ b/docs/reference/mapping/fields/synthetic-source.asciidoc @@ -207,13 +207,21 @@ Values of `geo_point` fields are represented in synthetic `_source` with reduced <>. [[synthetic-source-keep]] -====== Keeping the origical source +====== Minimizing source modifications -It is possible to record the original source of an object or field, at extra storage cost, using param -`synthetic_source_keep`. The default value is `none`; setting it to `arrays` leads to storing the original source for -arrays of the corresponding field or object, while setting it to `all` records both singleton instances and arrays of -the object or value it is applied to. When applied to objects, the source of all sub-objects and sub-fields gets -captured. For instance: +It is possible to avoid the synthetic source modifications for a particular object or field, at extra storage cost. +This is controlled through param `synthetic_source_keep` with the following values: + + - `none`: synthetic source diverges from the original source as described above (default). + - `arrays`: arrays of the corresponding field or object preserve the original element ordering and duplicate elements. +The synthetic source fragment for such arrays is not guaranteed to match the original source exactly, e.g. array +`[1, 2, [5], [[4, [3]]], 5]` may appear as-is or in an equivalent format like `[1, 2, 5, 4, 3, 5]`. The exact format +may change in the future, in an effort to reduce the storage overhead of this option. +- `all`: the source for both singleton instances and arrays of the corresponding field or object gets recorded. When +applied to objects, the source of all sub-objects and sub-fields gets captured. Furthermore, the original source of +arrays gets captured and appears in synthetic source with no modifications. + +For instance: [source,console,id=create-index-with-synthetic-source-keep] ---- diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java index 34683c825a5c3..99eb08977f627 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java @@ -802,8 +802,8 @@ private static void parseNonDynamicArray( // Check if we need to record the array source. This only applies to synthetic source. if (context.canAddIgnoredField()) { boolean objectRequiresStoringSource = mapper instanceof ObjectMapper objectMapper - && ((getSourceKeepMode(context, objectMapper.sourceKeepMode()) == Mapper.SourceKeepMode.ALL - || getSourceKeepMode(context, objectMapper.sourceKeepMode()) == Mapper.SourceKeepMode.ARRAYS + && (getSourceKeepMode(context, objectMapper.sourceKeepMode()) == Mapper.SourceKeepMode.ALL + || (getSourceKeepMode(context, objectMapper.sourceKeepMode()) == Mapper.SourceKeepMode.ARRAYS && objectMapper instanceof NestedObjectMapper == false)); boolean fieldWithFallbackSyntheticSource = mapper instanceof FieldMapper fieldMapper && fieldMapper.syntheticSourceMode() == FieldMapper.SyntheticSourceMode.FALLBACK; @@ -1116,15 +1116,8 @@ protected SyntheticSourceSupport syntheticSourceSupport() { private static class NoOpObjectMapper extends ObjectMapper { NoOpObjectMapper(String name, String fullPath) { - super( - name, - fullPath, - Explicit.IMPLICIT_TRUE, - Optional.empty(), - Explicit.IMPLICIT_FALSE, - Dynamic.RUNTIME, - Collections.emptyMap() - ); + super(name, fullPath, Explicit.IMPLICIT_TRUE, Optional.empty(), Optional.empty(), Dynamic.RUNTIME, Collections.emptyMap()); + } @Override From 97b6c017e07434d8e1243b8ed2d07b8b2043fa3e Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Thu, 3 Oct 2024 14:45:19 +0300 Subject: [PATCH 10/21] update documentation --- docs/reference/mapping/fields/synthetic-source.asciidoc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/reference/mapping/fields/synthetic-source.asciidoc b/docs/reference/mapping/fields/synthetic-source.asciidoc index 7549850025c7d..902b6c26611e5 100644 --- a/docs/reference/mapping/fields/synthetic-source.asciidoc +++ b/docs/reference/mapping/fields/synthetic-source.asciidoc @@ -209,8 +209,8 @@ Values of `geo_point` fields are represented in synthetic `_source` with reduced [[synthetic-source-keep]] ====== Minimizing source modifications -It is possible to avoid the synthetic source modifications for a particular object or field, at extra storage cost. -This is controlled through param `synthetic_source_keep` with the following values: +It is possible to avoid synthetic source modifications for a particular object or field, at extra storage cost. +This is controlled through param `synthetic_source_keep` with the following option: - `none`: synthetic source diverges from the original source as described above (default). - `arrays`: arrays of the corresponding field or object preserve the original element ordering and duplicate elements. From f4ddb0e5e5bf26b841a3216aa5969ed60dabde59 Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Thu, 3 Oct 2024 15:30:15 +0300 Subject: [PATCH 11/21] fix bwc --- .../indices.create/20_synthetic_source.yml | 2 +- .../21_synthetic_source_stored.yml | 18 +++++++++--------- .../index/mapper/MapperFeatures.java | 6 ++++++ 3 files changed, 16 insertions(+), 10 deletions(-) diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml index af12289e5de53..62a248cae6b2d 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml @@ -895,7 +895,7 @@ doubly nested object: --- subobjects auto: - requires: - cluster_features: ["mapper.subobjects_auto"] + cluster_features: ["mapper.subobjects_auto", "mapper.bwc_workaround_9_0"] reason: requires tracking ignored source and supporting subobjects auto setting - do: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/21_synthetic_source_stored.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/21_synthetic_source_stored.yml index 2b580418456d2..0bf402e6df549 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/21_synthetic_source_stored.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/21_synthetic_source_stored.yml @@ -1,7 +1,7 @@ --- object param - store complex object: - requires: - cluster_features: ["mapper.synthetic_source_keep"] + cluster_features: ["mapper.synthetic_source_keep", "mapper.bwc_workaround_9_0"] reason: requires tracking ignored source - do: @@ -65,7 +65,7 @@ object param - store complex object: --- object param - object array: - requires: - cluster_features: ["mapper.track_ignored_source"] + cluster_features: ["mapper.track_ignored_source", "mapper.bwc_workaround_9_0"] reason: requires tracking ignored source - do: @@ -129,7 +129,7 @@ object param - object array: --- object param - object array within array: - requires: - cluster_features: ["mapper.track_ignored_source"] + cluster_features: ["mapper.track_ignored_source", "mapper.bwc_workaround_9_0"] reason: requires tracking ignored source - do: @@ -172,7 +172,7 @@ object param - object array within array: --- object param - no object array: - requires: - cluster_features: ["mapper.track_ignored_source"] + cluster_features: ["mapper.track_ignored_source", "mapper.bwc_workaround_9_0"] reason: requires tracking ignored source - do: @@ -214,7 +214,7 @@ object param - no object array: --- object param - field ordering in object array: - requires: - cluster_features: ["mapper.track_ignored_source"] + cluster_features: ["mapper.track_ignored_source", "mapper.bwc_workaround_9_0"] reason: requires tracking ignored source - do: @@ -263,7 +263,7 @@ object param - field ordering in object array: --- object param - nested object array next to other fields: - requires: - cluster_features: ["mapper.track_ignored_source"] + cluster_features: ["mapper.track_ignored_source", "mapper.bwc_workaround_9_0"] reason: requires tracking ignored source - do: @@ -319,7 +319,7 @@ object param - nested object array next to other fields: --- object param - nested object with stored array: - requires: - cluster_features: ["mapper.track_ignored_source"] + cluster_features: ["mapper.track_ignored_source", "mapper.bwc_workaround_9_0"] reason: requires tracking ignored source - do: @@ -415,7 +415,7 @@ index param - nested array within array: # 112156 stored field under object with store_array_source: - requires: - cluster_features: ["mapper.source.synthetic_source_stored_fields_advance_fix"] + cluster_features: ["mapper.source.synthetic_source_stored_fields_advance_fix", "mapper.bwc_workaround_9_0"] reason: requires bug fix to be implemented - do: @@ -804,7 +804,7 @@ field param - nested array within array: --- index param - root arrays: - requires: - cluster_features: ["mapper.synthetic_source_keep"] + cluster_features: ["mapper.synthetic_source_keep", "gte_v8.16.0"] reason: requires keeping array source - do: diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java index cf8f391813c09..a8fbb0e0ae95b 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java @@ -21,9 +21,15 @@ * Spec for mapper-related features. */ public class MapperFeatures implements FeatureSpecification { + + // Used to avoid noise in mixed cluster and rest compatibility tests. Must not be backported to 8.x branch. + // This label gets added to tests with such failures before merging with main, then removed when backported to 8.x. + public static final NodeFeature BWC_WORKAROUND_9_0 = new NodeFeature("mapper.bwc_workaround_9_0"); + @Override public Set getFeatures() { return Set.of( + BWC_WORKAROUND_9_0, IgnoredSourceFieldMapper.TRACK_IGNORED_SOURCE, PassThroughObjectMapper.PASS_THROUGH_PRIORITY, RangeFieldMapper.NULL_VALUES_OFF_BY_ONE_FIX, From 18dc913eee209b27c14f5cdaac4247c18b65c6b1 Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Thu, 3 Oct 2024 15:54:56 +0300 Subject: [PATCH 12/21] fix bwc --- .../test/indices.create/21_synthetic_source_stored.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/21_synthetic_source_stored.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/21_synthetic_source_stored.yml index 0bf402e6df549..2e1e845e72ce8 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/21_synthetic_source_stored.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/21_synthetic_source_stored.yml @@ -804,7 +804,7 @@ field param - nested array within array: --- index param - root arrays: - requires: - cluster_features: ["mapper.synthetic_source_keep", "gte_v8.16.0"] + cluster_features: ["mapper.synthetic_source_keep", "mapper.bwc_workaround_9_0"] reason: requires keeping array source - do: From b5f1d7181c0c2252d44d4be053981e1a7856d02a Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Thu, 3 Oct 2024 15:55:48 +0300 Subject: [PATCH 13/21] fix unintended --- .../main/java/org/elasticsearch/index/mapper/DocumentParser.java | 1 - 1 file changed, 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java index 99eb08977f627..59a8c60e0d255 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java @@ -1117,7 +1117,6 @@ protected SyntheticSourceSupport syntheticSourceSupport() { private static class NoOpObjectMapper extends ObjectMapper { NoOpObjectMapper(String name, String fullPath) { super(name, fullPath, Explicit.IMPLICIT_TRUE, Optional.empty(), Optional.empty(), Dynamic.RUNTIME, Collections.emptyMap()); - } @Override From 432757fbe55714bf08f83c0e4adeffbac65acb21 Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Thu, 3 Oct 2024 16:37:49 +0300 Subject: [PATCH 14/21] Revert "fix bwc" This reverts commit 18dc913eee209b27c14f5cdaac4247c18b65c6b1. --- .../test/indices.create/21_synthetic_source_stored.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/21_synthetic_source_stored.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/21_synthetic_source_stored.yml index 2e1e845e72ce8..0bf402e6df549 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/21_synthetic_source_stored.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/21_synthetic_source_stored.yml @@ -804,7 +804,7 @@ field param - nested array within array: --- index param - root arrays: - requires: - cluster_features: ["mapper.synthetic_source_keep", "mapper.bwc_workaround_9_0"] + cluster_features: ["mapper.synthetic_source_keep", "gte_v8.16.0"] reason: requires keeping array source - do: From 771bc38d0fec5cf494b41c22b91daa9bdb038e62 Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Thu, 3 Oct 2024 16:37:58 +0300 Subject: [PATCH 15/21] Revert "fix bwc" This reverts commit f4ddb0e5e5bf26b841a3216aa5969ed60dabde59. --- .../indices.create/20_synthetic_source.yml | 2 +- .../21_synthetic_source_stored.yml | 18 +++++++++--------- .../index/mapper/MapperFeatures.java | 6 ------ 3 files changed, 10 insertions(+), 16 deletions(-) diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml index 62a248cae6b2d..af12289e5de53 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml @@ -895,7 +895,7 @@ doubly nested object: --- subobjects auto: - requires: - cluster_features: ["mapper.subobjects_auto", "mapper.bwc_workaround_9_0"] + cluster_features: ["mapper.subobjects_auto"] reason: requires tracking ignored source and supporting subobjects auto setting - do: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/21_synthetic_source_stored.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/21_synthetic_source_stored.yml index 0bf402e6df549..2b580418456d2 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/21_synthetic_source_stored.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/21_synthetic_source_stored.yml @@ -1,7 +1,7 @@ --- object param - store complex object: - requires: - cluster_features: ["mapper.synthetic_source_keep", "mapper.bwc_workaround_9_0"] + cluster_features: ["mapper.synthetic_source_keep"] reason: requires tracking ignored source - do: @@ -65,7 +65,7 @@ object param - store complex object: --- object param - object array: - requires: - cluster_features: ["mapper.track_ignored_source", "mapper.bwc_workaround_9_0"] + cluster_features: ["mapper.track_ignored_source"] reason: requires tracking ignored source - do: @@ -129,7 +129,7 @@ object param - object array: --- object param - object array within array: - requires: - cluster_features: ["mapper.track_ignored_source", "mapper.bwc_workaround_9_0"] + cluster_features: ["mapper.track_ignored_source"] reason: requires tracking ignored source - do: @@ -172,7 +172,7 @@ object param - object array within array: --- object param - no object array: - requires: - cluster_features: ["mapper.track_ignored_source", "mapper.bwc_workaround_9_0"] + cluster_features: ["mapper.track_ignored_source"] reason: requires tracking ignored source - do: @@ -214,7 +214,7 @@ object param - no object array: --- object param - field ordering in object array: - requires: - cluster_features: ["mapper.track_ignored_source", "mapper.bwc_workaround_9_0"] + cluster_features: ["mapper.track_ignored_source"] reason: requires tracking ignored source - do: @@ -263,7 +263,7 @@ object param - field ordering in object array: --- object param - nested object array next to other fields: - requires: - cluster_features: ["mapper.track_ignored_source", "mapper.bwc_workaround_9_0"] + cluster_features: ["mapper.track_ignored_source"] reason: requires tracking ignored source - do: @@ -319,7 +319,7 @@ object param - nested object array next to other fields: --- object param - nested object with stored array: - requires: - cluster_features: ["mapper.track_ignored_source", "mapper.bwc_workaround_9_0"] + cluster_features: ["mapper.track_ignored_source"] reason: requires tracking ignored source - do: @@ -415,7 +415,7 @@ index param - nested array within array: # 112156 stored field under object with store_array_source: - requires: - cluster_features: ["mapper.source.synthetic_source_stored_fields_advance_fix", "mapper.bwc_workaround_9_0"] + cluster_features: ["mapper.source.synthetic_source_stored_fields_advance_fix"] reason: requires bug fix to be implemented - do: @@ -804,7 +804,7 @@ field param - nested array within array: --- index param - root arrays: - requires: - cluster_features: ["mapper.synthetic_source_keep", "gte_v8.16.0"] + cluster_features: ["mapper.synthetic_source_keep"] reason: requires keeping array source - do: diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java index a8fbb0e0ae95b..cf8f391813c09 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java @@ -21,15 +21,9 @@ * Spec for mapper-related features. */ public class MapperFeatures implements FeatureSpecification { - - // Used to avoid noise in mixed cluster and rest compatibility tests. Must not be backported to 8.x branch. - // This label gets added to tests with such failures before merging with main, then removed when backported to 8.x. - public static final NodeFeature BWC_WORKAROUND_9_0 = new NodeFeature("mapper.bwc_workaround_9_0"); - @Override public Set getFeatures() { return Set.of( - BWC_WORKAROUND_9_0, IgnoredSourceFieldMapper.TRACK_IGNORED_SOURCE, PassThroughObjectMapper.PASS_THROUGH_PRIORITY, RangeFieldMapper.NULL_VALUES_OFF_BY_ONE_FIX, From 895e08f74b531bbdf57f758c3c347524f5a526cf Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Thu, 3 Oct 2024 16:48:35 +0300 Subject: [PATCH 16/21] add missing test --- .../indices.create/20_synthetic_source.yml | 2 +- .../21_synthetic_source_stored.yml | 20 +++++++++---------- .../index/mapper/MapperFeatures.java | 6 ++++++ 3 files changed, 17 insertions(+), 11 deletions(-) diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml index af12289e5de53..62a248cae6b2d 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml @@ -895,7 +895,7 @@ doubly nested object: --- subobjects auto: - requires: - cluster_features: ["mapper.subobjects_auto"] + cluster_features: ["mapper.subobjects_auto", "mapper.bwc_workaround_9_0"] reason: requires tracking ignored source and supporting subobjects auto setting - do: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/21_synthetic_source_stored.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/21_synthetic_source_stored.yml index 2b580418456d2..07ab5259e6fd6 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/21_synthetic_source_stored.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/21_synthetic_source_stored.yml @@ -1,7 +1,7 @@ --- object param - store complex object: - requires: - cluster_features: ["mapper.synthetic_source_keep"] + cluster_features: ["mapper.synthetic_source_keep", "mapper.bwc_workaround_9_0"] reason: requires tracking ignored source - do: @@ -65,7 +65,7 @@ object param - store complex object: --- object param - object array: - requires: - cluster_features: ["mapper.track_ignored_source"] + cluster_features: ["mapper.track_ignored_source", "mapper.bwc_workaround_9_0"] reason: requires tracking ignored source - do: @@ -129,7 +129,7 @@ object param - object array: --- object param - object array within array: - requires: - cluster_features: ["mapper.track_ignored_source"] + cluster_features: ["mapper.track_ignored_source", "mapper.bwc_workaround_9_0"] reason: requires tracking ignored source - do: @@ -172,7 +172,7 @@ object param - object array within array: --- object param - no object array: - requires: - cluster_features: ["mapper.track_ignored_source"] + cluster_features: ["mapper.track_ignored_source", "mapper.bwc_workaround_9_0"] reason: requires tracking ignored source - do: @@ -214,7 +214,7 @@ object param - no object array: --- object param - field ordering in object array: - requires: - cluster_features: ["mapper.track_ignored_source"] + cluster_features: ["mapper.track_ignored_source", "mapper.bwc_workaround_9_0"] reason: requires tracking ignored source - do: @@ -263,7 +263,7 @@ object param - field ordering in object array: --- object param - nested object array next to other fields: - requires: - cluster_features: ["mapper.track_ignored_source"] + cluster_features: ["mapper.track_ignored_source", "mapper.bwc_workaround_9_0"] reason: requires tracking ignored source - do: @@ -319,7 +319,7 @@ object param - nested object array next to other fields: --- object param - nested object with stored array: - requires: - cluster_features: ["mapper.track_ignored_source"] + cluster_features: ["mapper.track_ignored_source", "mapper.bwc_workaround_9_0"] reason: requires tracking ignored source - do: @@ -368,7 +368,7 @@ object param - nested object with stored array: --- index param - nested array within array: - requires: - cluster_features: ["mapper.synthetic_source_keep"] + cluster_features: ["mapper.synthetic_source_keep", "mapper.bwc_workaround_9_0"] reason: requires tracking ignored source - do: @@ -415,7 +415,7 @@ index param - nested array within array: # 112156 stored field under object with store_array_source: - requires: - cluster_features: ["mapper.source.synthetic_source_stored_fields_advance_fix"] + cluster_features: ["mapper.source.synthetic_source_stored_fields_advance_fix", "mapper.bwc_workaround_9_0"] reason: requires bug fix to be implemented - do: @@ -804,7 +804,7 @@ field param - nested array within array: --- index param - root arrays: - requires: - cluster_features: ["mapper.synthetic_source_keep"] + cluster_features: ["mapper.synthetic_source_keep", "mapper.bwc_workaround_9_0"] reason: requires keeping array source - do: diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java index cf8f391813c09..a8fbb0e0ae95b 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java @@ -21,9 +21,15 @@ * Spec for mapper-related features. */ public class MapperFeatures implements FeatureSpecification { + + // Used to avoid noise in mixed cluster and rest compatibility tests. Must not be backported to 8.x branch. + // This label gets added to tests with such failures before merging with main, then removed when backported to 8.x. + public static final NodeFeature BWC_WORKAROUND_9_0 = new NodeFeature("mapper.bwc_workaround_9_0"); + @Override public Set getFeatures() { return Set.of( + BWC_WORKAROUND_9_0, IgnoredSourceFieldMapper.TRACK_IGNORED_SOURCE, PassThroughObjectMapper.PASS_THROUGH_PRIORITY, RangeFieldMapper.NULL_VALUES_OFF_BY_ONE_FIX, From d3b5c3070f715949c1932e69f6b27800c79e68aa Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Thu, 3 Oct 2024 18:14:31 +0300 Subject: [PATCH 17/21] fix transform --- rest-api-spec/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rest-api-spec/build.gradle b/rest-api-spec/build.gradle index ed1cf905f7e9d..8e6ef811d0b3f 100644 --- a/rest-api-spec/build.gradle +++ b/rest-api-spec/build.gradle @@ -57,5 +57,5 @@ tasks.named("precommit").configure { tasks.named("yamlRestCompatTestTransform").configure({task -> task.skipTest("indices.sort/10_basic/Index Sort", "warning does not exist for compatibility") task.skipTest("search/330_fetch_fields/Test search rewrite", "warning does not exist for compatibility") - task.skipTestsByFilePattern("indices.create/synthetic_source*.yml", "@UpdateForV9 -> tests do not pass after bumping API version to 9 [ES-9597]") + task.skipTestsByFilePattern("indices.create/*synthetic_source*.yml", "@UpdateForV9 -> tests do not pass after bumping API version to 9 [ES-9597]") }) From de651f8a62a99843461332a411cf33fc1168f75a Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Thu, 3 Oct 2024 18:45:06 +0300 Subject: [PATCH 18/21] fix transform --- rest-api-spec/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rest-api-spec/build.gradle b/rest-api-spec/build.gradle index 8e6ef811d0b3f..3ffa14890a216 100644 --- a/rest-api-spec/build.gradle +++ b/rest-api-spec/build.gradle @@ -57,5 +57,5 @@ tasks.named("precommit").configure { tasks.named("yamlRestCompatTestTransform").configure({task -> task.skipTest("indices.sort/10_basic/Index Sort", "warning does not exist for compatibility") task.skipTest("search/330_fetch_fields/Test search rewrite", "warning does not exist for compatibility") - task.skipTestsByFilePattern("indices.create/*synthetic_source*.yml", "@UpdateForV9 -> tests do not pass after bumping API version to 9 [ES-9597]") + task.skipTestsByFilePattern("*indices.create/*synthetic_source*.yml", "@UpdateForV9 -> tests do not pass after bumping API version to 9 [ES-9597]") }) From 68875046345937956dc5db1ee5ec5a39ec060c93 Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Thu, 3 Oct 2024 19:14:18 +0300 Subject: [PATCH 19/21] fix transform --- rest-api-spec/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rest-api-spec/build.gradle b/rest-api-spec/build.gradle index 3ffa14890a216..b2aa5c222dc49 100644 --- a/rest-api-spec/build.gradle +++ b/rest-api-spec/build.gradle @@ -57,5 +57,5 @@ tasks.named("precommit").configure { tasks.named("yamlRestCompatTestTransform").configure({task -> task.skipTest("indices.sort/10_basic/Index Sort", "warning does not exist for compatibility") task.skipTest("search/330_fetch_fields/Test search rewrite", "warning does not exist for compatibility") - task.skipTestsByFilePattern("*indices.create/*synthetic_source*.yml", "@UpdateForV9 -> tests do not pass after bumping API version to 9 [ES-9597]") + task.skipTestsByFilePattern("**/*create/2*_synthetic_source*.yml", "@UpdateForV9 -> tests do not pass after bumping API version to 9 [ES-9597]") }) From 92d5c1bc12ebef90078652e447d964ab1c82ecb4 Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Thu, 3 Oct 2024 19:34:33 +0300 Subject: [PATCH 20/21] fix transform --- rest-api-spec/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rest-api-spec/build.gradle b/rest-api-spec/build.gradle index b2aa5c222dc49..cd69654aa35c4 100644 --- a/rest-api-spec/build.gradle +++ b/rest-api-spec/build.gradle @@ -57,5 +57,5 @@ tasks.named("precommit").configure { tasks.named("yamlRestCompatTestTransform").configure({task -> task.skipTest("indices.sort/10_basic/Index Sort", "warning does not exist for compatibility") task.skipTest("search/330_fetch_fields/Test search rewrite", "warning does not exist for compatibility") - task.skipTestsByFilePattern("**/*create/2*_synthetic_source*.yml", "@UpdateForV9 -> tests do not pass after bumping API version to 9 [ES-9597]") + task.skipTestsByFilePattern("*create/*synthetic_source*.yml", "@UpdateForV9 -> tests do not pass after bumping API version to 9 [ES-9597]") }) From c4d011116fd2d21456d5b1db2146c2e3a5b9ab35 Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Thu, 3 Oct 2024 20:15:25 +0300 Subject: [PATCH 21/21] fix transform --- rest-api-spec/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rest-api-spec/build.gradle b/rest-api-spec/build.gradle index cd69654aa35c4..05ed09e40f6bc 100644 --- a/rest-api-spec/build.gradle +++ b/rest-api-spec/build.gradle @@ -57,5 +57,5 @@ tasks.named("precommit").configure { tasks.named("yamlRestCompatTestTransform").configure({task -> task.skipTest("indices.sort/10_basic/Index Sort", "warning does not exist for compatibility") task.skipTest("search/330_fetch_fields/Test search rewrite", "warning does not exist for compatibility") - task.skipTestsByFilePattern("*create/*synthetic_source*.yml", "@UpdateForV9 -> tests do not pass after bumping API version to 9 [ES-9597]") + task.skipTest("indices.create/21_synthetic_source_stored/object param - nested object with stored array", "skip test to submit #113690") })