From 363143fa39822300534a8d4f80aa267c6b2ede59 Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Fri, 11 Oct 2024 21:10:28 +0300 Subject: [PATCH 01/14] Guard second doc parsing pass with index setting --- .../common/settings/IndexScopedSettings.java | 1 + .../elasticsearch/index/IndexSettings.java | 21 +++++++++++++++++++ .../index/mapper/DocumentParserContext.java | 2 +- 3 files changed, 23 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java b/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java index ad3d7d7f1c2ec..f04b7d6c160cb 100644 --- a/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java +++ b/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java @@ -186,6 +186,7 @@ public final class IndexScopedSettings extends AbstractScopedSettings { FieldMapper.SYNTHETIC_SOURCE_KEEP_INDEX_SETTING, IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_WRITE_SETTING, IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_READ_SETTING, + IndexSettings.SYNTHETIC_SOURCE_SECOND_DOC_PARSING_PASS_SETTING, // validate that built-in similarities don't get redefined Setting.groupSetting("index.similarity.", (s) -> { diff --git a/server/src/main/java/org/elasticsearch/index/IndexSettings.java b/server/src/main/java/org/elasticsearch/index/IndexSettings.java index e82f9eff7d5e0..835205e4f4e1c 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexSettings.java +++ b/server/src/main/java/org/elasticsearch/index/IndexSettings.java @@ -651,6 +651,13 @@ public Iterator> settings() { Property.Final ); + public static final Setting SYNTHETIC_SOURCE_SECOND_DOC_PARSING_PASS_SETTING = Setting.boolSetting( + "index.synthetic_source.enable_second_doc_parsing_pass", + false, + Property.IndexScope, + Property.Dynamic + ); + /** * Returns true if TSDB encoding is enabled. The default is true */ @@ -820,6 +827,7 @@ private void setRetentionLeaseMillis(final TimeValue retentionLease) { private volatile long mappingDimensionFieldsLimit; private volatile boolean skipIgnoredSourceWrite; private volatile boolean skipIgnoredSourceRead; + private volatile boolean syntheticSourceSecondDocParsingPassEnabled; /** * The maximum number of refresh listeners allows on this shard. @@ -980,6 +988,7 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti es87TSDBCodecEnabled = scopedSettings.get(TIME_SERIES_ES87TSDB_CODEC_ENABLED_SETTING); skipIgnoredSourceWrite = scopedSettings.get(IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_WRITE_SETTING); skipIgnoredSourceRead = scopedSettings.get(IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_READ_SETTING); + syntheticSourceSecondDocParsingPassEnabled = scopedSettings.get(SYNTHETIC_SOURCE_SECOND_DOC_PARSING_PASS_SETTING); scopedSettings.addSettingsUpdateConsumer( MergePolicyConfig.INDEX_COMPOUND_FORMAT_SETTING, @@ -1067,6 +1076,10 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti this::setSkipIgnoredSourceWrite ); scopedSettings.addSettingsUpdateConsumer(IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_READ_SETTING, this::setSkipIgnoredSourceRead); + scopedSettings.addSettingsUpdateConsumer( + SYNTHETIC_SOURCE_SECOND_DOC_PARSING_PASS_SETTING, + this::setSyntheticSourceSecondDocParsingPassEnabled + ); } private void setSearchIdleAfter(TimeValue searchIdleAfter) { @@ -1659,6 +1672,14 @@ private void setSkipIgnoredSourceRead(boolean value) { this.skipIgnoredSourceRead = value; } + private void setSyntheticSourceSecondDocParsingPassEnabled(boolean syntheticSourceSecondDocParsingPassEnabled) { + this.syntheticSourceSecondDocParsingPassEnabled = syntheticSourceSecondDocParsingPassEnabled; + } + + public boolean isSyntheticSourceSecondDocParsingPassEnabled() { + return syntheticSourceSecondDocParsingPassEnabled; + } + /** * The bounds for {@code @timestamp} on this index or * {@code null} if there are no bounds. diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java index ac236e5a7e5fd..407dca48ec1e3 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java @@ -371,7 +371,7 @@ public final Collection getIgnoredFieldsMiss * Applies to synthetic source only. */ public final DocumentParserContext maybeCloneForArray(Mapper mapper) throws IOException { - if (canAddIgnoredField() && mapper instanceof ObjectMapper) { + if (canAddIgnoredField() && mapper instanceof ObjectMapper && indexSettings().isSyntheticSourceSecondDocParsingPassEnabled()) { boolean isNested = mapper instanceof NestedObjectMapper; if ((inArrayScope == false && isNested == false) || (inArrayScope && isNested)) { DocumentParserContext subcontext = switchParser(parser()); From aba471e7d8a047fe58134d98317b58a9cb726aaf Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Fri, 11 Oct 2024 21:34:20 +0300 Subject: [PATCH 02/14] add test --- .../21_synthetic_source_stored.yml | 49 +++++++++++++++++++ .../elasticsearch/index/IndexSettings.java | 4 +- 2 files changed, 51 insertions(+), 2 deletions(-) diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/21_synthetic_source_stored.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/21_synthetic_source_stored.yml index dfe6c9820a16a..eab51427876aa 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/21_synthetic_source_stored.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/21_synthetic_source_stored.yml @@ -411,6 +411,55 @@ index param - nested array within array: - match: { hits.hits.0._source.path.to.some.3.id: [ 1000, 2000 ] } +--- +index param - nested array within array - disabled second pass: + - requires: + cluster_features: ["mapper.synthetic_source_keep", "mapper.bwc_workaround_9_0"] + reason: requires tracking ignored source + + - do: + indices.create: + index: test + body: + settings: + index: + synthetic_source: + enable_second_doc_parsing_pass: false + mappings: + _source: + mode: synthetic + properties: + name: + type: keyword + path: + properties: + to: + properties: + some: + synthetic_source_keep: arrays + properties: + id: + type: integer + + - do: + bulk: + index: test + refresh: true + body: + - '{ "create": { } }' + - '{ "name": "A", "path": [ { "to": [ { "some" : [ { "id": 10 }, { "id": [1, 3, 2] } ] }, { "some": { "id": 100 } } ] }, { "to": { "some": { "id": [1000, 2000] } } } ] }' + - match: { errors: false } + + - do: + search: + index: test + sort: name + - match: { hits.hits.0._source.name: A } + - length: { hits.hits.0._source.path.to.some: 2} + - match: { hits.hits.0._source.path.to.some.0.id: 10 } + - match: { hits.hits.0._source.path.to.some.1.id: [ 1, 3, 2] } + + --- # 112156 stored field under object with store_array_source: diff --git a/server/src/main/java/org/elasticsearch/index/IndexSettings.java b/server/src/main/java/org/elasticsearch/index/IndexSettings.java index 835205e4f4e1c..14498f65bebcb 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexSettings.java +++ b/server/src/main/java/org/elasticsearch/index/IndexSettings.java @@ -652,8 +652,8 @@ public Iterator> settings() { ); public static final Setting SYNTHETIC_SOURCE_SECOND_DOC_PARSING_PASS_SETTING = Setting.boolSetting( - "index.synthetic_source.enable_second_doc_parsing_pass", - false, + "index.synthetic_source.enable_second_doc_parsing_pass", + true, Property.IndexScope, Property.Dynamic ); From eeea953dabdfeb3256250a5f6c028ce8db4dda82 Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Fri, 11 Oct 2024 22:07:59 +0300 Subject: [PATCH 03/14] updates --- .../elasticsearch/index/mapper/DocumentParserContext.java | 7 ++++++- .../xpack/ccr/action/TransportResumeFollowActionTests.java | 1 + 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java index 407dca48ec1e3..2eec14bd1a8d6 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java @@ -111,6 +111,7 @@ public int get() { private final Set ignoredFields; private final List ignoredFieldValues; private final List ignoredFieldsMissingValues; + private final boolean inArrayScopeEnabled; private boolean inArrayScope; private final Map> dynamicMappers; @@ -143,6 +144,7 @@ private DocumentParserContext( Set ignoreFields, List ignoredFieldValues, List ignoredFieldsWithNoSource, + boolean inArrayScopeEnabled, boolean inArrayScope, Map> dynamicMappers, Map dynamicObjectMappers, @@ -164,6 +166,7 @@ private DocumentParserContext( this.ignoredFields = ignoreFields; this.ignoredFieldValues = ignoredFieldValues; this.ignoredFieldsMissingValues = ignoredFieldsWithNoSource; + this.inArrayScopeEnabled = inArrayScopeEnabled; this.inArrayScope = inArrayScope; this.dynamicMappers = dynamicMappers; this.dynamicObjectMappers = dynamicObjectMappers; @@ -188,6 +191,7 @@ private DocumentParserContext(ObjectMapper parent, ObjectMapper.Dynamic dynamic, in.ignoredFields, in.ignoredFieldValues, in.ignoredFieldsMissingValues, + in.inArrayScopeEnabled, in.inArrayScope, in.dynamicMappers, in.dynamicObjectMappers, @@ -219,6 +223,7 @@ protected DocumentParserContext( new HashSet<>(), new ArrayList<>(), new ArrayList<>(), + mappingParserContext.getIndexSettings().isSyntheticSourceSecondDocParsingPassEnabled(), false, new HashMap<>(), new HashMap<>(), @@ -371,7 +376,7 @@ public final Collection getIgnoredFieldsMiss * Applies to synthetic source only. */ public final DocumentParserContext maybeCloneForArray(Mapper mapper) throws IOException { - if (canAddIgnoredField() && mapper instanceof ObjectMapper && indexSettings().isSyntheticSourceSecondDocParsingPassEnabled()) { + if (canAddIgnoredField() && mapper instanceof ObjectMapper && inArrayScopeEnabled) { boolean isNested = mapper instanceof NestedObjectMapper; if ((inArrayScope == false && isNested == false) || (inArrayScope && isNested)) { DocumentParserContext subcontext = switchParser(parser()); diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowActionTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowActionTests.java index b4be0b33a464e..f88c6a69c4df0 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowActionTests.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowActionTests.java @@ -332,6 +332,7 @@ public void testDynamicIndexSettingsAreClassified() { replicatedSettings.add(IndexSettings.MAX_SHINGLE_DIFF_SETTING); replicatedSettings.add(IndexSettings.TIME_SERIES_END_TIME); replicatedSettings.add(IndexSettings.PREFER_ILM_SETTING); + replicatedSettings.add(IndexSettings.SYNTHETIC_SOURCE_SECOND_DOC_PARSING_PASS_SETTING); replicatedSettings.add(IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_READ_SETTING); replicatedSettings.add(IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_WRITE_SETTING); From c55f8c909707b1334ed52150a27bcfb80d7f0f46 Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Fri, 11 Oct 2024 22:13:14 +0300 Subject: [PATCH 04/14] updates --- server/src/main/java/org/elasticsearch/index/IndexSettings.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/index/IndexSettings.java b/server/src/main/java/org/elasticsearch/index/IndexSettings.java index 14498f65bebcb..953a389ea5ff7 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexSettings.java +++ b/server/src/main/java/org/elasticsearch/index/IndexSettings.java @@ -652,7 +652,7 @@ public Iterator> settings() { ); public static final Setting SYNTHETIC_SOURCE_SECOND_DOC_PARSING_PASS_SETTING = Setting.boolSetting( - "index.synthetic_source.enable_second_doc_parsing_pass", + "index.synthetic_source.enable_second_doc_parsing_pass", true, Property.IndexScope, Property.Dynamic From a0e09c6cd088b9bbb14b866fbef29a76d743999d Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Mon, 14 Oct 2024 10:18:11 +0300 Subject: [PATCH 05/14] merge --- .../java/org/elasticsearch/index/IndexSettings.java | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/server/src/main/java/org/elasticsearch/index/IndexSettings.java b/server/src/main/java/org/elasticsearch/index/IndexSettings.java index ce2ed342622fa..347b44a22e7c0 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexSettings.java +++ b/server/src/main/java/org/elasticsearch/index/IndexSettings.java @@ -652,6 +652,13 @@ public Iterator> settings() { Property.Final ); + public static final Setting SYNTHETIC_SOURCE_SECOND_DOC_PARSING_PASS_SETTING = Setting.boolSetting( + "index.synthetic_source.enable_second_doc_parsing_pass", + true, + Property.IndexScope, + Property.Dynamic + ); + /** * Returns true if TSDB encoding is enabled. The default is true */ @@ -1072,6 +1079,10 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti this::setSkipIgnoredSourceWrite ); scopedSettings.addSettingsUpdateConsumer(IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_READ_SETTING, this::setSkipIgnoredSourceRead); + scopedSettings.addSettingsUpdateConsumer( + SYNTHETIC_SOURCE_SECOND_DOC_PARSING_PASS_SETTING, + this::setSyntheticSourceSecondDocParsingPassEnabled + ); } private void setSearchIdleAfter(TimeValue searchIdleAfter) { From 8fbddca607866bb624c767aefad534821e9b4c8e Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Mon, 14 Oct 2024 16:53:02 +0300 Subject: [PATCH 06/14] Add documentation for passthrough field type --- .../data-streams/set-up-tsds.asciidoc | 21 +- docs/reference/data-streams/tsds.asciidoc | 12 +- .../mapping/params/subobjects.asciidoc | 1 + docs/reference/mapping/types.asciidoc | 15 +- .../mapping/types/passthrough.asciidoc | 217 ++++++++++++++++++ .../index/mapper/PassThroughObjectMapper.java | 3 - 6 files changed, 246 insertions(+), 23 deletions(-) create mode 100644 docs/reference/mapping/types/passthrough.asciidoc diff --git a/docs/reference/data-streams/set-up-tsds.asciidoc b/docs/reference/data-streams/set-up-tsds.asciidoc index 3a483ac351180..d082a9c4eebeb 100644 --- a/docs/reference/data-streams/set-up-tsds.asciidoc +++ b/docs/reference/data-streams/set-up-tsds.asciidoc @@ -121,7 +121,8 @@ naming scheme]. * Specify a mapping that defines your dimensions and metrics: ** One or more <> with a `time_series_dimension` value of `true`. - At least one of these dimensions must be a plain `keyword` field. + Alternatively, one or more <> fields configured as dimension containers, + provided that they will contain at least one sub-field (mapped statically or dynamically). ** One or more <>, marked using the `time_series_metric` mapping parameter. @@ -203,10 +204,9 @@ DELETE _ilm/policy/my-weather-sensor-lifecycle-policy Documents in a TSDS must include: * A `@timestamp` field -* One or more dimension fields. At least one dimension must be a `keyword` field -that matches the `index.routing_path` index setting, if specified. If not specified -explicitly, `index.routing_path` is set automatically to whichever mappings have - `time_series_dimension` set to `true`. +* One or more dimension fields. At least one dimension must match the `index.routing_path` index setting, +if specified. If not specified explicitly, `index.routing_path` is set automatically to whichever mappings have +`time_series_dimension` set to `true`. To automatically create your TSDS, submit an indexing request that targets the TSDS's name. This name must match one of your index template's @@ -285,13 +285,12 @@ POST metrics-weather_sensors-dev/_rollover Configuring a TSDS via an index template that uses component templates is a bit more complicated. Typically with component templates mappings and settings get scattered across multiple component templates. -When configuring the `index.mode` setting in a component template, the `index.routing_path` setting needs to -be defined in the same component template. Additionally the fields mentioned in the `index.routing_path` -also need to be defined in the same component template with the `time_series_dimension` attribute enabled. +If the `index.routing_path` is defined, the fields it references need to be defined in the same component +template with the `time_series_dimension` attribute enabled. -The reasons for this is that each component template needs to be valid on its own and the time series index mode -requires the `index.routing_path` setting. When configuring the `index.mode` setting in an index template, the `index.routing_path` setting is configured automatically. It is derived from -the field mappings with `time_series_dimension` attribute enabled. +The reasons for this is that each component template needs to be valid on its own. When configuring the +`index.mode` setting in an index template, the `index.routing_path` setting is configured automatically. +It is derived from the field mappings with `time_series_dimension` attribute enabled. [discrete] [[set-up-tsds-whats-next]] diff --git a/docs/reference/data-streams/tsds.asciidoc b/docs/reference/data-streams/tsds.asciidoc index 01573658c33d0..6dc559732bf90 100644 --- a/docs/reference/data-streams/tsds.asciidoc +++ b/docs/reference/data-streams/tsds.asciidoc @@ -109,7 +109,10 @@ parameter: * <> * <> -For a flattened field, use the `time_series_dimensions` parameter to configure an array of fields as dimensions. For details refer to <>. +For a flattened field, use the `time_series_dimensions` parameter to configure an array of fields as dimensions. +For details refer to <>. + +Dimension definitions can be simplified through <> fields. [discrete] [[time-series-metric]] @@ -294,12 +297,15 @@ When you create the matching index template for a TSDS, you must specify one or more dimensions in the `index.routing_path` setting. Each document in a TSDS must contain one or more dimensions that match the `index.routing_path` setting. -Dimensions in the `index.routing_path` setting must be plain `keyword` fields. The `index.routing_path` setting accepts wildcard patterns (for example `dim.*`) and can dynamically match new fields. However, {es} will reject any mapping -updates that add scripted, runtime, or non-dimension, non-`keyword` fields that +updates that add scripted, runtime, or non-dimension fields that match the `index.routing_path` value. +<> fields may be configured +as dimension containers. In this case, their sub-fields get included to the +routing path automatically. + TSDS documents don't support a custom `_routing` value. Similarly, you can't require a `_routing` value in mappings for a TSDS. diff --git a/docs/reference/mapping/params/subobjects.asciidoc b/docs/reference/mapping/params/subobjects.asciidoc index b0a5d3817c332..ff91f07cfb359 100644 --- a/docs/reference/mapping/params/subobjects.asciidoc +++ b/docs/reference/mapping/params/subobjects.asciidoc @@ -111,6 +111,7 @@ PUT my-index-000001/_doc/metric_1 The `subobjects` setting for existing fields and the top-level mapping definition cannot be updated. +[[subobjects-auto-flattening]] ==== Auto-flattening object mappings It is generally recommended to define the properties of an object that is configured with `subobjects: false` with dotted field names diff --git a/docs/reference/mapping/types.asciidoc b/docs/reference/mapping/types.asciidoc index 7e2e7083fa70b..82a974fe69c1b 100644 --- a/docs/reference/mapping/types.asciidoc +++ b/docs/reference/mapping/types.asciidoc @@ -35,12 +35,13 @@ Dates:: Date types, including <> and [[object-types]] ==== Objects and relational types -<>:: A JSON object. -<>:: An entire JSON object as a single field value. -<>:: A JSON object that preserves the relationship - between its subfields. -<>:: Defines a parent/child relationship for documents - in the same index. +<>:: A JSON object. +<>:: An entire JSON object as a single field value. +<>:: A JSON object that preserves the relationship + between its subfields. +<>:: Defines a parent/child relationship for documents + in the same index. +<>:: Provides aliases for sub-fields at the same level. [discrete] @@ -167,6 +168,8 @@ include::types/numeric.asciidoc[] include::types/object.asciidoc[] +include::types/passthrough.asciidoc[] + include::types/percolator.asciidoc[] include::types/point.asciidoc[] diff --git a/docs/reference/mapping/types/passthrough.asciidoc b/docs/reference/mapping/types/passthrough.asciidoc new file mode 100644 index 0000000000000..980b98b7dc3e9 --- /dev/null +++ b/docs/reference/mapping/types/passthrough.asciidoc @@ -0,0 +1,217 @@ +[[passthrough]] +=== Pass-through object field type +++++ +Pass-through object +++++ + +Pass-through objects extend the functionality of <> by allowing to access +their subfields without including the name of the pass-through object as prefix. For instance: + +[source,console] +-------------------------------------------------- +PUT my-index-000001 +{ + "mappings": { + "properties": { + "attributes": { + "type": "passthrough", <1> + "priority": 10, + "properties": { + "id": { + "type": "keyword" + } + } + } + } + } +} + +PUT my-index-000001/_doc/1 +{ + "attributes" : { <2> + "id": "foo", + "zone": 10 + } +} + +GET my-index-000001/_search +{ + "query": { + "bool": { + "must": [ + { "match": { "id": "foo" }}, <3> + { "match": { "zone": 10 }} + ] + } + } +} + +GET my-index-000001/_search +{ + "query": { + "bool": { + "must": [ + { "match": { "attributes.id": "foo" }}, <4> + { "match": { "attributes.zone": 10 }} + ] + } + } +} + +-------------------------------------------------- + +<1> An object is defined as pass-through. Its priority (required) is used for conflict resolution. +<2> Object contents get indexed as usual, including dynamic mappings. +<3> Sub-fields can be referenced in queries as if they're defined at the same level with the object. +<4> Sub-fields can also be referenced including the object nam as prefix. + +[[passthrough-conflicts]] +==== Conflict resolution + +It's possible for conflicting names to arise, for fields that are defined within different scopes: + + 1. A pass-through object is defined next to a field that has the same name as one of the pass-through object + sub-fields, e.g. + +[source,console] +-------------------------------------------------- +PUT my-index-000001/_doc/1 +{ + "attributes" : { + "id": "foo" + }, + "id": "bar" +} +-------------------------------------------------- + + In this case, references to `id` point to the field at the root level, while field `attributes.id` + can only be accessed using the full path. + + 1. Two (or more) pass-through objects are defined within the same object and contain fields with the same name, e.g. + +[source,console] +-------------------------------------------------- +PUT my-index-000002 +{ + "mappings": { + "properties": { + "attributes": { + "type": "passthrough", + "priority": 10, + "properties": { + "id": { + "type": "keyword" + } + } + }, + "resource-attributes": { + "type": "passthrough", + "priority": 5, + "properties": { + "id": { + "type": "keyword" + } + } + } + } + } +} +-------------------------------------------------- + +In this case, param `priority` is used for conflict resolution, with the higher values taking precedence. In the +example above, `attributes` has higher priority than `resource-attributes`, so references to `id` point to the field +within `attributes`. `resource-attributes.id` can still be accessed using its full path. + +[[passthrough-dimensions]] +==== Defining sub-fields as time-series dimensions + +It is possible to configure a pass-through field as a container for <>. +In this case, all sub-fields get annotated with the same parameter under the covers,and they're also +included in <> and <> calculations, thus simplifying +the <> setup: + +[source,console] +-------------------------------------------------- +PUT _index_template/my-metrics +{ + "index_patterns": ["metrics-*"], + "data_stream": { }, + "template": { + "settings": { + "index.mode": "time_series" + }, + "mappings": { + "properties": { + "attributes": { + "type": "passthrough", + "priority": 10, + "time_series_dimension": true, + "properties": { + "host": { + "type": "keyword" + } + } + }, + "cpu": { + "type": "integer", + "time_series_metric": "counter" + } + } + } + } +} + +POST metrics-test/_doc +{ + "@timestamp": "2020-01-01T00:00:00.000Z", + "attributes" : { + "host": "foo", + "zone": "bar" + }, + "cpu": 10 +} +-------------------------------------------------- +// TEST[skip: The @timestamp value won't match an accepted range in the TSDS] + +In the example above, `attributes` is defined as a dimension container. Its sub-fields `host` (static) and `zone` +(dynamic) get included in the routing path and tsid, and can be referenced in queries without the `attributes.` prefix. + +[[passthrough-flattening]] +==== Sub-field auto-flattening + +Pass-through fields apply <> to sub-fields by default, to reduce dynamic +mapping conflicts. As a consequence, no sub-object definitions are allowed within pass-through fields. + +[[passthrough-params]] +==== Parameters for `passthrough` fields + +The following parameters are accepted by `passthrough` fields: + +[horizontal] + +<>:: + + (Required) used for naming conflict resolution between pass-through fields. The field with the highest value wins. + Accepts non-negative integer values. + +<>:: + + Whether or not to treat sub-fields as <>. + Accepts `false` (default) or `true`. + +<>:: + + Whether or not new `properties` should be added dynamically to an existing object. + Accepts `true` (default), `runtime`, `false` and `strict`. + +<>:: + + Whether the JSON value given for the object field should be parsed and indexed (`true`, default) + or completely ignored (`false`). + +<>:: + + The fields within the object, which can be of any <>, including `object`. + New properties may be added to an existing object. + +IMPORTANT: If you need to index arrays of objects instead of single objects, read <> first. diff --git a/server/src/main/java/org/elasticsearch/index/mapper/PassThroughObjectMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/PassThroughObjectMapper.java index 80f845d626a2f..decc6d40a2f8e 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/PassThroughObjectMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/PassThroughObjectMapper.java @@ -34,9 +34,6 @@ * In case different pass-through objects contain subfields with the same name (excluding the pass-through prefix), their aliases conflict. * To resolve this, the pass-through spec specifies which object takes precedence through required parameter "priority"; non-negative * integer values are accepted, with the highest priority value winning in case of conflicting aliases. - * - * Note that this is an experimental, undocumented mapper type, currently intended for prototyping purposes only. - * It has not been vetted for use in production systems. */ public class PassThroughObjectMapper extends ObjectMapper { public static final String CONTENT_TYPE = "passthrough"; From c46f72f55b301e091d885778492fc205bfc315ea Mon Sep 17 00:00:00 2001 From: Kostas Krikellas <131142368+kkrik-es@users.noreply.github.com> Date: Mon, 14 Oct 2024 21:57:37 +0300 Subject: [PATCH 07/14] Apply suggestions from code review Co-authored-by: Felix Barnsteiner --- docs/reference/mapping/types.asciidoc | 2 +- docs/reference/mapping/types/passthrough.asciidoc | 13 +++++++------ 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/docs/reference/mapping/types.asciidoc b/docs/reference/mapping/types.asciidoc index 82a974fe69c1b..babe4f508b5f0 100644 --- a/docs/reference/mapping/types.asciidoc +++ b/docs/reference/mapping/types.asciidoc @@ -41,7 +41,7 @@ Dates:: Date types, including <> and between its subfields. <>:: Defines a parent/child relationship for documents in the same index. -<>:: Provides aliases for sub-fields at the same level. +<>:: Provides aliases for sub-fields at the same level. [discrete] diff --git a/docs/reference/mapping/types/passthrough.asciidoc b/docs/reference/mapping/types/passthrough.asciidoc index 980b98b7dc3e9..5b06ace364e68 100644 --- a/docs/reference/mapping/types/passthrough.asciidoc +++ b/docs/reference/mapping/types/passthrough.asciidoc @@ -62,8 +62,8 @@ GET my-index-000001/_search <1> An object is defined as pass-through. Its priority (required) is used for conflict resolution. <2> Object contents get indexed as usual, including dynamic mappings. -<3> Sub-fields can be referenced in queries as if they're defined at the same level with the object. -<4> Sub-fields can also be referenced including the object nam as prefix. +<3> Sub-fields can be referenced in queries as if they're defined at the root level. +<4> Sub-fields can also be referenced including the object name as prefix. [[passthrough-conflicts]] ==== Conflict resolution @@ -104,9 +104,9 @@ PUT my-index-000002 } } }, - "resource-attributes": { + "resource.attributes": { "type": "passthrough", - "priority": 5, + "priority": 20, "properties": { "id": { "type": "keyword" @@ -126,7 +126,7 @@ within `attributes`. `resource-attributes.id` can still be accessed using its fu ==== Defining sub-fields as time-series dimensions It is possible to configure a pass-through field as a container for <>. -In this case, all sub-fields get annotated with the same parameter under the covers,and they're also +In this case, all sub-fields get annotated with the same parameter under the covers, and they're also included in <> and <> calculations, thus simplifying the <> setup: @@ -134,7 +134,8 @@ the <> setup: -------------------------------------------------- PUT _index_template/my-metrics { - "index_patterns": ["metrics-*"], + "index_patterns": ["metrics-mymetrics-*"], + "priority": 200, "data_stream": { }, "template": { "settings": { From 4a5542ec12bf0f9c581dd890449978260a8febab Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Mon, 14 Oct 2024 21:59:46 +0300 Subject: [PATCH 08/14] updates --- docs/reference/mapping/types/passthrough.asciidoc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/reference/mapping/types/passthrough.asciidoc b/docs/reference/mapping/types/passthrough.asciidoc index 5b06ace364e68..724581d7f1be3 100644 --- a/docs/reference/mapping/types/passthrough.asciidoc +++ b/docs/reference/mapping/types/passthrough.asciidoc @@ -119,8 +119,8 @@ PUT my-index-000002 -------------------------------------------------- In this case, param `priority` is used for conflict resolution, with the higher values taking precedence. In the -example above, `attributes` has higher priority than `resource-attributes`, so references to `id` point to the field -within `attributes`. `resource-attributes.id` can still be accessed using its full path. +example above, `attributes` has higher priority than `resource.attributes`, so references to `id` point to the field +within `attributes`. `resource.attributes.id` can still be accessed using its full path. [[passthrough-dimensions]] ==== Defining sub-fields as time-series dimensions From 8f795d4f5603fe10877a0436498771116e643f28 Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Mon, 14 Oct 2024 22:24:56 +0300 Subject: [PATCH 09/14] updates --- docs/reference/mapping/types/passthrough.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/reference/mapping/types/passthrough.asciidoc b/docs/reference/mapping/types/passthrough.asciidoc index 724581d7f1be3..97dc480005fe2 100644 --- a/docs/reference/mapping/types/passthrough.asciidoc +++ b/docs/reference/mapping/types/passthrough.asciidoc @@ -162,7 +162,7 @@ PUT _index_template/my-metrics } } -POST metrics-test/_doc +POST metrics-mymetrics-test/_doc { "@timestamp": "2020-01-01T00:00:00.000Z", "attributes" : { From 442d3f174dda5d531eb981cf6de4a0985db0efea Mon Sep 17 00:00:00 2001 From: Kostas Krikellas <131142368+kkrik-es@users.noreply.github.com> Date: Tue, 15 Oct 2024 09:38:56 +0300 Subject: [PATCH 10/14] Update docs/reference/mapping/types/passthrough.asciidoc Co-authored-by: Felix Barnsteiner --- docs/reference/mapping/types/passthrough.asciidoc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/reference/mapping/types/passthrough.asciidoc b/docs/reference/mapping/types/passthrough.asciidoc index 97dc480005fe2..a2bdeb0636b60 100644 --- a/docs/reference/mapping/types/passthrough.asciidoc +++ b/docs/reference/mapping/types/passthrough.asciidoc @@ -119,8 +119,8 @@ PUT my-index-000002 -------------------------------------------------- In this case, param `priority` is used for conflict resolution, with the higher values taking precedence. In the -example above, `attributes` has higher priority than `resource.attributes`, so references to `id` point to the field -within `attributes`. `resource.attributes.id` can still be accessed using its full path. +example above, `resource.attributes` has higher priority than `attributes`, so references to `id` point to the field +within `resource.attributes`. `attributes.id` can still be accessed using its full path. [[passthrough-dimensions]] ==== Defining sub-fields as time-series dimensions From dead0e186420059c535c7a71115d83998781a779 Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Tue, 15 Oct 2024 09:40:12 +0300 Subject: [PATCH 11/14] address comment --- docs/reference/mapping/types/passthrough.asciidoc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/reference/mapping/types/passthrough.asciidoc b/docs/reference/mapping/types/passthrough.asciidoc index a2bdeb0636b60..181304c2606e9 100644 --- a/docs/reference/mapping/types/passthrough.asciidoc +++ b/docs/reference/mapping/types/passthrough.asciidoc @@ -148,7 +148,7 @@ PUT _index_template/my-metrics "priority": 10, "time_series_dimension": true, "properties": { - "host": { + "host.name": { "type": "keyword" } } @@ -166,7 +166,7 @@ POST metrics-mymetrics-test/_doc { "@timestamp": "2020-01-01T00:00:00.000Z", "attributes" : { - "host": "foo", + "host.name": "foo", "zone": "bar" }, "cpu": 10 From b0d97d4de8a0178301aafabd37ab2e67b7890028 Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Tue, 15 Oct 2024 09:40:34 +0300 Subject: [PATCH 12/14] address comment --- docs/reference/mapping/types/passthrough.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/reference/mapping/types/passthrough.asciidoc b/docs/reference/mapping/types/passthrough.asciidoc index 181304c2606e9..202c9b96f6cf4 100644 --- a/docs/reference/mapping/types/passthrough.asciidoc +++ b/docs/reference/mapping/types/passthrough.asciidoc @@ -174,7 +174,7 @@ POST metrics-mymetrics-test/_doc -------------------------------------------------- // TEST[skip: The @timestamp value won't match an accepted range in the TSDS] -In the example above, `attributes` is defined as a dimension container. Its sub-fields `host` (static) and `zone` +In the example above, `attributes` is defined as a dimension container. Its sub-fields `host.name` (static) and `zone` (dynamic) get included in the routing path and tsid, and can be referenced in queries without the `attributes.` prefix. [[passthrough-flattening]] From db68feff0129a8ef25613c0411ab494a34c19a1e Mon Sep 17 00:00:00 2001 From: Kostas Krikellas <131142368+kkrik-es@users.noreply.github.com> Date: Tue, 15 Oct 2024 09:57:39 +0300 Subject: [PATCH 13/14] Update docs/reference/mapping/types/passthrough.asciidoc Co-authored-by: Felix Barnsteiner --- docs/reference/mapping/types/passthrough.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/reference/mapping/types/passthrough.asciidoc b/docs/reference/mapping/types/passthrough.asciidoc index 202c9b96f6cf4..3ada7e64b6d6c 100644 --- a/docs/reference/mapping/types/passthrough.asciidoc +++ b/docs/reference/mapping/types/passthrough.asciidoc @@ -190,7 +190,7 @@ The following parameters are accepted by `passthrough` fields: [horizontal] -<>:: +<>:: (Required) used for naming conflict resolution between pass-through fields. The field with the highest value wins. Accepts non-negative integer values. From 9da1ae06c1a28487195b6cf04ce9d9fd559de43d Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Tue, 15 Oct 2024 11:03:42 +0300 Subject: [PATCH 14/14] address comment --- docs/reference/mapping/types/passthrough.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/reference/mapping/types/passthrough.asciidoc b/docs/reference/mapping/types/passthrough.asciidoc index 3ada7e64b6d6c..53b2cc5ed260b 100644 --- a/docs/reference/mapping/types/passthrough.asciidoc +++ b/docs/reference/mapping/types/passthrough.asciidoc @@ -195,7 +195,7 @@ The following parameters are accepted by `passthrough` fields: (Required) used for naming conflict resolution between pass-through fields. The field with the highest value wins. Accepts non-negative integer values. -<>:: +<>:: Whether or not to treat sub-fields as <>. Accepts `false` (default) or `true`.