diff --git a/docs/changelog/132131.yaml b/docs/changelog/132131.yaml new file mode 100644 index 0000000000000..a1dde51ab64c9 --- /dev/null +++ b/docs/changelog/132131.yaml @@ -0,0 +1,7 @@ +pr: 132131 +summary: Updating `TransportSimulateIndexTemplateAction.resolveTemplate()` to account + for data stream overrides +area: Indices APIs +type: bug +issues: + - 131425 diff --git a/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/80_ingest_simulate.yml b/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/80_ingest_simulate.yml index 94d9622334c91..e1c200b23576b 100644 --- a/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/80_ingest_simulate.yml +++ b/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/80_ingest_simulate.yml @@ -2085,3 +2085,100 @@ setup: - match: { docs.0.doc._index: "subobject-index-1" } - match: { docs.0.doc._source.a\.b: "some text" } - not_exists: docs.0.doc.error + +--- +"Simulate ingest with data stream with mapping overrides": + - skip: + features: headers + + - do: + indices.put_index_template: + name: test + body: + index_patterns: test* + template: + lifecycle: + data_retention: "7d" + data_stream: {} + + - do: + indices.create_data_stream: + name: test + - is_true: acknowledged + + - do: + cluster.health: + wait_for_status: yellow + + - do: + indices.put_data_stream_mappings: + name: test + body: + properties: + foo: + type: boolean + + - match: { data_streams.0.applied_to_data_stream: true } + + # For an ordinary simulate request with no overrides, we fetch the mapping from the write index, and do not take the + # data stream mapping override into account. So the foo field is unmapped, and we can write text to it. + - do: + headers: + Content-Type: application/json + simulate.ingest: + index: test + body: > + { + "docs": [ + { + "_id": "asdf", + "_source": { + "@timestamp": 1234, + "foo": "bar" + } + } + ] + } + - length: { docs: 1 } + - match: { docs.0.doc._index: "test" } + - match: { docs.0.doc._source.foo: "bar" } + - not_exists: docs.0.doc.error + + # If template overrides are given, we go to the data stream's template and mapping override (even if the given + # template overrides are irrelevant as is the case in this test). Now we see that the foo field is mapped as a + # boolean, so we get a validation error when trying to write text to that field. + - do: + headers: + Content-Type: application/json + simulate.ingest: + index: test + body: > + { + "docs": [ + { + "_id": "asdf", + "_source": { + "@timestamp": 1234, + "foo": "bar" + } + } + ], + "component_template_substitutions": { + "mappings_template": { + "template": { + "mappings": { + "dynamic": "true", + "properties": { + "foo": { + "type": "keyword" + } + } + } + } + } + } + } + - length: { docs: 1 } + - match: { docs.0.doc._index: "test" } + - match: { docs.0.doc._source.foo: "bar" } + - match: { docs.0.doc.error.type: "document_parsing_exception" } diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.simulate_index_template/10_basic.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.simulate_index_template/10_basic.yml index f62e06d43b857..2c40b6bffdd7a 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.simulate_index_template/10_basic.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.simulate_index_template/10_basic.yml @@ -250,3 +250,55 @@ - match: {template.lifecycle.data_retention: "7d"} - is_true: template.lifecycle.rollover - match: {overlapping: []} + +--- +"Simulate index template with data stream with mapping and setting overrides": + - requires: + cluster_features: [ "logs_stream" ] + reason: requires setting 'logs_stream' to get or set data stream mappings + + - do: + indices.put_index_template: + name: test + body: + index_patterns: test* + template: + lifecycle: + data_retention: "7d" + data_stream: {} + + - do: + indices.create_data_stream: + name: test + - is_true: acknowledged + + - do: + cluster.health: + wait_for_status: yellow + + - do: + indices.put_data_stream_mappings: + name: test + body: + properties: + foo: + type: keyword + + - match: { data_streams.0.applied_to_data_stream: true } + + - do: + indices.put_data_stream_settings: + name: test + body: + index: + refresh_interval: "47s" + + - match: { data_streams.0.applied_to_data_stream: true } + + - do: + indices.simulate_index_template: + name: test + include_defaults: true + + - match: {template.mappings.properties.foo.type: "keyword"} + - match: {template.settings.index.refresh_interval: "47s"} diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java index ca0ce743799a1..af268cffffbd0 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java @@ -33,6 +33,7 @@ import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.core.Nullable; import org.elasticsearch.core.UpdateForV10; import org.elasticsearch.index.IndexService; import org.elasticsearch.index.IndexSettingProvider; @@ -48,7 +49,9 @@ import org.elasticsearch.transport.TransportService; import org.elasticsearch.xcontent.NamedXContentRegistry; +import java.io.IOException; import java.time.Instant; +import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -158,7 +161,6 @@ protected void localClusterStateOperation( listener.onResponse(new SimulateIndexTemplateResponse(null, null)); return; } - final ProjectMetadata tempProjectMetadata = resolveTemporaryState(matchingTemplate, request.getIndexName(), projectWithTemplate); ComposableIndexTemplate templateV2 = tempProjectMetadata.templatesV2().get(matchingTemplate); assert templateV2 != null : "the matched template must exist"; @@ -167,6 +169,7 @@ protected void localClusterStateOperation( matchingTemplate, request.getIndexName(), projectWithTemplate, + state.metadata().dataStreams().get(request.getIndexName()), isDslOnlyMode, xContentRegistry, indicesService, @@ -233,22 +236,34 @@ public static ProjectMetadata resolveTemporaryState( /** * Take a template and index name as well as state where the template exists, and return a final * {@link Template} that represents all the resolved Settings, Mappings, Aliases and Lifecycle + * + * @param matchingTemplate + * @param indexName + * @param simulatedProject + * @param dataStream Used to get any data stream mapping or settings overrides to merge into the returned template + * @param isDslOnlyMode + * @param xContentRegistry + * @param indicesService + * @param systemIndices + * @param indexSettingProviders + * @return + * @throws Exception */ public static Template resolveTemplate( final String matchingTemplate, final String indexName, final ProjectMetadata simulatedProject, + @Nullable final DataStream dataStream, final boolean isDslOnlyMode, final NamedXContentRegistry xContentRegistry, final IndicesService indicesService, final SystemIndices systemIndices, Set indexSettingProviders ) throws Exception { - Settings templateSettings = resolveSettings(simulatedProject, matchingTemplate); - List> resolvedAliases = MetadataIndexTemplateService.resolveAliases(simulatedProject, matchingTemplate); ComposableIndexTemplate template = simulatedProject.templatesV2().get(matchingTemplate); + Settings templateSettings = collectSettings(simulatedProject, dataStream, matchingTemplate, template); // create the index with dummy settings in the cluster state so we can parse and validate the aliases Settings.Builder dummySettings = Settings.builder() .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()) @@ -256,21 +271,7 @@ public static Template resolveTemplate( .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) .put(IndexMetadata.SETTING_INDEX_UUID, UUIDs.randomBase64UUID()); - /* - * If the index name doesn't look like a data stream backing index, then MetadataCreateIndexService.collectV2Mappings() won't - * include data stream specific mappings in its response. - */ - String simulatedIndexName = template.getDataStreamTemplate() != null - && indexName.startsWith(DataStream.BACKING_INDEX_PREFIX) == false - ? DataStream.getDefaultBackingIndexName(indexName, 1) - : indexName; - List mappings = MetadataCreateIndexService.collectV2Mappings( - null, // empty request mapping as the user can't specify any explicit mappings via the simulate api - simulatedProject, - template, - xContentRegistry, - simulatedIndexName - ); + List mappings = collectMappings(simulatedProject, dataStream, template, indexName, xContentRegistry); // First apply settings sourced from index settings providers final var now = Instant.now(); @@ -360,6 +361,73 @@ public static Template resolveTemplate( ); } + /** + * This method collects the mappings from the given template, pulling them from the given simulatedProject. If the template is a data + * stream template and the given dataStream is not null, this method also appends any mapping overrides from the data stream itself. + * @param simulatedProject Used to fetch the component templates referenced from the template + * @param dataStream Used to fetch any mappings explicitly set on the data stream + * @param template The template matching the index, used to fetch mappings + * @param indexName The name of the index whose templates we are fetching + * @param xContentRegistry Used to parse mappings if necessary + * @return A list of matching mappings in ascending priority order + * @throws IOException + */ + private static List collectMappings( + ProjectMetadata simulatedProject, + @Nullable DataStream dataStream, + ComposableIndexTemplate template, + String indexName, + NamedXContentRegistry xContentRegistry + ) throws IOException { + /* + * If the index name doesn't look like a data stream backing index, then MetadataCreateIndexService.collectV2Mappings() won't + * include data stream specific mappings in its response. + */ + String simulatedIndexName = template.getDataStreamTemplate() != null + && indexName.startsWith(DataStream.BACKING_INDEX_PREFIX) == false + ? DataStream.getDefaultBackingIndexName(indexName, 1) + : indexName; + List mappings = MetadataCreateIndexService.collectV2Mappings( + null, // empty request mapping as the user can't specify any explicit mappings via the simulate api + simulatedProject, + template, + xContentRegistry, // This is never used since requestMappings is always null, but it is not marked explicitly as @Nullable + simulatedIndexName + ); + if (template.getDataStreamTemplate() != null && dataStream != null) { + CompressedXContent dataStreamMappingOverrides = dataStream.getMappings(); + if (ComposableIndexTemplate.EMPTY_MAPPINGS.equals(dataStreamMappingOverrides) == false) { + // The data stream has had mapping overrides applied, so include these + mappings = new ArrayList<>(mappings); + mappings.add(dataStreamMappingOverrides); + } + } + return mappings; + } + + /** + * This method collects the settings from the given template using the given simulatedProjcet. If dataStream is not null, it also merges + * in any settings overrides on the data stream itself. + * @param simulatedProject The project metadata used to get the settings + * @param dataStream If not null, this is used to get data stream settings overrides + * @param templateName The name of the template + * @param template The template, used to check whether this is a data strema template + * @return The settings to be used + */ + private static Settings collectSettings( + final ProjectMetadata simulatedProject, + @Nullable final DataStream dataStream, + String templateName, + ComposableIndexTemplate template + ) { + Settings templateSettings = resolveSettings(simulatedProject, templateName); + if (template.getDataStreamTemplate() != null && dataStream != null) { + // The data stream has had settings overrides applied, so include them + templateSettings = templateSettings.merge(dataStream.getSettings()); + } + return templateSettings; + } + private static IndexLongFieldRange getEventIngestedRange(String indexName, ProjectMetadata simulatedProject) { final IndexMetadata indexMetadata = simulatedProject.index(indexName); return indexMetadata == null ? IndexLongFieldRange.NO_SHARDS : indexMetadata.getEventIngestedRange(); diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateTemplateAction.java index 8fb0df63213ab..7701c9ace7b20 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateTemplateAction.java @@ -179,6 +179,7 @@ protected void localClusterStateOperation( matchingTemplate, temporaryIndexName, projectWithTemplate, + null, // we never match a data stream isDslOnlyMode, xContentRegistry, indicesService, diff --git a/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java b/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java index 1cd3a25b37140..7da12e05925af 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java @@ -270,6 +270,7 @@ private Tuple, Exception> validateMappings( matchingTemplate, request.index(), simulatedProjectMetadata, + project.dataStreams().get(request.index()), isDataStreamsLifecycleOnlyMode(clusterService.getSettings()), xContentRegistry, indicesService, diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateActionTests.java index c671bc4f18388..ad85ebf02f181 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateActionTests.java @@ -97,6 +97,7 @@ public boolean overrulesTemplateAndRequestSettings() { matchingTemplate, indexName, simulatedProject, + null, isDslOnlyMode, xContentRegistry(), indicesService,