diff --git a/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/80_ingest_simulate.yml b/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/80_ingest_simulate.yml index f3a977cd96f62..8d255b4965bf1 100644 --- a/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/80_ingest_simulate.yml +++ b/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/80_ingest_simulate.yml @@ -606,3 +606,204 @@ setup: - match: { docs.0.doc._source.foo: "FOO" } - match: { docs.0.doc.executed_pipelines: [] } - not_exists: docs.0.doc.error + +--- +"Test ingest simulate with component template substitutions for data streams": + # In this test, we make sure that when the index template is a data stream template, simulte ingest works the same whether the data stream + # has been created or not -- either way, we expect it to use the template rather than the data stream / index mappings and settings. + + - skip: + features: + - headers + - allowed_warnings + + - requires: + cluster_features: ["simulate.component.template.substitutions"] + reason: "ingest simulate component template substitutions added in 8.16" + + - do: + headers: + Content-Type: application/json + ingest.put_pipeline: + id: "foo-pipeline" + body: > + { + "processors": [ + { + "set": { + "field": "foo", + "value": true + } + } + ] + } + - match: { acknowledged: true } + + - do: + cluster.put_component_template: + name: mappings_template + body: + template: + mappings: + dynamic: strict + properties: + foo: + type: keyword + + - do: + cluster.put_component_template: + name: settings_template + body: + template: + settings: + index: + default_pipeline: "foo-pipeline" + + - do: + allowed_warnings: + - "index template [test-composable-1] has index patterns [foo*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [test-composable-1] will take precedence during new index creation" + indices.put_index_template: + name: test-composable-1 + body: + index_patterns: + - foo* + composed_of: + - mappings_template + - settings_template + + - do: + allowed_warnings: + - "index template [my-template1] has index patterns [simple-data-stream1] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template1] will take precedence during new index creation" + indices.put_index_template: + name: my-template1 + body: + index_patterns: [simple-data-stream1] + composed_of: + - mappings_template + - settings_template + data_stream: {} + + - do: + headers: + Content-Type: application/json + simulate.ingest: + index: simple-data-stream1 + body: > + { + "docs": [ + { + "_id": "asdf", + "_source": { + "@timestamp": 1234, + "foo": false + } + } + ], + "pipeline_substitutions": { + "foo-pipeline-2": { + "processors": [ + { + "set": { + "field": "foo", + "value": "FOO" + } + } + ] + } + }, + "component_template_substitutions": { + "settings_template": { + "template": { + "settings": { + "index": { + "default_pipeline": "foo-pipeline-2" + } + } + } + }, + "mappings_template": { + "template": { + "mappings": { + "dynamic": "strict", + "properties": { + "foo": { + "type": "keyword" + } + } + } + } + } + } + } + - length: { docs: 1 } + - match: { docs.0.doc._index: "simple-data-stream1" } + - match: { docs.0.doc._source.foo: "FOO" } + - match: { docs.0.doc.executed_pipelines: ["foo-pipeline-2"] } + - not_exists: docs.0.doc.error + + - do: + indices.create_data_stream: + name: simple-data-stream1 + - is_true: acknowledged + + - do: + cluster.health: + wait_for_status: yellow + + - do: + headers: + Content-Type: application/json + simulate.ingest: + index: simple-data-stream1 + body: > + { + "docs": [ + { + "_id": "asdf", + "_source": { + "@timestamp": 1234, + "foo": false + } + } + ], + "pipeline_substitutions": { + "foo-pipeline-2": { + "processors": [ + { + "set": { + "field": "foo", + "value": "FOO" + } + } + ] + } + }, + "component_template_substitutions": { + "settings_template": { + "template": { + "settings": { + "index": { + "default_pipeline": "foo-pipeline-2" + } + } + } + }, + "mappings_template": { + "template": { + "mappings": { + "dynamic": "strict", + "properties": { + "foo": { + "type": "keyword" + } + } + } + } + } + } + } + - length: { docs: 1 } + - match: { docs.0.doc._index: "simple-data-stream1" } + - match: { docs.0.doc._source.foo: "FOO" } + - match: { docs.0.doc.executed_pipelines: ["foo-pipeline-2"] } + - not_exists: docs.0.doc.error diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java index 3561a4d0e2cb4..fdced5fc18ac9 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java @@ -16,7 +16,6 @@ import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.metadata.AliasMetadata; -import org.elasticsearch.cluster.metadata.ComponentTemplate; import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.metadata.DataStreamLifecycle; @@ -157,8 +156,7 @@ protected void masterOperation( xContentRegistry, indicesService, systemIndices, - indexSettingProviders, - Map.of() + indexSettingProviders ); final Map> overlapping = new HashMap<>(); @@ -235,8 +233,7 @@ public static Template resolveTemplate( final NamedXContentRegistry xContentRegistry, final IndicesService indicesService, final SystemIndices systemIndices, - Set indexSettingProviders, - Map componentTemplateSubstitutions + Set indexSettingProviders ) throws Exception { var metadata = simulatedState.getMetadata(); Settings templateSettings = resolveSettings(simulatedState.metadata(), matchingTemplate); @@ -266,7 +263,6 @@ public static Template resolveTemplate( null, // empty request mapping as the user can't specify any explicit mappings via the simulate api simulatedState, matchingTemplate, - componentTemplateSubstitutions, xContentRegistry, simulatedIndexName ); diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateTemplateAction.java index af7a253b5a042..30bbad0b57df0 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateTemplateAction.java @@ -170,8 +170,7 @@ protected void masterOperation( xContentRegistry, indicesService, systemIndices, - indexSettingProviders, - Map.of() + indexSettingProviders ); if (request.includeDefaults()) { listener.onResponse( diff --git a/server/src/main/java/org/elasticsearch/action/bulk/TransportAbstractBulkAction.java b/server/src/main/java/org/elasticsearch/action/bulk/TransportAbstractBulkAction.java index 5eae1c660d7d0..8c6565e52daa7 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/TransportAbstractBulkAction.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/TransportAbstractBulkAction.java @@ -41,6 +41,7 @@ import org.elasticsearch.transport.TransportService; import java.io.IOException; +import java.util.HashMap; import java.util.Map; import java.util.Objects; import java.util.concurrent.Executor; @@ -180,12 +181,48 @@ protected void doRun() throws IOException { private boolean applyPipelines(Task task, BulkRequest bulkRequest, Executor executor, ActionListener listener) throws IOException { boolean hasIndexRequestsWithPipelines = false; - final Metadata metadata = clusterService.state().getMetadata(); - Map templateSubstitutions = bulkRequest.getComponentTemplateSubstitutions(); + final Metadata metadata; + Map componentTemplateSubstitutions = bulkRequest.getComponentTemplateSubstitutions(); + if (bulkRequest.isSimulated() && componentTemplateSubstitutions.isEmpty() == false) { + /* + * If this is a simulated request, and there are template substitutions, then we want to create and use a new metadata that has + * those templates. That is, we want to add the new templates (which will replace any that already existed with the same name), + * and remove the indices and data streams that are referred to from the bulkRequest so that we get settings from the templates + * rather than from the indices/data streams. + */ + Metadata.Builder simulatedMetadataBuilder = Metadata.builder(clusterService.state().getMetadata()); + if (componentTemplateSubstitutions.isEmpty() == false) { + Map updatedComponentTemplates = new HashMap<>(); + updatedComponentTemplates.putAll(clusterService.state().metadata().componentTemplates()); + updatedComponentTemplates.putAll(componentTemplateSubstitutions); + simulatedMetadataBuilder.componentTemplates(updatedComponentTemplates); + } + /* + * We now remove the index from the simulated metadata to force the templates to be used. Note that simulated requests are + * always index requests -- no other type of request is supported. + */ + for (DocWriteRequest actionRequest : bulkRequest.requests) { + assert actionRequest != null : "Requests cannot be null in simulate mode"; + assert actionRequest instanceof IndexRequest + : "Only IndexRequests are supported in simulate mode, but got " + actionRequest.getClass(); + if (actionRequest != null) { + IndexRequest indexRequest = (IndexRequest) actionRequest; + String indexName = indexRequest.index(); + if (indexName != null) { + simulatedMetadataBuilder.remove(indexName); + simulatedMetadataBuilder.removeDataStream(indexName); + } + } + } + metadata = simulatedMetadataBuilder.build(); + } else { + metadata = clusterService.state().getMetadata(); + } + for (DocWriteRequest actionRequest : bulkRequest.requests) { IndexRequest indexRequest = getIndexWriteRequest(actionRequest); if (indexRequest != null) { - IngestService.resolvePipelinesAndUpdateIndexRequest(actionRequest, indexRequest, metadata, templateSubstitutions); + IngestService.resolvePipelinesAndUpdateIndexRequest(actionRequest, indexRequest, metadata); hasIndexRequestsWithPipelines |= IngestService.hasPipeline(indexRequest); } diff --git a/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java b/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java index c860c49809cb5..713116c4cf98e 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java @@ -51,6 +51,7 @@ import org.elasticsearch.xcontent.NamedXContentRegistry; import java.io.IOException; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; @@ -197,12 +198,33 @@ private Exception validateMappings(Map componentTempl * path for when the index does not exist). And it does not deal with system indices since we do not intend for users to * simulate writing to system indices. */ - // First, we remove the index from the cluster state if necessary (since we're going to use the templates) - ClusterState simulatedState = indexAbstraction == null - ? state - : new ClusterState.Builder(state).metadata(Metadata.builder(state.metadata()).remove(request.index()).build()).build(); + ClusterState.Builder simulatedClusterStateBuilder = new ClusterState.Builder(state); + Metadata.Builder simulatedMetadata = Metadata.builder(state.metadata()); + if (indexAbstraction != null) { + /* + * We remove the index or data stream from the cluster state so that we are forced to fall back to the templates to get + * mappings. + */ + String indexRequest = request.index(); + assert indexRequest != null : "Index requests cannot be null in a simulate bulk call"; + if (indexRequest != null) { + simulatedMetadata.remove(indexRequest); + simulatedMetadata.removeDataStream(indexRequest); + } + } + if (componentTemplateSubstitutions.isEmpty() == false) { + /* + * We put the template substitutions into the cluster state. If they have the same name as an existing one, the + * existing one is replaced. + */ + Map updatedComponentTemplates = new HashMap<>(); + updatedComponentTemplates.putAll(state.metadata().componentTemplates()); + updatedComponentTemplates.putAll(componentTemplateSubstitutions); + simulatedMetadata.componentTemplates(updatedComponentTemplates); + } + ClusterState simulatedState = simulatedClusterStateBuilder.metadata(simulatedMetadata).build(); - String matchingTemplate = findV2Template(state.metadata(), request.index(), false); + String matchingTemplate = findV2Template(simulatedState.metadata(), request.index(), false); if (matchingTemplate != null) { final Template template = TransportSimulateIndexTemplateAction.resolveTemplate( matchingTemplate, @@ -212,8 +234,7 @@ private Exception validateMappings(Map componentTempl xContentRegistry, indicesService, systemIndices, - indexSettingProviders, - componentTemplateSubstitutions + indexSettingProviders ); CompressedXContent mappings = template.mappings(); if (mappings != null) { @@ -247,7 +268,7 @@ private Exception validateMappings(Map componentTempl }); } } else { - List matchingTemplates = findV1Templates(state.metadata(), request.index(), false); + List matchingTemplates = findV1Templates(simulatedState.metadata(), request.index(), false); final Map mappingsMap = MetadataCreateIndexService.parseV1Mappings( "{}", matchingTemplates.stream().map(IndexTemplateMetadata::getMappings).collect(toList()), diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java index 4cdf1508a7987..f43f1c6b05a15 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java @@ -665,7 +665,6 @@ private ClusterState applyCreateIndexRequestWithV2Template( request.mappings(), currentState, templateName, - Map.of(), xContentRegistry, request.index() ); @@ -824,7 +823,6 @@ private static List collectSystemV2Mappings( List templateMappings = MetadataIndexTemplateService.collectMappings( composableIndexTemplate, componentTemplates, - Map.of(), indexName ); return collectV2Mappings(null, templateMappings, xContentRegistry); @@ -834,16 +832,10 @@ public static List collectV2Mappings( @Nullable final String requestMappings, final ClusterState currentState, final String templateName, - Map componentTemplateSubstitutions, final NamedXContentRegistry xContentRegistry, final String indexName ) throws Exception { - List templateMappings = MetadataIndexTemplateService.collectMappings( - currentState, - templateName, - componentTemplateSubstitutions, - indexName - ); + List templateMappings = MetadataIndexTemplateService.collectMappings(currentState, templateName, indexName); return collectV2Mappings(requestMappings, templateMappings, xContentRegistry); } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java index 1f9f6f636c1cf..abeb3279b7b50 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java @@ -698,7 +698,7 @@ private void validateIndexTemplateV2(String name, ComposableIndexTemplate indexT final var now = Instant.now(); final var metadata = currentState.getMetadata(); - final var combinedMappings = collectMappings(indexTemplate, metadata.componentTemplates(), Map.of(), "tmp_idx"); + final var combinedMappings = collectMappings(indexTemplate, metadata.componentTemplates(), "tmp_idx"); final var combinedSettings = resolveSettings(indexTemplate, metadata.componentTemplates()); // First apply settings sourced from index setting providers: for (var provider : indexSettingProviders) { @@ -1341,12 +1341,7 @@ private static boolean isGlobalAndHasIndexHiddenSetting(Metadata metadata, Compo /** * Collect the given v2 template into an ordered list of mappings. */ - public static List collectMappings( - final ClusterState state, - final String templateName, - Map componentTemplateSubstitutions, - final String indexName - ) { + public static List collectMappings(final ClusterState state, final String templateName, final String indexName) { final ComposableIndexTemplate template = state.metadata().templatesV2().get(templateName); assert template != null : "attempted to resolve mappings for a template [" + templateName + "] that did not exist in the cluster state"; @@ -1355,7 +1350,7 @@ public static List collectMappings( } final Map componentTemplates = state.metadata().componentTemplates(); - return collectMappings(template, componentTemplates, componentTemplateSubstitutions, indexName); + return collectMappings(template, componentTemplates, indexName); } /** @@ -1364,7 +1359,6 @@ public static List collectMappings( public static List collectMappings( final ComposableIndexTemplate template, final Map componentTemplates, - final Map componentTemplateSubstitutions, final String indexName ) { Objects.requireNonNull(template, "Composable index template must be provided"); @@ -1375,12 +1369,9 @@ public static List collectMappings( ComposableIndexTemplate.DataStreamTemplate.DATA_STREAM_MAPPING_SNIPPET ); } - final Map combinedComponentTemplates = new HashMap<>(); - combinedComponentTemplates.putAll(componentTemplates); - combinedComponentTemplates.putAll(componentTemplateSubstitutions); List mappings = template.composedOf() .stream() - .map(combinedComponentTemplates::get) + .map(componentTemplates::get) .filter(Objects::nonNull) .map(ComponentTemplate::template) .map(Template::mappings) @@ -1716,7 +1707,7 @@ private static void validateCompositeTemplate( String indexName = DataStream.BACKING_INDEX_PREFIX + temporaryIndexName; // Parse mappings to ensure they are valid after being composed - List mappings = collectMappings(stateWithIndex, templateName, Map.of(), indexName); + List mappings = collectMappings(stateWithIndex, templateName, indexName); try { MapperService mapperService = tempIndexService.mapperService(); mapperService.merge(MapperService.SINGLE_MAPPING_NAME, mappings, MapperService.MergeReason.INDEX_TEMPLATE); diff --git a/server/src/main/java/org/elasticsearch/ingest/IngestService.java b/server/src/main/java/org/elasticsearch/ingest/IngestService.java index 8d8d0e87cc259..6f78302c2de9c 100644 --- a/server/src/main/java/org/elasticsearch/ingest/IngestService.java +++ b/server/src/main/java/org/elasticsearch/ingest/IngestService.java @@ -33,7 +33,6 @@ import org.elasticsearch.cluster.ClusterStateApplier; import org.elasticsearch.cluster.ClusterStateTaskExecutor; import org.elasticsearch.cluster.ClusterStateTaskListener; -import org.elasticsearch.cluster.metadata.ComponentTemplate; import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.metadata.IndexAbstraction; import org.elasticsearch.cluster.metadata.IndexMetadata; @@ -271,30 +270,14 @@ public static void resolvePipelinesAndUpdateIndexRequest( final IndexRequest indexRequest, final Metadata metadata ) { - resolvePipelinesAndUpdateIndexRequest(originalRequest, indexRequest, metadata, Map.of()); - } - - public static void resolvePipelinesAndUpdateIndexRequest( - final DocWriteRequest originalRequest, - final IndexRequest indexRequest, - final Metadata metadata, - Map componentTemplateSubstitutions - ) { - resolvePipelinesAndUpdateIndexRequest( - originalRequest, - indexRequest, - metadata, - System.currentTimeMillis(), - componentTemplateSubstitutions - ); + resolvePipelinesAndUpdateIndexRequest(originalRequest, indexRequest, metadata, System.currentTimeMillis()); } static void resolvePipelinesAndUpdateIndexRequest( final DocWriteRequest originalRequest, final IndexRequest indexRequest, final Metadata metadata, - final long epochMillis, - final Map componentTemplateSubstitutions + final long epochMillis ) { if (indexRequest.isPipelineResolved()) { return; @@ -302,21 +285,11 @@ static void resolvePipelinesAndUpdateIndexRequest( /* * Here we look for the pipelines associated with the index if the index exists. If the index does not exist we fall back to using - * templates to find the pipelines. But if a user has passed in component template substitutions, they want the settings from those - * used in place of the settings used to create any previous indices. So in that case we use the templates to find the pipelines -- - * we don't fall back to the existing index if we don't find any because it is possible the user has intentionally removed the - * pipeline. + * templates to find the pipelines. */ - final Pipelines pipelines; - if (componentTemplateSubstitutions.isEmpty()) { - pipelines = resolvePipelinesFromMetadata(originalRequest, indexRequest, metadata, epochMillis) // - .or(() -> resolvePipelinesFromIndexTemplates(indexRequest, metadata, Map.of())) - .orElse(Pipelines.NO_PIPELINES_DEFINED); - } else { - pipelines = resolvePipelinesFromIndexTemplates(indexRequest, metadata, componentTemplateSubstitutions).orElse( - Pipelines.NO_PIPELINES_DEFINED - ); - } + final Pipelines pipelines = resolvePipelinesFromMetadata(originalRequest, indexRequest, metadata, epochMillis).or( + () -> resolvePipelinesFromIndexTemplates(indexRequest, metadata) + ).orElse(Pipelines.NO_PIPELINES_DEFINED); // The pipeline coming as part of the request always has priority over the resolved one from metadata or templates String requestPipeline = indexRequest.getPipeline(); @@ -1473,11 +1446,7 @@ private static Optional resolvePipelinesFromMetadata( return Optional.of(new Pipelines(IndexSettings.DEFAULT_PIPELINE.get(settings), IndexSettings.FINAL_PIPELINE.get(settings))); } - private static Optional resolvePipelinesFromIndexTemplates( - IndexRequest indexRequest, - Metadata metadata, - Map componentTemplateSubstitutions - ) { + private static Optional resolvePipelinesFromIndexTemplates(IndexRequest indexRequest, Metadata metadata) { if (indexRequest.index() == null) { return Optional.empty(); } @@ -1487,7 +1456,7 @@ private static Optional resolvePipelinesFromIndexTemplates( // precedence), or if a V2 template does not match, any V1 templates String v2Template = MetadataIndexTemplateService.findV2Template(metadata, indexRequest.index(), false); if (v2Template != null) { - final Settings settings = MetadataIndexTemplateService.resolveSettings(metadata, v2Template, componentTemplateSubstitutions); + final Settings settings = MetadataIndexTemplateService.resolveSettings(metadata, v2Template); return Optional.of(new Pipelines(IndexSettings.DEFAULT_PIPELINE.get(settings), IndexSettings.FINAL_PIPELINE.get(settings))); } diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateActionTests.java index 9b1d8c15619ad..8f0ff82beab4b 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateActionTests.java @@ -87,8 +87,7 @@ public Settings getAdditionalIndexSettings( xContentRegistry(), indicesService, systemIndices, - indexSettingsProviders, - Map.of() + indexSettingsProviders ); assertThat(resolvedTemplate.settings().getAsInt("test-setting", -1), is(1)); diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java index 5fadd8f263f7c..8d4b04746e7a4 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java @@ -1074,7 +1074,7 @@ public void testResolveConflictingMappings() throws Exception { .build(); state = service.addIndexTemplateV2(state, true, "my-template", it); - List mappings = MetadataIndexTemplateService.collectMappings(state, "my-template", Map.of(), "my-index"); + List mappings = MetadataIndexTemplateService.collectMappings(state, "my-template", "my-index"); assertNotNull(mappings); assertThat(mappings.size(), equalTo(3)); @@ -1136,7 +1136,7 @@ public void testResolveMappings() throws Exception { .build(); state = service.addIndexTemplateV2(state, true, "my-template", it); - List mappings = MetadataIndexTemplateService.collectMappings(state, "my-template", Map.of(), "my-index"); + List mappings = MetadataIndexTemplateService.collectMappings(state, "my-template", "my-index"); assertNotNull(mappings); assertThat(mappings.size(), equalTo(3)); @@ -1190,7 +1190,6 @@ public void testDefinedTimestampMappingIsAddedForDataStreamTemplates() throws Ex List mappings = MetadataIndexTemplateService.collectMappings( state, "logs-data-stream-template", - Map.of(), DataStream.getDefaultBackingIndexName("logs", 1L) ); @@ -1242,12 +1241,7 @@ public void testDefinedTimestampMappingIsAddedForDataStreamTemplates() throws Ex .build(); state = service.addIndexTemplateV2(state, true, "timeseries-template", it); - List mappings = MetadataIndexTemplateService.collectMappings( - state, - "timeseries-template", - Map.of(), - "timeseries" - ); + List mappings = MetadataIndexTemplateService.collectMappings(state, "timeseries-template", "timeseries"); assertNotNull(mappings); assertThat(mappings.size(), equalTo(2)); @@ -1269,7 +1263,6 @@ public void testDefinedTimestampMappingIsAddedForDataStreamTemplates() throws Ex mappings = MetadataIndexTemplateService.collectMappings( state, "timeseries-template", - Map.of(), DataStream.getDefaultBackingIndexName("timeseries", 1L) ); @@ -1318,7 +1311,6 @@ public void testUserDefinedMappingTakesPrecedenceOverDefault() throws Exception List mappings = MetadataIndexTemplateService.collectMappings( state, "logs-template", - Map.of(), DataStream.getDefaultBackingIndexName("logs", 1L) ); @@ -1375,7 +1367,6 @@ public void testUserDefinedMappingTakesPrecedenceOverDefault() throws Exception List mappings = MetadataIndexTemplateService.collectMappings( state, "timeseries-template", - Map.of(), DataStream.getDefaultBackingIndexName("timeseries-template", 1L) ); @@ -2442,12 +2433,7 @@ public void testComposableTemplateWithSubobjectsFalse() throws Exception { .build(); state = service.addIndexTemplateV2(state, true, "composable-template", it); - List mappings = MetadataIndexTemplateService.collectMappings( - state, - "composable-template", - Map.of(), - "test-index" - ); + List mappings = MetadataIndexTemplateService.collectMappings(state, "composable-template", "test-index"); assertNotNull(mappings); assertThat(mappings.size(), equalTo(2)); diff --git a/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java b/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java index a8f556649cb8c..799ea5c0a1c74 100644 --- a/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java +++ b/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java @@ -32,20 +32,16 @@ import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.AliasMetadata; -import org.elasticsearch.cluster.metadata.ComponentTemplate; -import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.IndexTemplateMetadata; import org.elasticsearch.cluster.metadata.Metadata; -import org.elasticsearch.cluster.metadata.Template; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodeUtils; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.cluster.service.ClusterStateTaskExecutorUtils; import org.elasticsearch.common.TriConsumer; import org.elasticsearch.common.bytes.BytesArray; -import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.time.DateFormatter; import org.elasticsearch.common.util.Maps; @@ -77,7 +73,6 @@ import org.mockito.ArgumentMatcher; import org.mockito.invocation.InvocationOnMock; -import java.io.IOException; import java.io.OutputStream; import java.nio.charset.StandardCharsets; import java.util.ArrayList; @@ -2591,7 +2586,7 @@ public void testResolveFinalPipelineWithDateMathExpression() { // index name matches with IDM: IndexRequest indexRequest = new IndexRequest(""); - IngestService.resolvePipelinesAndUpdateIndexRequest(indexRequest, indexRequest, metadata, epochMillis, Map.of()); + IngestService.resolvePipelinesAndUpdateIndexRequest(indexRequest, indexRequest, metadata, epochMillis); assertTrue(hasPipeline(indexRequest)); assertTrue(indexRequest.isPipelineResolved()); assertThat(indexRequest.getPipeline(), equalTo("_none")); @@ -2921,83 +2916,6 @@ public void testResolvePipelinesWithNonePipeline() { } } - public void testResolvePipelinesAndUpdateIndexRequestWithComponentTemplateSubstitutions() throws IOException { - final String componentTemplateName = "test-component-template"; - final String indexName = "my-index-1"; - final String indexPipeline = "index-pipeline"; - final String realTemplatePipeline = "template-pipeline"; - final String substitutePipeline = "substitute-pipeline"; - - Metadata metadata; - { - // Build up cluster state metadata - IndexMetadata.Builder builder = IndexMetadata.builder(indexName) - .settings(settings(IndexVersion.current())) - .numberOfShards(1) - .numberOfReplicas(0); - ComponentTemplate realComponentTemplate = new ComponentTemplate( - new Template( - Settings.builder().put("index.default_pipeline", realTemplatePipeline).build(), - CompressedXContent.fromJSON("{}"), - null - ), - null, - null - ); - ComposableIndexTemplate composableIndexTemplate = ComposableIndexTemplate.builder() - .indexPatterns(List.of("my-index-*")) - .componentTemplates(List.of(componentTemplateName)) - .build(); - metadata = Metadata.builder() - .put(builder) - .indexTemplates(Map.of("my-index-template", composableIndexTemplate)) - .componentTemplates(Map.of("test-component-template", realComponentTemplate)) - .build(); - } - - Map componentTemplateSubstitutions; - { - ComponentTemplate simulatedComponentTemplate = new ComponentTemplate( - new Template( - Settings.builder().put("index.default_pipeline", substitutePipeline).build(), - CompressedXContent.fromJSON("{}"), - null - ), - null, - null - ); - componentTemplateSubstitutions = Map.of(componentTemplateName, simulatedComponentTemplate); - } - - { - /* - * Here there is a pipeline in the request. This takes precedence over anything in the index or templates or component template - * substitutions. - */ - IndexRequest indexRequest = new IndexRequest(indexName).setPipeline(indexPipeline); - IngestService.resolvePipelinesAndUpdateIndexRequest(indexRequest, indexRequest, metadata, 0, componentTemplateSubstitutions); - assertThat(indexRequest.getPipeline(), equalTo(indexPipeline)); - } - { - /* - * Here there is no pipeline in the request, but there is one in the substitute component template. So it takes precedence. - */ - IndexRequest indexRequest = new IndexRequest(indexName); - IngestService.resolvePipelinesAndUpdateIndexRequest(indexRequest, indexRequest, metadata, 0, componentTemplateSubstitutions); - assertThat(indexRequest.getPipeline(), equalTo(substitutePipeline)); - } - { - /* - * This one is tricky. Since the index exists and there are no component template substitutions, we're going to use the actual - * index in this case rather than its template. The index does not have a default pipeline set, so it's "_none" instead of - * realTemplatePipeline. - */ - IndexRequest indexRequest = new IndexRequest(indexName); - IngestService.resolvePipelinesAndUpdateIndexRequest(indexRequest, indexRequest, metadata, 0, Map.of()); - assertThat(indexRequest.getPipeline(), equalTo("_none")); - } - } - private static Tuple randomMapEntry() { return tuple(randomAlphaOfLength(5), randomObject()); }