From 33efab839a3ec8c6af52ebd1f9f214ddd58848e7 Mon Sep 17 00:00:00 2001 From: Keith Massey Date: Fri, 4 Oct 2024 07:46:13 -0500 Subject: [PATCH 1/8] Adding index_template_substitutions to the simulate ingest API --- .../test/ingest/80_ingest_simulate.yml | 148 ++++++++++++++++++ .../bulk/TransportSimulateBulkActionIT.java | 61 ++++++-- .../org/elasticsearch/TransportVersions.java | 3 +- .../action/bulk/BulkFeatures.java | 8 +- .../action/bulk/BulkRequest.java | 5 + .../action/bulk/SimulateBulkRequest.java | 56 ++++++- .../bulk/TransportAbstractBulkAction.java | 12 +- .../bulk/TransportSimulateBulkAction.java | 19 ++- .../MetadataIndexTemplateService.java | 24 +-- .../ingest/RestSimulateIngestAction.java | 20 ++- .../action/bulk/SimulateBulkRequestTests.java | 117 ++++++++++++-- .../TransportSimulateBulkActionTests.java | 4 +- .../ingest/SimulateIngestServiceTests.java | 6 +- 13 files changed, 423 insertions(+), 60 deletions(-) diff --git a/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/80_ingest_simulate.yml b/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/80_ingest_simulate.yml index b4672b1d8924d..a913308e2c9d0 100644 --- a/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/80_ingest_simulate.yml +++ b/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/80_ingest_simulate.yml @@ -807,3 +807,151 @@ setup: - match: { docs.0.doc._source.foo: "FOO" } - match: { docs.0.doc.executed_pipelines: ["foo-pipeline-2"] } - not_exists: docs.0.doc.error + +--- +"Test ingest simulate with index template substitutions": + + - skip: + features: + - headers + - allowed_warnings + + - requires: + cluster_features: ["simulate.index.template.substitutions"] + reason: "ingest simulate index template substitutions added in 8.16" + + - do: + headers: + Content-Type: application/json + ingest.put_pipeline: + id: "foo-pipeline" + body: > + { + "processors": [ + { + "set": { + "field": "foo", + "value": true + } + } + ] + } + - match: { acknowledged: true } + + - do: + cluster.put_component_template: + name: settings_template + body: + template: + settings: + index: + default_pipeline: "foo_pipeline" + + - do: + cluster.put_component_template: + name: mappings_template + body: + template: + mappings: + dynamic: strict + properties: + foo: + type: keyword + + - do: + allowed_warnings: + - "index template [test-composable-1] has index patterns [foo*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [test-composable-1] will take precedence during new index creation" + indices.put_index_template: + name: test-composable-1 + body: + index_patterns: + - foo* + composed_of: + - mappings_template + + - do: + headers: + Content-Type: application/json + simulate.ingest: + index: foo-1 + body: > + { + "docs": [ + { + "_id": "asdf", + "_source": { + "foo": "FOO" + } + } + ], + "component_template_substitutions": { + "settings_template": { + "template": { + "settings": { + "index": { + "default_pipeline": null + } + } + } + } + }, + "index_template_substitutions": { + "foo_index_template": { + "index_patterns":[ + "foo*" + ], + "composed_of": ["settings_template"] + } + } + } + - length: { docs: 1 } + - match: { docs.0.doc._index: "foo-1" } + - match: { docs.0.doc._source.foo: "FOO" } + - match: { docs.0.doc.executed_pipelines: [] } + - not_exists: docs.0.doc.error + + - do: + indices.create: + index: foo-1 + - match: { acknowledged: true } + + - do: + headers: + Content-Type: application/json + simulate.ingest: + index: foo-1 + body: > + { + "docs": [ + { + "_id": "asdf", + "_source": { + "foo": "FOO" + } + } + ], + "component_template_substitutions": { + "settings_template": { + "template": { + "settings": { + "index": { + "default_pipeline": null + } + } + } + } + }, + "index_template_substitutions": { + "foo_index_template": { + "index_patterns":[ + "foo*" + ], + "composed_of": ["settings_template"] + } + } + } + - length: { docs: 1 } + - match: { docs.0.doc._index: "foo-1" } + - match: { docs.0.doc._source.foo: "FOO" } + - match: { docs.0.doc.executed_pipelines: [] } + - not_exists: docs.0.doc.error diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/TransportSimulateBulkActionIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/TransportSimulateBulkActionIT.java index 91674b7ce9050..6a870a4df2d34 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/TransportSimulateBulkActionIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/TransportSimulateBulkActionIT.java @@ -59,7 +59,7 @@ public void testMappingValidationIndexExists() { } """; indicesAdmin().create(new CreateIndexRequest(indexName).mapping(mapping)).actionGet(); - BulkRequest bulkRequest = new SimulateBulkRequest(Map.of(), Map.of()); + BulkRequest bulkRequest = new SimulateBulkRequest(Map.of(), Map.of(), Map.of()); bulkRequest.add(new IndexRequest(indexName).source(""" { "foo1": "baz" @@ -122,16 +122,19 @@ public void testMappingValidationIndexExistsWithComponentTemplate() throws IOExc .indexPatterns(List.of("my-index-*")) .componentTemplates(List.of("test-component-template")) .build(); - TransportPutComposableIndexTemplateAction.Request request = new TransportPutComposableIndexTemplateAction.Request("test"); + final String indexTemplateName = "test-index-template"; + TransportPutComposableIndexTemplateAction.Request request = new TransportPutComposableIndexTemplateAction.Request( + indexTemplateName + ); request.indexTemplate(composableIndexTemplate); client().execute(TransportPutComposableIndexTemplateAction.TYPE, request).actionGet(); String indexName = "my-index-1"; // First, run before the index is created: - assertMappingsUpdatedFromComponentTemplateSubstitutions(indexName); + assertMappingsUpdatedFromComponentTemplateSubstitutions(indexName, indexTemplateName); // Now, create the index and make sure the component template substitutions work the same: indicesAdmin().create(new CreateIndexRequest(indexName)).actionGet(); - assertMappingsUpdatedFromComponentTemplateSubstitutions(indexName); + assertMappingsUpdatedFromComponentTemplateSubstitutions(indexName, indexTemplateName); // Now make sure nothing was actually changed: indicesAdmin().refresh(new RefreshRequest(indexName)).actionGet(); SearchResponse searchResponse = client().search(new SearchRequest(indexName)).actionGet(); @@ -143,7 +146,7 @@ public void testMappingValidationIndexExistsWithComponentTemplate() throws IOExc assertThat(fields.size(), equalTo(1)); } - private void assertMappingsUpdatedFromComponentTemplateSubstitutions(String indexName) { + private void assertMappingsUpdatedFromComponentTemplateSubstitutions(String indexName, String indexTemplateName) { IndexRequest indexRequest1 = new IndexRequest(indexName).source(""" { "foo1": "baz" @@ -156,7 +159,7 @@ private void assertMappingsUpdatedFromComponentTemplateSubstitutions(String inde """, XContentType.JSON).id(randomUUID()); { // First we use the original component template, and expect a failure in the second document: - BulkRequest bulkRequest = new SimulateBulkRequest(Map.of(), Map.of()); + BulkRequest bulkRequest = new SimulateBulkRequest(Map.of(), Map.of(), Map.of()); bulkRequest.add(indexRequest1); bulkRequest.add(indexRequest2); BulkResponse response = client().execute(new ActionType(SimulateBulkAction.NAME), bulkRequest).actionGet(); @@ -188,6 +191,42 @@ private void assertMappingsUpdatedFromComponentTemplateSubstitutions(String inde ) ) ) + ), + Map.of() + ); + bulkRequest.add(indexRequest1); + bulkRequest.add(indexRequest2); + BulkResponse response = client().execute(new ActionType(SimulateBulkAction.NAME), bulkRequest).actionGet(); + assertThat(response.getItems().length, equalTo(2)); + assertThat(response.getItems()[0].getResponse().getResult(), equalTo(DocWriteResponse.Result.CREATED)); + assertNull(((SimulateIndexResponse) response.getItems()[0].getResponse()).getException()); + assertThat(response.getItems()[1].getResponse().getResult(), equalTo(DocWriteResponse.Result.CREATED)); + assertNull(((SimulateIndexResponse) response.getItems()[1].getResponse()).getException()); + } + + { + // Now we substitute a "test-component-template-2" that defines both fields, and a "test-index-template" that pulls it in, so + // we expect no exception: + BulkRequest bulkRequest = new SimulateBulkRequest( + Map.of(), + Map.of( + "test-component-template-2", + Map.of( + "template", + Map.of( + "mappings", + Map.of( + "dynamic", + "strict", + "properties", + Map.of("foo1", Map.of("type", "text"), "foo3", Map.of("type", "text")) + ) + ) + ) + ), + Map.of( + "test-index-template", + Map.of("index_patterns", List.of(indexName), "composed_of", List.of("test-component-template-2")) ) ); bulkRequest.add(indexRequest1); @@ -207,7 +246,7 @@ public void testMappingValidationIndexDoesNotExistsNoTemplate() { * mapping-less "random-index-template" created by the parent class), so we expect no mapping validation failure. */ String indexName = randomAlphaOfLength(20).toLowerCase(Locale.ROOT); - BulkRequest bulkRequest = new SimulateBulkRequest(Map.of(), Map.of()); + BulkRequest bulkRequest = new SimulateBulkRequest(Map.of(), Map.of(), Map.of()); bulkRequest.add(new IndexRequest(indexName).source(""" { "foo1": "baz" @@ -254,7 +293,7 @@ public void testMappingValidationIndexDoesNotExistsV2Template() throws IOExcepti request.indexTemplate(composableIndexTemplate); client().execute(TransportPutComposableIndexTemplateAction.TYPE, request).actionGet(); - BulkRequest bulkRequest = new SimulateBulkRequest(Map.of(), Map.of()); + BulkRequest bulkRequest = new SimulateBulkRequest(Map.of(), Map.of(), Map.of()); bulkRequest.add(new IndexRequest(indexName).source(""" { "foo1": "baz" @@ -286,7 +325,7 @@ public void testMappingValidationIndexDoesNotExistsV1Template() { indicesAdmin().putTemplate( new PutIndexTemplateRequest("test-template").patterns(List.of("my-index-*")).mapping("foo1", "type=integer") ).actionGet(); - BulkRequest bulkRequest = new SimulateBulkRequest(Map.of(), Map.of()); + BulkRequest bulkRequest = new SimulateBulkRequest(Map.of(), Map.of(), Map.of()); bulkRequest.add(new IndexRequest(indexName).source(""" { "foo1": "baz" @@ -340,7 +379,7 @@ public void testMappingValidationIndexDoesNotExistsDataStream() throws IOExcepti client().execute(TransportPutComposableIndexTemplateAction.TYPE, request).actionGet(); { // First, try with no @timestamp to make sure we're picking up data-stream-specific templates - BulkRequest bulkRequest = new SimulateBulkRequest(Map.of(), Map.of()); + BulkRequest bulkRequest = new SimulateBulkRequest(Map.of(), Map.of(), Map.of()); bulkRequest.add(new IndexRequest(indexName).source(""" { "foo1": "baz" @@ -366,7 +405,7 @@ public void testMappingValidationIndexDoesNotExistsDataStream() throws IOExcepti } { // Now with @timestamp - BulkRequest bulkRequest = new SimulateBulkRequest(Map.of(), Map.of()); + BulkRequest bulkRequest = new SimulateBulkRequest(Map.of(), Map.of(), Map.of()); bulkRequest.add(new IndexRequest(indexName).source(""" { "@timestamp": "2024-08-27", diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java index c55436f85a6e3..2cc33b36e7c59 100644 --- a/server/src/main/java/org/elasticsearch/TransportVersions.java +++ b/server/src/main/java/org/elasticsearch/TransportVersions.java @@ -233,7 +233,8 @@ static TransportVersion def(int id) { public static final TransportVersion REGEX_AND_RANGE_INTERVAL_QUERIES = def(8_757_00_0); public static final TransportVersion RRF_QUERY_REWRITE = def(8_758_00_0); public static final TransportVersion SEARCH_FAILURE_STATS = def(8_759_00_0); - public static final TransportVersion INGEST_GEO_DATABASE_PROVIDERS = def(8_760_00_0); + public static final TransportVersion SIMULATE_INDEX_TEMPLATES_SUBSTITUTIONS = def(8_760_00_0); + public static final TransportVersion INGEST_GEO_DATABASE_PROVIDERS = def(8_761_00_0); /* * STOP! READ THIS FIRST! No, really, diff --git a/server/src/main/java/org/elasticsearch/action/bulk/BulkFeatures.java b/server/src/main/java/org/elasticsearch/action/bulk/BulkFeatures.java index af1782ac1ade3..78e603fba9be0 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/BulkFeatures.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/BulkFeatures.java @@ -15,11 +15,17 @@ import java.util.Set; import static org.elasticsearch.action.bulk.TransportSimulateBulkAction.SIMULATE_COMPONENT_TEMPLATE_SUBSTITUTIONS; +import static org.elasticsearch.action.bulk.TransportSimulateBulkAction.SIMULATE_INDEX_TEMPLATE_SUBSTITUTIONS; import static org.elasticsearch.action.bulk.TransportSimulateBulkAction.SIMULATE_MAPPING_VALIDATION; import static org.elasticsearch.action.bulk.TransportSimulateBulkAction.SIMULATE_MAPPING_VALIDATION_TEMPLATES; public class BulkFeatures implements FeatureSpecification { public Set getFeatures() { - return Set.of(SIMULATE_MAPPING_VALIDATION, SIMULATE_MAPPING_VALIDATION_TEMPLATES, SIMULATE_COMPONENT_TEMPLATE_SUBSTITUTIONS); + return Set.of( + SIMULATE_MAPPING_VALIDATION, + SIMULATE_MAPPING_VALIDATION_TEMPLATES, + SIMULATE_COMPONENT_TEMPLATE_SUBSTITUTIONS, + SIMULATE_INDEX_TEMPLATE_SUBSTITUTIONS + ); } } diff --git a/server/src/main/java/org/elasticsearch/action/bulk/BulkRequest.java b/server/src/main/java/org/elasticsearch/action/bulk/BulkRequest.java index 558901f102299..f62b2f48fa2fd 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/BulkRequest.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/BulkRequest.java @@ -23,6 +23,7 @@ import org.elasticsearch.action.support.replication.ReplicationRequest; import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.cluster.metadata.ComponentTemplate; +import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; @@ -506,6 +507,10 @@ public Map getComponentTemplateSubstitutions() throws return Map.of(); } + public Map getIndexTemplateSubstitutions() throws IOException { + return Map.of(); + } + record IncrementalState(Map shardLevelFailures, boolean indexingPressureAccounted) implements Writeable { static final IncrementalState EMPTY = new IncrementalState(Collections.emptyMap(), false); diff --git a/server/src/main/java/org/elasticsearch/action/bulk/SimulateBulkRequest.java b/server/src/main/java/org/elasticsearch/action/bulk/SimulateBulkRequest.java index 3cc7fa12733bf..6fa22151396df 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/SimulateBulkRequest.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/SimulateBulkRequest.java @@ -11,6 +11,7 @@ import org.elasticsearch.TransportVersions; import org.elasticsearch.cluster.metadata.ComponentTemplate; +import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.xcontent.XContentHelper; @@ -22,8 +23,8 @@ import java.util.Map; /** - * This extends BulkRequest with support for providing substitute pipeline definitions and component template definitions. In a user - * request, the substitutions will look something like this: + * This extends BulkRequest with support for providing substitute pipeline definitions, component template definitions, and index template + * substitutions. In a user request, the substitutions will look something like this: * * "pipeline_substitutions": { * "my-pipeline-1": { @@ -72,6 +73,16 @@ * } * } * } + * }, + * "index_template_substitutions": { + * "my-index-template-1": { + * "template": { + * "index_patterns": ["foo*", "bar*"] + * "composed_of": [ + * "component-template-1", + * "component-template-2" + * ] + * } * } * } * @@ -82,6 +93,7 @@ public class SimulateBulkRequest extends BulkRequest { private final Map> pipelineSubstitutions; private final Map> componentTemplateSubstitutions; + private final Map> indexTemplateSubstitutions; /** * @param pipelineSubstitutions The pipeline definitions that are to be used in place of any pre-existing pipeline definitions with @@ -89,14 +101,18 @@ public class SimulateBulkRequest extends BulkRequest { * parsed by XContentHelper.convertToMap(). * @param componentTemplateSubstitutions The component template definitions that are to be used in place of any pre-existing * component template definitions with the same name. + * @param indexTemplateSubstitutions The index template definitions that are to be used in place of any pre-existing + * index template definitions with the same name. */ public SimulateBulkRequest( @Nullable Map> pipelineSubstitutions, - @Nullable Map> componentTemplateSubstitutions + @Nullable Map> componentTemplateSubstitutions, + @Nullable Map> indexTemplateSubstitutions ) { super(); this.pipelineSubstitutions = pipelineSubstitutions; this.componentTemplateSubstitutions = componentTemplateSubstitutions; + this.indexTemplateSubstitutions = indexTemplateSubstitutions; } @SuppressWarnings("unchecked") @@ -108,6 +124,11 @@ public SimulateBulkRequest(StreamInput in) throws IOException { } else { componentTemplateSubstitutions = Map.of(); } + if (in.getTransportVersion().onOrAfter(TransportVersions.SIMULATE_INDEX_TEMPLATES_SUBSTITUTIONS)) { + this.indexTemplateSubstitutions = (Map>) in.readGenericValue(); + } else { + indexTemplateSubstitutions = Map.of(); + } } @Override @@ -117,6 +138,9 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getTransportVersion().onOrAfter(TransportVersions.SIMULATE_COMPONENT_TEMPLATES_SUBSTITUTIONS)) { out.writeGenericValue(componentTemplateSubstitutions); } + if (out.getTransportVersion().onOrAfter(TransportVersions.SIMULATE_INDEX_TEMPLATES_SUBSTITUTIONS)) { + out.writeGenericValue(indexTemplateSubstitutions); + } } public Map> getPipelineSubstitutions() { @@ -140,6 +164,18 @@ public Map getComponentTemplateSubstitutions() throws return result; } + @Override + public Map getIndexTemplateSubstitutions() throws IOException { + if (indexTemplateSubstitutions == null) { + return Map.of(); + } + Map result = new HashMap<>(indexTemplateSubstitutions.size()); + for (Map.Entry> rawEntry : indexTemplateSubstitutions.entrySet()) { + result.put(rawEntry.getKey(), convertRawTemplateToIndexTemplate(rawEntry.getValue())); + } + return result; + } + private static ComponentTemplate convertRawTemplateToComponentTemplate(Map rawTemplate) throws IOException { ComponentTemplate componentTemplate; try (var parser = XContentHelper.mapToXContentParser(XContentParserConfiguration.EMPTY, rawTemplate)) { @@ -148,9 +184,21 @@ private static ComponentTemplate convertRawTemplateToComponentTemplate(Map rawTemplate) throws IOException { + ComposableIndexTemplate indexTemplate; + try (var parser = XContentHelper.mapToXContentParser(XContentParserConfiguration.EMPTY, rawTemplate)) { + indexTemplate = ComposableIndexTemplate.parse(parser); + } + return indexTemplate; + } + @Override public BulkRequest shallowClone() { - BulkRequest bulkRequest = new SimulateBulkRequest(pipelineSubstitutions, componentTemplateSubstitutions); + BulkRequest bulkRequest = new SimulateBulkRequest( + pipelineSubstitutions, + componentTemplateSubstitutions, + indexTemplateSubstitutions + ); bulkRequest.setRefreshPolicy(getRefreshPolicy()); bulkRequest.waitForActiveShards(waitForActiveShards()); bulkRequest.timeout(timeout()); diff --git a/server/src/main/java/org/elasticsearch/action/bulk/TransportAbstractBulkAction.java b/server/src/main/java/org/elasticsearch/action/bulk/TransportAbstractBulkAction.java index 8c6565e52daa7..bb614ca2eb23c 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/TransportAbstractBulkAction.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/TransportAbstractBulkAction.java @@ -25,6 +25,7 @@ import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.metadata.ComponentTemplate; +import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.io.stream.Writeable; @@ -183,7 +184,10 @@ private boolean applyPipelines(Task task, BulkRequest bulkRequest, Executor exec boolean hasIndexRequestsWithPipelines = false; final Metadata metadata; Map componentTemplateSubstitutions = bulkRequest.getComponentTemplateSubstitutions(); - if (bulkRequest.isSimulated() && componentTemplateSubstitutions.isEmpty() == false) { + Map indexTemplateSubstitutions = bulkRequest.getIndexTemplateSubstitutions(); + if (bulkRequest.isSimulated() + && (componentTemplateSubstitutions.isEmpty() == false + || indexTemplateSubstitutions.isEmpty() == false)) { /* * If this is a simulated request, and there are template substitutions, then we want to create and use a new metadata that has * those templates. That is, we want to add the new templates (which will replace any that already existed with the same name), @@ -197,6 +201,12 @@ private boolean applyPipelines(Task task, BulkRequest bulkRequest, Executor exec updatedComponentTemplates.putAll(componentTemplateSubstitutions); simulatedMetadataBuilder.componentTemplates(updatedComponentTemplates); } + if (indexTemplateSubstitutions.isEmpty() == false) { + Map updatedIndexTemplates = new HashMap<>(); + updatedIndexTemplates.putAll(clusterService.state().metadata().templatesV2()); + updatedIndexTemplates.putAll(indexTemplateSubstitutions); + simulatedMetadataBuilder.indexTemplates(updatedIndexTemplates); + } /* * We now remove the index from the simulated metadata to force the templates to be used. Note that simulated requests are * always index requests -- no other type of request is supported. diff --git a/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java b/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java index 713116c4cf98e..d7c555879c00f 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java @@ -17,6 +17,7 @@ import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.ComponentTemplate; +import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; import org.elasticsearch.cluster.metadata.IndexAbstraction; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.IndexTemplateMetadata; @@ -73,6 +74,7 @@ public class TransportSimulateBulkAction extends TransportAbstractBulkAction { public static final NodeFeature SIMULATE_COMPONENT_TEMPLATE_SUBSTITUTIONS = new NodeFeature( "simulate.component.template.substitutions" ); + public static final NodeFeature SIMULATE_INDEX_TEMPLATE_SUBSTITUTIONS = new NodeFeature("simulate.index.template.substitutions"); private final IndicesService indicesService; private final NamedXContentRegistry xContentRegistry; private final Set indexSettingProviders; @@ -119,11 +121,12 @@ protected void doInternalExecute( : "TransportSimulateBulkAction should only ever be called with a SimulateBulkRequest but got a " + bulkRequest.getClass(); final AtomicArray responses = new AtomicArray<>(bulkRequest.requests.size()); Map componentTemplateSubstitutions = bulkRequest.getComponentTemplateSubstitutions(); + Map indexTemplateSubstitutions = bulkRequest.getIndexTemplateSubstitutions(); for (int i = 0; i < bulkRequest.requests.size(); i++) { DocWriteRequest docRequest = bulkRequest.requests.get(i); assert docRequest instanceof IndexRequest : "TransportSimulateBulkAction should only ever be called with IndexRequests"; IndexRequest request = (IndexRequest) docRequest; - Exception mappingValidationException = validateMappings(componentTemplateSubstitutions, request); + Exception mappingValidationException = validateMappings(componentTemplateSubstitutions, indexTemplateSubstitutions, request); responses.set( i, BulkItemResponse.success( @@ -153,7 +156,11 @@ protected void doInternalExecute( * @param request The IndexRequest whose source will be validated against the mapping (if it exists) of its index * @return a mapping exception if the source does not match the mappings, otherwise null */ - private Exception validateMappings(Map componentTemplateSubstitutions, IndexRequest request) { + private Exception validateMappings( + Map componentTemplateSubstitutions, + Map indexTemplateSubstitutions, + IndexRequest request + ) { final SourceToParse sourceToParse = new SourceToParse( request.id(), request.source(), @@ -167,7 +174,7 @@ private Exception validateMappings(Map componentTempl Exception mappingValidationException = null; IndexAbstraction indexAbstraction = state.metadata().getIndicesLookup().get(request.index()); try { - if (indexAbstraction != null && componentTemplateSubstitutions.isEmpty()) { + if (indexAbstraction != null && componentTemplateSubstitutions.isEmpty() && indexTemplateSubstitutions.isEmpty()) { /* * In this case the index exists and we don't have any component template overrides. So we can just use withTempIndexService * to do the mapping validation, using all the existing logic for validation. @@ -222,6 +229,12 @@ private Exception validateMappings(Map componentTempl updatedComponentTemplates.putAll(componentTemplateSubstitutions); simulatedMetadata.componentTemplates(updatedComponentTemplates); } + if (indexTemplateSubstitutions.isEmpty() == false) { + Map updatedIndexTemplates = new HashMap<>(); + updatedIndexTemplates.putAll(state.metadata().templatesV2()); + updatedIndexTemplates.putAll(indexTemplateSubstitutions); + simulatedMetadata.indexTemplates(updatedIndexTemplates); + } ClusterState simulatedState = simulatedClusterStateBuilder.metadata(simulatedMetadata).build(); String matchingTemplate = findV2Template(simulatedState.metadata(), request.index(), false); diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java index abeb3279b7b50..2a2cf6743a877 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java @@ -1421,44 +1421,24 @@ public static Settings resolveSettings(final List templat * Resolve the given v2 template into a collected {@link Settings} object */ public static Settings resolveSettings(final Metadata metadata, final String templateName) { - return resolveSettings(metadata, templateName, Map.of()); - } - - public static Settings resolveSettings( - final Metadata metadata, - final String templateName, - Map templateSubstitutions - ) { final ComposableIndexTemplate template = metadata.templatesV2().get(templateName); assert template != null : "attempted to resolve settings for a template [" + templateName + "] that did not exist in the cluster state"; if (template == null) { return Settings.EMPTY; } - return resolveSettings(template, metadata.componentTemplates(), templateSubstitutions); + return resolveSettings(template, metadata.componentTemplates()); } /** * Resolve the provided v2 template and component templates into a collected {@link Settings} object */ public static Settings resolveSettings(ComposableIndexTemplate template, Map componentTemplates) { - return resolveSettings(template, componentTemplates, Map.of()); - } - - public static Settings resolveSettings( - ComposableIndexTemplate template, - Map componentTemplates, - Map templateSubstitutions - ) { Objects.requireNonNull(template, "attempted to resolve settings for a null template"); Objects.requireNonNull(componentTemplates, "attempted to resolve settings with null component templates"); - Map combinedComponentTemplates = new HashMap<>(); - combinedComponentTemplates.putAll(componentTemplates); - // We want any substitutions to take precedence: - combinedComponentTemplates.putAll(templateSubstitutions); List componentSettings = template.composedOf() .stream() - .map(combinedComponentTemplates::get) + .map(componentTemplates::get) .filter(Objects::nonNull) .map(ComponentTemplate::template) .map(Template::settings) diff --git a/server/src/main/java/org/elasticsearch/rest/action/ingest/RestSimulateIngestAction.java b/server/src/main/java/org/elasticsearch/rest/action/ingest/RestSimulateIngestAction.java index 6de15b0046f1b..dc65a8e1c391e 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/ingest/RestSimulateIngestAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/ingest/RestSimulateIngestAction.java @@ -17,6 +17,7 @@ import org.elasticsearch.client.internal.node.NodeClient; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.core.Nullable; import org.elasticsearch.core.Tuple; import org.elasticsearch.ingest.ConfigurationUtils; import org.elasticsearch.ingest.IngestDocument; @@ -38,6 +39,7 @@ import java.nio.ByteBuffer; import java.util.List; import java.util.Map; +import java.util.Set; import static org.elasticsearch.rest.RestRequest.Method.GET; import static org.elasticsearch.rest.RestRequest.Method.POST; @@ -51,6 +53,17 @@ @ServerlessScope(Scope.PUBLIC) public class RestSimulateIngestAction extends BaseRestHandler { + private static final Set SUPPORTED_QUERY_PARAMETERS = Set.of( + "source_content_type", + "source", + "index", + "pipeline", + "pretty", + "pipeline_substitutions", + "component_template_substitutions", + "index_template_substitutions" + ); + @Override public List routes() { return List.of( @@ -66,6 +79,10 @@ public String getName() { return "ingest_simulate_ingest_action"; } + public @Nullable Set supportedQueryParameters() { + return SUPPORTED_QUERY_PARAMETERS; + } + @Override @SuppressWarnings("unchecked") public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { @@ -76,7 +93,8 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC Map sourceMap = XContentHelper.convertToMap(sourceTuple.v2(), false, sourceTuple.v1()).v2(); SimulateBulkRequest bulkRequest = new SimulateBulkRequest( (Map>) sourceMap.remove("pipeline_substitutions"), - (Map>) sourceMap.remove("component_template_substitutions") + (Map>) sourceMap.remove("component_template_substitutions"), + (Map>) sourceMap.remove("index_template_substitutions") ); BytesReference transformedData = convertToBulkRequestXContentBytes(sourceMap); bulkRequest.add( diff --git a/server/src/test/java/org/elasticsearch/action/bulk/SimulateBulkRequestTests.java b/server/src/test/java/org/elasticsearch/action/bulk/SimulateBulkRequestTests.java index b6b1770e2ed5c..b766eb89d5f52 100644 --- a/server/src/test/java/org/elasticsearch/action/bulk/SimulateBulkRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/bulk/SimulateBulkRequestTests.java @@ -11,6 +11,7 @@ import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.cluster.metadata.ComponentTemplate; +import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.test.ESTestCase; @@ -27,18 +28,27 @@ public class SimulateBulkRequestTests extends ESTestCase { public void testSerialization() throws Exception { - testSerialization(getTestPipelineSubstitutions(), getTestTemplateSubstitutions()); - testSerialization(getTestPipelineSubstitutions(), null); - testSerialization(null, getTestTemplateSubstitutions()); - testSerialization(null, null); - testSerialization(Map.of(), Map.of()); + testSerialization(getTestPipelineSubstitutions(), getTestComponentTemplateSubstitutions(), getTestIndexTemplateSubstitutions()); + testSerialization(getTestPipelineSubstitutions(), null, null); + testSerialization(getTestPipelineSubstitutions(), getTestComponentTemplateSubstitutions(), null); + testSerialization(getTestPipelineSubstitutions(), null, getTestIndexTemplateSubstitutions()); + testSerialization(null, getTestComponentTemplateSubstitutions(), getTestIndexTemplateSubstitutions()); + testSerialization(null, getTestComponentTemplateSubstitutions(), null); + testSerialization(null, null, getTestIndexTemplateSubstitutions()); + testSerialization(null, null, null); + testSerialization(Map.of(), Map.of(), Map.of()); } private void testSerialization( Map> pipelineSubstitutions, - Map> templateSubstitutions + Map> componentTemplateSubstitutions, + Map> indexTemplateSubstitutions ) throws IOException { - SimulateBulkRequest simulateBulkRequest = new SimulateBulkRequest(pipelineSubstitutions, templateSubstitutions); + SimulateBulkRequest simulateBulkRequest = new SimulateBulkRequest( + pipelineSubstitutions, + componentTemplateSubstitutions, + indexTemplateSubstitutions + ); /* * Note: SimulateBulkRequest does not implement equals or hashCode, so we can't test serialization in the usual way for a * Writable @@ -49,7 +59,7 @@ private void testSerialization( @SuppressWarnings({ "unchecked", "rawtypes" }) public void testGetComponentTemplateSubstitutions() throws IOException { - SimulateBulkRequest simulateBulkRequest = new SimulateBulkRequest(Map.of(), Map.of()); + SimulateBulkRequest simulateBulkRequest = new SimulateBulkRequest(Map.of(), Map.of(), Map.of()); assertThat(simulateBulkRequest.getComponentTemplateSubstitutions(), equalTo(Map.of())); String substituteComponentTemplatesString = """ { @@ -83,7 +93,7 @@ public void testGetComponentTemplateSubstitutions() throws IOException { XContentType.JSON ).v2(); Map> substituteComponentTemplates = (Map>) tempMap; - simulateBulkRequest = new SimulateBulkRequest(Map.of(), substituteComponentTemplates); + simulateBulkRequest = new SimulateBulkRequest(Map.of(), substituteComponentTemplates, Map.of()); Map componentTemplateSubstitutions = simulateBulkRequest.getComponentTemplateSubstitutions(); assertThat(componentTemplateSubstitutions.size(), equalTo(2)); assertThat( @@ -107,8 +117,72 @@ public void testGetComponentTemplateSubstitutions() throws IOException { ); } + public void testGetIndexTemplateSubstitutions() throws IOException { + SimulateBulkRequest simulateBulkRequest = new SimulateBulkRequest(Map.of(), Map.of(), Map.of()); + assertThat(simulateBulkRequest.getIndexTemplateSubstitutions(), equalTo(Map.of())); + String substituteIndexTemplatesString = """ + { + "foo_template": { + "template": { + "index_patterns": ["foo*"], + "composed_of": ["foo_mapping_template", "foo_settings_template"], + "mappings": { + "dynamic": "true", + "properties": { + "foo": { + "type": "keyword" + } + } + }, + "settings": { + "index": { + "default_pipeline": "foo-pipeline" + } + } + } + }, + "bar_template": { + "template": { + "index_patterns": ["bar*"], + "composed_of": ["bar_mapping_template", "bar_settings_template"] + } + } + } + """; + + @SuppressWarnings("unchecked") + Map> substituteIndexTemplates = (Map>) (Map) XContentHelper.convertToMap( + new BytesArray(substituteIndexTemplatesString.getBytes(StandardCharsets.UTF_8)), + randomBoolean(), + XContentType.JSON + ).v2(); + simulateBulkRequest = new SimulateBulkRequest(Map.of(), Map.of(), substituteIndexTemplates); + Map indexTemplateSubstitutions = simulateBulkRequest.getIndexTemplateSubstitutions(); + assertThat(indexTemplateSubstitutions.size(), equalTo(2)); + assertThat( + XContentHelper.convertToMap( + XContentHelper.toXContent(indexTemplateSubstitutions.get("foo_template").template(), XContentType.JSON, randomBoolean()), + randomBoolean(), + XContentType.JSON + ).v2(), + equalTo(substituteIndexTemplates.get("foo_template").get("template")) + ); + + assertThat(indexTemplateSubstitutions.get("foo_template").template().settings().size(), equalTo(1)); + assertThat( + indexTemplateSubstitutions.get("foo_template").template().settings().get("index.default_pipeline"), + equalTo("bar-pipeline") + ); + assertNull(indexTemplateSubstitutions.get("bar_template").template().mappings()); + assertNull(indexTemplateSubstitutions.get("bar_template").template().settings()); + } + public void testShallowClone() throws IOException { - SimulateBulkRequest simulateBulkRequest = new SimulateBulkRequest(getTestPipelineSubstitutions(), getTestTemplateSubstitutions()); + SimulateBulkRequest simulateBulkRequest = new SimulateBulkRequest( + getTestPipelineSubstitutions(), + getTestComponentTemplateSubstitutions(), + getTestIndexTemplateSubstitutions() + ); simulateBulkRequest.setRefreshPolicy(randomFrom(WriteRequest.RefreshPolicy.values())); simulateBulkRequest.waitForActiveShards(randomIntBetween(1, 10)); simulateBulkRequest.timeout(randomTimeValue()); @@ -144,7 +218,7 @@ private static Map> getTestPipelineSubstitutions() { ); } - private static Map> getTestTemplateSubstitutions() { + private static Map> getTestComponentTemplateSubstitutions() { return Map.of( "template1", Map.of( @@ -155,4 +229,25 @@ private static Map> getTestTemplateSubstitutions() { Map.of("template", Map.of("mappings", Map.of(), "settings", Map.of())) ); } + + private static Map> getTestIndexTemplateSubstitutions() { + return Map.of( + "template1", + Map.of( + "template", + Map.of( + "index_patterns", + List.of("foo*", "bar*"), + "composed_of", + List.of("template_1", "template_2"), + "mappings", + Map.of("_source", Map.of("enabled", false), "properties", Map.of()), + "settings", + Map.of() + ) + ), + "template2", + Map.of("template", Map.of("index_patterns", List.of("foo*", "bar*"), "mappings", Map.of(), "settings", Map.of())) + ); + } } diff --git a/server/src/test/java/org/elasticsearch/action/bulk/TransportSimulateBulkActionTests.java b/server/src/test/java/org/elasticsearch/action/bulk/TransportSimulateBulkActionTests.java index f4e53912d09a7..71bc31334920e 100644 --- a/server/src/test/java/org/elasticsearch/action/bulk/TransportSimulateBulkActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/bulk/TransportSimulateBulkActionTests.java @@ -135,7 +135,7 @@ public void tearDown() throws Exception { public void testIndexData() throws IOException { Task task = mock(Task.class); // unused - BulkRequest bulkRequest = new SimulateBulkRequest(null, null); + BulkRequest bulkRequest = new SimulateBulkRequest(null, null, null); int bulkItemCount = randomIntBetween(0, 200); for (int i = 0; i < bulkItemCount; i++) { Map source = Map.of(randomAlphaOfLength(10), randomAlphaOfLength(5)); @@ -218,7 +218,7 @@ public void testIndexDataWithValidation() throws IOException { * (7) An indexing request to a nonexistent index that matches no templates */ Task task = mock(Task.class); // unused - BulkRequest bulkRequest = new SimulateBulkRequest(null, null); + BulkRequest bulkRequest = new SimulateBulkRequest(null, null, null); int bulkItemCount = randomIntBetween(0, 200); Map indicesMap = new HashMap<>(); Map v1Templates = new HashMap<>(); diff --git a/server/src/test/java/org/elasticsearch/ingest/SimulateIngestServiceTests.java b/server/src/test/java/org/elasticsearch/ingest/SimulateIngestServiceTests.java index 332a04e40e43d..3b3f5bdc747b5 100644 --- a/server/src/test/java/org/elasticsearch/ingest/SimulateIngestServiceTests.java +++ b/server/src/test/java/org/elasticsearch/ingest/SimulateIngestServiceTests.java @@ -65,7 +65,7 @@ public void testGetPipeline() { ingestService.innerUpdatePipelines(ingestMetadata); { // First we make sure that if there are no substitutions that we get our original pipeline back: - SimulateBulkRequest simulateBulkRequest = new SimulateBulkRequest(null, null); + SimulateBulkRequest simulateBulkRequest = new SimulateBulkRequest(null, null, null); SimulateIngestService simulateIngestService = new SimulateIngestService(ingestService, simulateBulkRequest); Pipeline pipeline = simulateIngestService.getPipeline("pipeline1"); assertThat(pipeline.getProcessors(), contains(transformedMatch(Processor::getType, equalTo("processor1")))); @@ -83,7 +83,7 @@ public void testGetPipeline() { ); pipelineSubstitutions.put("pipeline2", newHashMap("processors", List.of(newHashMap("processor3", Collections.emptyMap())))); - SimulateBulkRequest simulateBulkRequest = new SimulateBulkRequest(pipelineSubstitutions, null); + SimulateBulkRequest simulateBulkRequest = new SimulateBulkRequest(pipelineSubstitutions, null, null); SimulateIngestService simulateIngestService = new SimulateIngestService(ingestService, simulateBulkRequest); Pipeline pipeline1 = simulateIngestService.getPipeline("pipeline1"); assertThat( @@ -103,7 +103,7 @@ public void testGetPipeline() { */ Map> pipelineSubstitutions = new HashMap<>(); pipelineSubstitutions.put("pipeline2", newHashMap("processors", List.of(newHashMap("processor3", Collections.emptyMap())))); - SimulateBulkRequest simulateBulkRequest = new SimulateBulkRequest(pipelineSubstitutions, null); + SimulateBulkRequest simulateBulkRequest = new SimulateBulkRequest(pipelineSubstitutions, null, null); SimulateIngestService simulateIngestService = new SimulateIngestService(ingestService, simulateBulkRequest); Pipeline pipeline1 = simulateIngestService.getPipeline("pipeline1"); assertThat(pipeline1.getProcessors(), contains(transformedMatch(Processor::getType, equalTo("processor1")))); From 87b70a96c9e107b50861c66ad997b1da82656f0c Mon Sep 17 00:00:00 2001 From: Keith Massey Date: Fri, 4 Oct 2024 07:47:20 -0500 Subject: [PATCH 2/8] Update docs/changelog/114128.yaml --- docs/changelog/114128.yaml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 docs/changelog/114128.yaml diff --git a/docs/changelog/114128.yaml b/docs/changelog/114128.yaml new file mode 100644 index 0000000000000..721649d0d6fe0 --- /dev/null +++ b/docs/changelog/114128.yaml @@ -0,0 +1,5 @@ +pr: 114128 +summary: Adding `index_template_substitutions` to the simulate ingest API +area: Ingest Node +type: enhancement +issues: [] From 7f3d998adaaad7ee5b9cce59275bf4f657ecb205 Mon Sep 17 00:00:00 2001 From: Keith Massey Date: Fri, 4 Oct 2024 08:17:12 -0500 Subject: [PATCH 3/8] removing incorrect supported query params --- .../action/bulk/TransportAbstractBulkAction.java | 3 +-- .../rest/action/ingest/RestSimulateIngestAction.java | 11 +---------- 2 files changed, 2 insertions(+), 12 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/bulk/TransportAbstractBulkAction.java b/server/src/main/java/org/elasticsearch/action/bulk/TransportAbstractBulkAction.java index bb614ca2eb23c..111e4d72c57c6 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/TransportAbstractBulkAction.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/TransportAbstractBulkAction.java @@ -186,8 +186,7 @@ private boolean applyPipelines(Task task, BulkRequest bulkRequest, Executor exec Map componentTemplateSubstitutions = bulkRequest.getComponentTemplateSubstitutions(); Map indexTemplateSubstitutions = bulkRequest.getIndexTemplateSubstitutions(); if (bulkRequest.isSimulated() - && (componentTemplateSubstitutions.isEmpty() == false - || indexTemplateSubstitutions.isEmpty() == false)) { + && (componentTemplateSubstitutions.isEmpty() == false || indexTemplateSubstitutions.isEmpty() == false)) { /* * If this is a simulated request, and there are template substitutions, then we want to create and use a new metadata that has * those templates. That is, we want to add the new templates (which will replace any that already existed with the same name), diff --git a/server/src/main/java/org/elasticsearch/rest/action/ingest/RestSimulateIngestAction.java b/server/src/main/java/org/elasticsearch/rest/action/ingest/RestSimulateIngestAction.java index dc65a8e1c391e..ef1cd69626cd0 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/ingest/RestSimulateIngestAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/ingest/RestSimulateIngestAction.java @@ -53,16 +53,7 @@ @ServerlessScope(Scope.PUBLIC) public class RestSimulateIngestAction extends BaseRestHandler { - private static final Set SUPPORTED_QUERY_PARAMETERS = Set.of( - "source_content_type", - "source", - "index", - "pipeline", - "pretty", - "pipeline_substitutions", - "component_template_substitutions", - "index_template_substitutions" - ); + private static final Set SUPPORTED_QUERY_PARAMETERS = Set.of("index", "pipeline"); @Override public List routes() { From 96c702c3ec5bedf936001a9c9b7224a3eebaca67 Mon Sep 17 00:00:00 2001 From: Keith Massey Date: Fri, 4 Oct 2024 08:49:13 -0500 Subject: [PATCH 4/8] minor improvement to TransportSimulateBulkActionIT --- .../action/bulk/TransportSimulateBulkActionIT.java | 13 ++++++------- .../action/ingest/RestSimulateIngestAction.java | 8 +++++++- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/TransportSimulateBulkActionIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/TransportSimulateBulkActionIT.java index 6a870a4df2d34..af99a0344e030 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/TransportSimulateBulkActionIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/TransportSimulateBulkActionIT.java @@ -90,7 +90,7 @@ public void testMappingValidationIndexExists() { } @SuppressWarnings("unchecked") - public void testMappingValidationIndexExistsWithComponentTemplate() throws IOException { + public void testMappingValidationIndexExistsTemplateSubstitutions() throws IOException { /* * This test simulates a BulkRequest of two documents into an existing index. Then we make sure the index contains no documents, and * that the index's mapping in the cluster state has not been updated with the two new field. With the mapping from the template @@ -205,8 +205,10 @@ private void assertMappingsUpdatedFromComponentTemplateSubstitutions(String inde } { - // Now we substitute a "test-component-template-2" that defines both fields, and a "test-index-template" that pulls it in, so - // we expect no exception: + /* + * Now we substitute a "test-component-template-2" that defines both fields, and an index template that pulls it in, so we + * expect no exception: + */ BulkRequest bulkRequest = new SimulateBulkRequest( Map.of(), Map.of( @@ -224,10 +226,7 @@ private void assertMappingsUpdatedFromComponentTemplateSubstitutions(String inde ) ) ), - Map.of( - "test-index-template", - Map.of("index_patterns", List.of(indexName), "composed_of", List.of("test-component-template-2")) - ) + Map.of(indexTemplateName, Map.of("index_patterns", List.of(indexName), "composed_of", List.of("test-component-template-2"))) ); bulkRequest.add(indexRequest1); bulkRequest.add(indexRequest2); diff --git a/server/src/main/java/org/elasticsearch/rest/action/ingest/RestSimulateIngestAction.java b/server/src/main/java/org/elasticsearch/rest/action/ingest/RestSimulateIngestAction.java index ef1cd69626cd0..c42a4bded8c37 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/ingest/RestSimulateIngestAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/ingest/RestSimulateIngestAction.java @@ -53,7 +53,13 @@ @ServerlessScope(Scope.PUBLIC) public class RestSimulateIngestAction extends BaseRestHandler { - private static final Set SUPPORTED_QUERY_PARAMETERS = Set.of("index", "pipeline"); + private static final Set SUPPORTED_QUERY_PARAMETERS = Set.of( + "_source", + "_source_excludes", + "_source_includes", + "index", + "pipeline" + ); @Override public List routes() { From 640af1d613023e36c1f89901cecc01a30ccd0fd7 Mon Sep 17 00:00:00 2001 From: Keith Massey Date: Fri, 4 Oct 2024 15:43:39 -0500 Subject: [PATCH 5/8] Adding API documentation --- .../indices/put-index-template.asciidoc | 11 +++++---- .../ingest/apis/simulate-ingest.asciidoc | 23 +++++++++++++++++++ .../ingest/RestSimulateIngestAction.java | 14 ----------- 3 files changed, 30 insertions(+), 18 deletions(-) diff --git a/docs/reference/indices/put-index-template.asciidoc b/docs/reference/indices/put-index-template.asciidoc index 772bd51afdce8..36fc66ecb90b8 100644 --- a/docs/reference/indices/put-index-template.asciidoc +++ b/docs/reference/indices/put-index-template.asciidoc @@ -85,6 +85,8 @@ include::{docdir}/rest-api/common-parms.asciidoc[tag=master-timeout] [[put-index-template-api-request-body]] ==== {api-request-body-title} +// tag::request-body[] + `composed_of`:: (Optional, array of strings) An ordered list of component template names. Component templates are merged in the order @@ -102,7 +104,7 @@ See <>. + .Properties of `data_stream` [%collapsible%open] -==== +===== `allow_custom_routing`:: (Optional, Boolean) If `true`, the data stream supports <>. Defaults to `false`. @@ -117,7 +119,7 @@ See <>. + If `time_series`, each backing index has an `index.mode` index setting of `time_series`. -==== +===== `index_patterns`:: (Required, array of strings) @@ -146,7 +148,7 @@ Template to be applied. It may optionally include an `aliases`, `mappings`, or + .Properties of `template` [%collapsible%open] -==== +===== `aliases`:: (Optional, object of objects) Aliases to add. + @@ -161,7 +163,7 @@ include::{es-ref-dir}/indices/create-index.asciidoc[tag=aliases-props] include::{docdir}/rest-api/common-parms.asciidoc[tag=mappings] include::{docdir}/rest-api/common-parms.asciidoc[tag=settings] -==== +===== `version`:: (Optional, integer) @@ -174,6 +176,7 @@ Marks this index template as deprecated. When creating or updating a non-deprecated index template that uses deprecated components, {es} will emit a deprecation warning. // end::index-template-api-body[] +// end::request-body[] [[put-index-template-api-example]] ==== {api-examples-title} diff --git a/docs/reference/ingest/apis/simulate-ingest.asciidoc b/docs/reference/ingest/apis/simulate-ingest.asciidoc index ac6da515402bb..4fd3644eea07d 100644 --- a/docs/reference/ingest/apis/simulate-ingest.asciidoc +++ b/docs/reference/ingest/apis/simulate-ingest.asciidoc @@ -102,6 +102,14 @@ POST /_ingest/_simulate } } } + }, + "index_template_substitutions": { <3> + "my-index-template": { + "template": { + "index_patterns": ["my-index-*"], + "componsed_of": ["component_template_1", "component_template_2"] + } + } } } ---- @@ -109,6 +117,8 @@ POST /_ingest/_simulate <1> This replaces the existing `my-pipeline` pipeline with the contents given here for the duration of this request. <2> This replaces the existing `my-component-template` component template with the contents given here for the duration of this request. These templates can be used to change the pipeline(s) used, or to modify the mapping that will be used to validate the result. +<3> This replaces the existing `my-index-template` index template with the contents given here for the duration of this request. +These templates can be used to change the pipeline(s) used, or to modify the mapping that will be used to validate the result. [[simulate-ingest-api-request]] ==== {api-request-title} @@ -225,6 +235,19 @@ include::{es-ref-dir}/indices/put-component-template.asciidoc[tag=template] ==== +`index_template_substitutions`:: +(Optional, map of strings to objects) +Map of index template names to substitute index template definition objects. ++ +.Properties of index template definition objects +[%collapsible%open] + +==== + +include::{es-ref-dir}/indices/put-index-template.asciidoc[tag=request-body] + +==== + [[simulate-ingest-api-example]] ==== {api-examples-title} diff --git a/server/src/main/java/org/elasticsearch/rest/action/ingest/RestSimulateIngestAction.java b/server/src/main/java/org/elasticsearch/rest/action/ingest/RestSimulateIngestAction.java index c42a4bded8c37..680860332fe74 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/ingest/RestSimulateIngestAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/ingest/RestSimulateIngestAction.java @@ -17,7 +17,6 @@ import org.elasticsearch.client.internal.node.NodeClient; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.xcontent.XContentHelper; -import org.elasticsearch.core.Nullable; import org.elasticsearch.core.Tuple; import org.elasticsearch.ingest.ConfigurationUtils; import org.elasticsearch.ingest.IngestDocument; @@ -39,7 +38,6 @@ import java.nio.ByteBuffer; import java.util.List; import java.util.Map; -import java.util.Set; import static org.elasticsearch.rest.RestRequest.Method.GET; import static org.elasticsearch.rest.RestRequest.Method.POST; @@ -53,14 +51,6 @@ @ServerlessScope(Scope.PUBLIC) public class RestSimulateIngestAction extends BaseRestHandler { - private static final Set SUPPORTED_QUERY_PARAMETERS = Set.of( - "_source", - "_source_excludes", - "_source_includes", - "index", - "pipeline" - ); - @Override public List routes() { return List.of( @@ -76,10 +66,6 @@ public String getName() { return "ingest_simulate_ingest_action"; } - public @Nullable Set supportedQueryParameters() { - return SUPPORTED_QUERY_PARAMETERS; - } - @Override @SuppressWarnings("unchecked") public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { From 7542c35876498c297da3c9defe28a062ba17231b Mon Sep 17 00:00:00 2001 From: Keith Massey Date: Fri, 4 Oct 2024 16:41:22 -0500 Subject: [PATCH 6/8] improving yaml rest tests --- .../test/ingest/80_ingest_simulate.yml | 269 +++++++++++++++++- 1 file changed, 265 insertions(+), 4 deletions(-) diff --git a/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/80_ingest_simulate.yml b/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/80_ingest_simulate.yml index a913308e2c9d0..813bbf021b31a 100644 --- a/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/80_ingest_simulate.yml +++ b/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/80_ingest_simulate.yml @@ -371,7 +371,7 @@ setup: template: settings: index: - default_pipeline: "foo_pipeline" + default_pipeline: "foo-pipeline" - do: allowed_warnings: @@ -523,7 +523,7 @@ setup: template: settings: index: - default_pipeline: "foo_pipeline" + default_pipeline: "foo-pipeline" - do: allowed_warnings: @@ -845,7 +845,7 @@ setup: template: settings: index: - default_pipeline: "foo_pipeline" + default_pipeline: "foo-pipeline" - do: cluster.put_component_template: @@ -946,7 +946,7 @@ setup: "index_patterns":[ "foo*" ], - "composed_of": ["settings_template"] + "composed_of": ["settings_template", "mappings_template"] } } } @@ -955,3 +955,264 @@ setup: - match: { docs.0.doc._source.foo: "FOO" } - match: { docs.0.doc.executed_pipelines: [] } - not_exists: docs.0.doc.error + + - do: + headers: + Content-Type: application/json + simulate.ingest: + index: foo-1 + body: > + { + "docs": [ + { + "_id": "asdf", + "_source": { + "foo": "FOO" + } + } + ], + "component_template_substitutions": { + "mappings_template": { + "template": { + "mappings": { + "dynamic": "strict", + "properties": { + "foo": { + "type": "boolean" + } + } + } + } + } + }, + "index_template_substitutions": { + "foo_index_template": { + "index_patterns":[ + "foo*" + ], + "composed_of": ["settings_template", "mappings_template"] + } + } + } + - length: { docs: 1 } + - match: { docs.0.doc._index: "foo-1" } + - match: { docs.0.doc._source.foo: true } + - match: { docs.0.doc.executed_pipelines: ["foo-pipeline"] } + - not_exists: docs.0.doc.error + +--- +"Test ingest simulate with index template substitutions for data streams": + # In this test, we make sure that when the index template is a data stream template, simulate ingest works the same whether the data + # stream has been created or not -- either way, we expect it to use the template rather than the data stream / index mappings and settings. + + - skip: + features: + - headers + - allowed_warnings + + - requires: + cluster_features: ["simulate.index.template.substitutions"] + reason: "ingest simulate component template substitutions added in 8.16" + + - do: + headers: + Content-Type: application/json + ingest.put_pipeline: + id: "foo-pipeline" + body: > + { + "processors": [ + { + "set": { + "field": "foo", + "value": true + } + } + ] + } + - match: { acknowledged: true } + + - do: + cluster.put_component_template: + name: mappings_template + body: + template: + mappings: + dynamic: strict + properties: + foo: + type: keyword + + - do: + cluster.put_component_template: + name: settings_template + body: + template: + settings: + index: + default_pipeline: "foo-pipeline" + + - do: + allowed_warnings: + - "index template [test-composable-1] has index patterns [foo*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [test-composable-1] will take precedence during new index creation" + indices.put_index_template: + name: test-composable-1 + body: + index_patterns: + - foo* + composed_of: + - mappings_template + - settings_template + + - do: + allowed_warnings: + - "index template [my-template-1] has index patterns [simple-data-stream1] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template-1] will take precedence during new index creation" + indices.put_index_template: + name: my-template-1 + body: + index_patterns: [simple-data-stream1] + composed_of: + - mappings_template + - settings_template + data_stream: {} + + # Here we replace my-template-1 with a substitute version that uses the settings_template_2 and mappings_template_2 templates defined in + # this request, and foo-pipeline-2 defined in this request. + - do: + headers: + Content-Type: application/json + simulate.ingest: + index: simple-data-stream1 + body: > + { + "docs": [ + { + "_id": "asdf", + "_source": { + "@timestamp": 1234, + "foo": false + } + } + ], + "pipeline_substitutions": { + "foo-pipeline-2": { + "processors": [ + { + "set": { + "field": "foo", + "value": "FOO" + } + } + ] + } + }, + "component_template_substitutions": { + "settings_template_2": { + "template": { + "settings": { + "index": { + "default_pipeline": "foo-pipeline-2" + } + } + } + }, + "mappings_template_2": { + "template": { + "mappings": { + "dynamic": "strict", + "properties": { + "foo": { + "type": "keyword" + } + } + } + } + } + }, + "index_template_substitutions": { + "my-template-1": { + "index_patterns": ["simple-data-stream1"], + "composed_of": ["settings_template_2", "mappings_template_2"], + "data_stream": {} + } + } + } + - length: { docs: 1 } + - match: { docs.0.doc._index: "simple-data-stream1" } + - match: { docs.0.doc._source.foo: "FOO" } + - match: { docs.0.doc.executed_pipelines: ["foo-pipeline-2"] } + - not_exists: docs.0.doc.error + + - do: + indices.create_data_stream: + name: simple-data-stream1 + - is_true: acknowledged + + - do: + cluster.health: + wait_for_status: yellow + + - do: + headers: + Content-Type: application/json + simulate.ingest: + index: simple-data-stream1 + body: > + { + "docs": [ + { + "_id": "asdf", + "_source": { + "@timestamp": 1234, + "foo": false + } + } + ], + "pipeline_substitutions": { + "foo-pipeline-2": { + "processors": [ + { + "set": { + "field": "foo", + "value": "FOO" + } + } + ] + } + }, + "component_template_substitutions": { + "settings_template_2": { + "template": { + "settings": { + "index": { + "default_pipeline": "foo-pipeline-2" + } + } + } + }, + "mappings_template_2": { + "template": { + "mappings": { + "dynamic": "strict", + "properties": { + "foo": { + "type": "keyword" + } + } + } + } + } + }, + "index_template_substitutions": { + "my-template-1": { + "index_patterns": ["simple-data-stream1"], + "composed_of": ["settings_template_2", "mappings_template_2"], + "data_stream": {} + } + } + } + - length: { docs: 1 } + - match: { docs.0.doc._index: "simple-data-stream1" } + - match: { docs.0.doc._source.foo: "FOO" } + - match: { docs.0.doc.executed_pipelines: ["foo-pipeline-2"] } + - not_exists: docs.0.doc.error From b3b2801c038911368d202079aebe8bc0fc3bb492 Mon Sep 17 00:00:00 2001 From: Keith Massey Date: Mon, 7 Oct 2024 11:18:51 -0500 Subject: [PATCH 7/8] fixing tests --- .../ingest/apis/simulate-ingest.asciidoc | 6 ++---- .../action/bulk/SimulateBulkRequestTests.java | 16 +++++++--------- 2 files changed, 9 insertions(+), 13 deletions(-) diff --git a/docs/reference/ingest/apis/simulate-ingest.asciidoc b/docs/reference/ingest/apis/simulate-ingest.asciidoc index 4fd3644eea07d..1bee03ea3e58a 100644 --- a/docs/reference/ingest/apis/simulate-ingest.asciidoc +++ b/docs/reference/ingest/apis/simulate-ingest.asciidoc @@ -105,10 +105,8 @@ POST /_ingest/_simulate }, "index_template_substitutions": { <3> "my-index-template": { - "template": { - "index_patterns": ["my-index-*"], - "componsed_of": ["component_template_1", "component_template_2"] - } + "index_patterns": ["my-index-*"], + "composed_of": ["component_template_1", "component_template_2"] } } } diff --git a/server/src/test/java/org/elasticsearch/action/bulk/SimulateBulkRequestTests.java b/server/src/test/java/org/elasticsearch/action/bulk/SimulateBulkRequestTests.java index b766eb89d5f52..c94e4e46c9ee3 100644 --- a/server/src/test/java/org/elasticsearch/action/bulk/SimulateBulkRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/bulk/SimulateBulkRequestTests.java @@ -123,9 +123,9 @@ public void testGetIndexTemplateSubstitutions() throws IOException { String substituteIndexTemplatesString = """ { "foo_template": { + "index_patterns": ["foo*"], + "composed_of": ["foo_mapping_template", "foo_settings_template"], "template": { - "index_patterns": ["foo*"], - "composed_of": ["foo_mapping_template", "foo_settings_template"], "mappings": { "dynamic": "true", "properties": { @@ -142,10 +142,8 @@ public void testGetIndexTemplateSubstitutions() throws IOException { } }, "bar_template": { - "template": { - "index_patterns": ["bar*"], - "composed_of": ["bar_mapping_template", "bar_settings_template"] - } + "index_patterns": ["bar*"], + "composed_of": ["bar_mapping_template", "bar_settings_template"] } } """; @@ -171,10 +169,10 @@ public void testGetIndexTemplateSubstitutions() throws IOException { assertThat(indexTemplateSubstitutions.get("foo_template").template().settings().size(), equalTo(1)); assertThat( indexTemplateSubstitutions.get("foo_template").template().settings().get("index.default_pipeline"), - equalTo("bar-pipeline") + equalTo("foo-pipeline") ); - assertNull(indexTemplateSubstitutions.get("bar_template").template().mappings()); - assertNull(indexTemplateSubstitutions.get("bar_template").template().settings()); + assertNull(indexTemplateSubstitutions.get("bar_template").template()); + assertNull(indexTemplateSubstitutions.get("bar_template").template()); } public void testShallowClone() throws IOException { From 32cb224872cbe89bd56a950302133e617406dc9a Mon Sep 17 00:00:00 2001 From: Keith Massey Date: Tue, 8 Oct 2024 16:44:25 -0500 Subject: [PATCH 8/8] improving a yaml rest test --- .../resources/rest-api-spec/test/ingest/80_ingest_simulate.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/80_ingest_simulate.yml b/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/80_ingest_simulate.yml index 813bbf021b31a..18eb401aaa0fe 100644 --- a/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/80_ingest_simulate.yml +++ b/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/80_ingest_simulate.yml @@ -1041,7 +1041,7 @@ setup: dynamic: strict properties: foo: - type: keyword + type: boolean - do: cluster.put_component_template: