From e2eb1de8d1aca6300e56ca904213607bf3c05d8e Mon Sep 17 00:00:00 2001 From: Keith Massey Date: Wed, 11 Sep 2024 14:36:29 -0500 Subject: [PATCH 01/28] Simulate ingest API template_substitutions prototype --- .../test/ingest/80_ingest_simulate.yml | 174 ++++++++++++++++++ .../org/elasticsearch/TransportVersions.java | 1 + .../TransportSimulateIndexTemplateAction.java | 26 +++ .../action/bulk/BulkFeatures.java | 3 +- .../action/bulk/BulkRequestModifier.java | 10 +- .../action/bulk/SimulateBulkRequest.java | 45 ++++- .../bulk/TransportAbstractBulkAction.java | 15 +- .../bulk/TransportSimulateBulkAction.java | 60 +++++- .../cluster/metadata/Metadata.java | 2 +- .../metadata/MetadataCreateIndexService.java | 18 +- .../MetadataIndexTemplateService.java | 48 ++++- .../elasticsearch/ingest/IngestService.java | 41 ++++- .../ingest/SimulateIngestService.java | 5 + .../ingest/RestSimulateIngestAction.java | 3 +- .../action/bulk/SimulateBulkRequestTests.java | 24 ++- .../TransportSimulateBulkActionTests.java | 4 +- .../ingest/SimulateIngestServiceTests.java | 10 +- 17 files changed, 454 insertions(+), 35 deletions(-) diff --git a/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/80_ingest_simulate.yml b/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/80_ingest_simulate.yml index 35ec9979c3250..cb62e28fed97c 100644 --- a/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/80_ingest_simulate.yml +++ b/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/80_ingest_simulate.yml @@ -183,6 +183,7 @@ setup: body: settings: default_pipeline: "my-pipeline" + - match: { acknowledged: true } - do: headers: @@ -303,3 +304,176 @@ setup: - match: { docs.1.doc._index: "second-index" } - match: { docs.1.doc._source.bar: "foo" } - not_exists: docs.1.doc.error + +--- +"Test ingest simulate with template substitutions for component templates": + + - skip: + features: + - headers + - allowed_warnings + + - requires: + cluster_features: ["simulate.component.template.substitutions"] + reason: "ingest simulate component template substitutions added in 8.16" + + - do: + headers: + Content-Type: application/json + ingest.put_pipeline: + id: "foo-pipeline" + body: > + { + "processors": [ + { + "set": { + "field": "foo", + "value": true + } + } + ] + } + - match: { acknowledged: true } + + - do: + headers: + Content-Type: application/json + ingest.put_pipeline: + id: "bar-pipeline" + body: > + { + "processors": [ + { + "set": { + "field": "bar", + "value": true + } + } + ] + } + - match: { acknowledged: true } + + - do: + cluster.put_component_template: + name: mappings_template + body: + template: + mappings: + dynamic: strict + properties: + foo: + type: keyword + + - do: + cluster.put_component_template: + name: settings_template + body: + template: + settings: + index: + default_pipeline: "foo_pipeline" + + - do: + allowed_warnings: + - "index template [test-composable-1] has index patterns [tsdb_templated_*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [test-composable-1] will take precedence during new index creation" + indices.put_index_template: + name: test-composable-1 + body: + index_patterns: + - foo* + composed_of: + - mappings_template + - settings_template + + - do: + headers: + Content-Type: application/json + simulate.ingest: + index: foo-1 + body: > + { + "docs": [ + { + "_id": "asdf", + "_source": { + "foo": "FOO", + "other": "other" + } + } + ], + "template_substitutions": { + "mappings_template": { + "mappings": { + "dynamic": "true", + "properties": { + "foo": { + "type": "keyword" + } + } + } + }, + "settings_template": { + "settings": { + "index": { + "default_pipeline": "bar-pipeline" + } + } + } + } + } + - length: { docs: 1 } + - match: { docs.0.doc._index: "foo-1" } + - match: { docs.0.doc._source.other: "other" } + - match: { docs.0.doc._source.bar: true } + - match: { docs.0.doc._source.foo: "FOO" } + - match: { docs.0.doc.executed_pipelines: ["bar-pipeline"] } + - not_exists: docs.0.doc.error + + - do: + indices.create: + index: foo-1 + - match: { acknowledged: true } + + - do: + headers: + Content-Type: application/json + simulate.ingest: + index: foo-1 + body: > + { + "docs": [ + { + "_id": "asdf", + "_source": { + "foo": "FOO", + "other": "other" + } + } + ], + "template_substitutions": { + "mappings_template": { + "mappings": { + "dynamic": "true", + "properties": { + "foo": { + "type": "keyword" + } + } + } + }, + "settings_template": { + "settings": { + "index": { + "default_pipeline": "bar-pipeline" + } + } + } + } + } + - length: { docs: 1 } + - match: { docs.0.doc._index: "foo-1" } + - match: { docs.0.doc._source.other: "other" } + - match: { docs.0.doc._source.bar: true } + - match: { docs.0.doc._source.foo: "FOO" } + - match: { docs.0.doc.executed_pipelines: ["bar-pipeline"] } + - not_exists: docs.0.doc.error diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java index ad50856c556f7..70c5ef1d11688 100644 --- a/server/src/main/java/org/elasticsearch/TransportVersions.java +++ b/server/src/main/java/org/elasticsearch/TransportVersions.java @@ -208,6 +208,7 @@ static TransportVersion def(int id) { public static final TransportVersion ESQL_AGGREGATE_EXEC_TRACKS_INTERMEDIATE_ATTRS = def(8_738_00_0); public static final TransportVersion CCS_TELEMETRY_STATS = def(8_739_00_0); public static final TransportVersion GLOBAL_RETENTION_TELEMETRY = def(8_740_00_0); + public static final TransportVersion SIMULATE_TEMPLATES_SUBSTITUTIONS = def(8_741_00_0); /* * STOP! READ THIS FIRST! No, really, diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java index cd8ffea3d3824..d8780f49315f9 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java @@ -15,6 +15,7 @@ import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.metadata.AliasMetadata; +import org.elasticsearch.cluster.metadata.ComponentTemplate; import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.metadata.DataStreamLifecycle; @@ -233,6 +234,30 @@ public static Template resolveTemplate( final IndicesService indicesService, final SystemIndices systemIndices, Set indexSettingProviders + ) throws Exception { + return resolveTemplate( + matchingTemplate, + indexName, + simulatedState, + isDslOnlyMode, + xContentRegistry, + indicesService, + systemIndices, + indexSettingProviders, + Map.of() + ); + } + + public static Template resolveTemplate( + final String matchingTemplate, + final String indexName, + final ClusterState simulatedState, + final boolean isDslOnlyMode, + final NamedXContentRegistry xContentRegistry, + final IndicesService indicesService, + final SystemIndices systemIndices, + Set indexSettingProviders, + Map componentTemplateSubstitutions ) throws Exception { var metadata = simulatedState.getMetadata(); Settings templateSettings = resolveSettings(simulatedState.metadata(), matchingTemplate); @@ -262,6 +287,7 @@ public static Template resolveTemplate( null, // empty request mapping as the user can't specify any explicit mappings via the simulate api simulatedState, matchingTemplate, + componentTemplateSubstitutions, xContentRegistry, simulatedIndexName ); diff --git a/server/src/main/java/org/elasticsearch/action/bulk/BulkFeatures.java b/server/src/main/java/org/elasticsearch/action/bulk/BulkFeatures.java index b8dd0d1fe415e..365f561be68ac 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/BulkFeatures.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/BulkFeatures.java @@ -13,11 +13,12 @@ import java.util.Set; +import static org.elasticsearch.action.bulk.TransportSimulateBulkAction.SIMULATE_COMPONENT_TEMPLATE_SUBSTITUTIONS; import static org.elasticsearch.action.bulk.TransportSimulateBulkAction.SIMULATE_MAPPING_VALIDATION; import static org.elasticsearch.action.bulk.TransportSimulateBulkAction.SIMULATE_MAPPING_VALIDATION_TEMPLATES; public class BulkFeatures implements FeatureSpecification { public Set getFeatures() { - return Set.of(SIMULATE_MAPPING_VALIDATION, SIMULATE_MAPPING_VALIDATION_TEMPLATES); + return Set.of(SIMULATE_MAPPING_VALIDATION, SIMULATE_MAPPING_VALIDATION_TEMPLATES, SIMULATE_COMPONENT_TEMPLATE_SUBSTITUTIONS); } } diff --git a/server/src/main/java/org/elasticsearch/action/bulk/BulkRequestModifier.java b/server/src/main/java/org/elasticsearch/action/bulk/BulkRequestModifier.java index d0a75bdf109c5..be0fd67263ae5 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/BulkRequestModifier.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/BulkRequestModifier.java @@ -86,7 +86,15 @@ BulkRequest getBulkRequest() { if (itemResponses.isEmpty()) { return bulkRequest; } else { - BulkRequest modifiedBulkRequest = new BulkRequest(); + BulkRequest modifiedBulkRequest; + if (bulkRequest instanceof SimulateBulkRequest simulateBulkRequest) { + modifiedBulkRequest = new SimulateBulkRequest( + simulateBulkRequest.getPipelineSubstitutions(), + simulateBulkRequest.getTemplateSubstitutions() + ); + } else { + modifiedBulkRequest = new BulkRequest(); + } modifiedBulkRequest.setRefreshPolicy(bulkRequest.getRefreshPolicy()); modifiedBulkRequest.waitForActiveShards(bulkRequest.waitForActiveShards()); modifiedBulkRequest.timeout(bulkRequest.timeout()); diff --git a/server/src/main/java/org/elasticsearch/action/bulk/SimulateBulkRequest.java b/server/src/main/java/org/elasticsearch/action/bulk/SimulateBulkRequest.java index 1987d758eb09a..cf93003fd9f45 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/SimulateBulkRequest.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/SimulateBulkRequest.java @@ -8,6 +8,7 @@ package org.elasticsearch.action.bulk; +import org.elasticsearch.TransportVersions; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.core.Nullable; @@ -16,8 +17,8 @@ import java.util.Map; /** - * This extends BulkRequest with support for providing substitute pipeline definitions. In a user request, the pipeline substitutions - * will look something like this: + * This extends BulkRequest with support for providing substitute pipeline definitions and template definitions. In a user request, the + * substitutions will look something like this: * * "pipeline_substitutions": { * "my-pipeline-1": { @@ -44,6 +45,27 @@ * } * ] * } + * }, + * "template_substitutions": { + * "my-template-1": { + * "settings": { + * "number_of_shards": 1 + * }, + * "mappings": { + * "_source": { + * "enabled": false + * }, + * "properties": { + * "host_name": { + * "type": "keyword" + * }, + * "created_at": { + * "type": "date", + * "format": "EEE MMM dd HH:mm:ss Z yyyy" + * } + * } + * } + * } * } * * The pipelineSubstitutions Map held by this class is intended to be the result of XContentHelper.convertToMap(). The top-level keys @@ -52,33 +74,50 @@ */ public class SimulateBulkRequest extends BulkRequest { private final Map> pipelineSubstitutions; + private final Map> templateSubstitutions; /** * @param pipelineSubstitutions The pipeline definitions that are to be used in place of any pre-existing pipeline definitions with * the same pipelineId. The key of the map is the pipelineId, and the value the pipeline definition as * parsed by XContentHelper.convertToMap(). */ - public SimulateBulkRequest(@Nullable Map> pipelineSubstitutions) { + public SimulateBulkRequest( + @Nullable Map> pipelineSubstitutions, + @Nullable Map> templateSubstitutions + ) { super(); this.pipelineSubstitutions = pipelineSubstitutions; + this.templateSubstitutions = templateSubstitutions; } @SuppressWarnings("unchecked") public SimulateBulkRequest(StreamInput in) throws IOException { super(in); this.pipelineSubstitutions = (Map>) in.readGenericValue(); + if (in.getTransportVersion().onOrAfter(TransportVersions.SIMULATE_TEMPLATES_SUBSTITUTIONS)) { + this.templateSubstitutions = (Map>) in.readGenericValue(); + } else { + templateSubstitutions = Map.of(); + } } @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); out.writeGenericValue(pipelineSubstitutions); + if (out.getTransportVersion().onOrAfter(TransportVersions.SIMULATE_TEMPLATES_SUBSTITUTIONS)) { + out.writeGenericValue(templateSubstitutions); + } } public Map> getPipelineSubstitutions() { return pipelineSubstitutions; } + public Map> getTemplateSubstitutions() { + return templateSubstitutions; + } + @Override public boolean isSimulated() { return true; diff --git a/server/src/main/java/org/elasticsearch/action/bulk/TransportAbstractBulkAction.java b/server/src/main/java/org/elasticsearch/action/bulk/TransportAbstractBulkAction.java index 74864abe3ec50..42023f284e980 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/TransportAbstractBulkAction.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/TransportAbstractBulkAction.java @@ -23,6 +23,7 @@ import org.elasticsearch.cluster.ClusterStateObserver; import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.block.ClusterBlockLevel; +import org.elasticsearch.cluster.metadata.ComponentTemplate; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.io.stream.Writeable; @@ -38,6 +39,8 @@ import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; +import java.io.IOException; +import java.util.Map; import java.util.Objects; import java.util.concurrent.Executor; import java.util.concurrent.TimeUnit; @@ -171,10 +174,20 @@ protected void doRun() { private boolean applyPipelines(Task task, BulkRequest bulkRequest, Executor executor, ActionListener listener) { boolean hasIndexRequestsWithPipelines = false; final Metadata metadata = clusterService.state().getMetadata(); + Map templateSubstitutions = Map.of(); + if (bulkRequest instanceof SimulateBulkRequest simulateBulkRequest) { + Map> rawTemplateSubstitutions = simulateBulkRequest.getTemplateSubstitutions(); + try { + templateSubstitutions = TransportSimulateBulkAction.getComponentTemplateSubstitutionsFromRaw(rawTemplateSubstitutions); + // TODO: fix the horribleness + } catch (IOException e) { + throw new RuntimeException(e); + } + } for (DocWriteRequest actionRequest : bulkRequest.requests) { IndexRequest indexRequest = getIndexWriteRequest(actionRequest); if (indexRequest != null) { - IngestService.resolvePipelinesAndUpdateIndexRequest(actionRequest, indexRequest, metadata); + IngestService.resolvePipelinesAndUpdateIndexRequest(actionRequest, indexRequest, metadata, templateSubstitutions); hasIndexRequestsWithPipelines |= IngestService.hasPipeline(indexRequest); } diff --git a/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java b/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java index 8da6fb409cb90..64d39f29949ad 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java @@ -15,6 +15,9 @@ import org.elasticsearch.action.ingest.SimulateIndexResponse; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.AliasMetadata; +import org.elasticsearch.cluster.metadata.ComponentTemplate; +import org.elasticsearch.cluster.metadata.DataStreamLifecycle; import org.elasticsearch.cluster.metadata.IndexAbstraction; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.IndexTemplateMetadata; @@ -48,6 +51,8 @@ import org.elasticsearch.transport.TransportService; import org.elasticsearch.xcontent.NamedXContentRegistry; +import java.io.IOException; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; @@ -66,6 +71,9 @@ public class TransportSimulateBulkAction extends TransportAbstractBulkAction { public static final NodeFeature SIMULATE_MAPPING_VALIDATION = new NodeFeature("simulate.mapping.validation"); public static final NodeFeature SIMULATE_MAPPING_VALIDATION_TEMPLATES = new NodeFeature("simulate.mapping.validation.templates"); + public static final NodeFeature SIMULATE_COMPONENT_TEMPLATE_SUBSTITUTIONS = new NodeFeature( + "simulate.component.template.substitutions" + ); private final IndicesService indicesService; private final NamedXContentRegistry xContentRegistry; private final Set indexSettingProviders; @@ -109,11 +117,20 @@ protected void doInternalExecute( long relativeStartTimeNanos ) { final AtomicArray responses = new AtomicArray<>(bulkRequest.requests.size()); + assert bulkRequest instanceof SimulateBulkRequest + : "TransportSimulateBulkAction should only ever be called with a SimulateBulkRequest but got a " + bulkRequest.getClass(); + Map> rawTemplateSubstitutions = ((SimulateBulkRequest) bulkRequest).getTemplateSubstitutions(); + Map componentTemplateSubstitutions; + try { + componentTemplateSubstitutions = getComponentTemplateSubstitutionsFromRaw(rawTemplateSubstitutions); + } catch (IOException e) { + throw new RuntimeException(e); + } for (int i = 0; i < bulkRequest.requests.size(); i++) { DocWriteRequest docRequest = bulkRequest.requests.get(i); assert docRequest instanceof IndexRequest : "TransportSimulateBulkAction should only ever be called with IndexRequests"; IndexRequest request = (IndexRequest) docRequest; - Exception mappingValidationException = validateMappings(request); + Exception mappingValidationException = validateMappings(componentTemplateSubstitutions, request); responses.set( i, BulkItemResponse.success( @@ -136,13 +153,42 @@ protected void doInternalExecute( ); } + static Map getComponentTemplateSubstitutionsFromRaw( + Map> rawTemplateSubstitutions + ) throws IOException { + if (rawTemplateSubstitutions == null) { + return Map.of(); + } + Map result = new HashMap<>(rawTemplateSubstitutions.size()); + for (Map.Entry> rawEntry : rawTemplateSubstitutions.entrySet()) { + result.put(rawEntry.getKey(), convertRawTemplateToComponentTemplate(rawEntry.getValue())); + } + return result; + } + + @SuppressWarnings("unchecked") + private static ComponentTemplate convertRawTemplateToComponentTemplate(Map rawTemplate) throws IOException { + Settings settings = null; + CompressedXContent mappings = null; + if (rawTemplate.containsKey("mappings")) { + mappings = new CompressedXContent((Map) rawTemplate.get("mappings")); + } + if (rawTemplate.containsKey("settings")) { + settings = Settings.builder().loadFromMap((Map) rawTemplate.get("settings")).build(); + } + Map aliases = null; + DataStreamLifecycle lifecycle = null; + Template template = new Template(settings, mappings, aliases, lifecycle); + return new ComponentTemplate(template, null, null); + } + /** * This creates a temporary index with the mappings of the index in the request, and then attempts to index the source from the request * into it. If there is a mapping exception, that exception is returned. On success the returned exception is null. * @param request The IndexRequest whose source will be validated against the mapping (if it exists) of its index * @return a mapping exception if the source does not match the mappings, otherwise null */ - private Exception validateMappings(IndexRequest request) { + private Exception validateMappings(Map componentTemplateSubstitutions, IndexRequest request) { final SourceToParse sourceToParse = new SourceToParse( request.id(), request.source(), @@ -156,7 +202,7 @@ private Exception validateMappings(IndexRequest request) { Exception mappingValidationException = null; IndexAbstraction indexAbstraction = state.metadata().getIndicesLookup().get(request.index()); try { - if (indexAbstraction != null) { + if (indexAbstraction != null && componentTemplateSubstitutions.isEmpty()) { IndexMetadata imd = state.metadata().getIndexSafe(indexAbstraction.getWriteIndex(request, state.metadata())); indicesService.withTempIndexService(imd, indexService -> { indexService.mapperService().updateMapping(null, imd); @@ -183,17 +229,21 @@ private Exception validateMappings(IndexRequest request) { * when the index does not exist). And it does not deal with system indices since we do not intend for users to simulate * writing to system indices. */ + ClusterState simulatedState = new ClusterState.Builder(state).metadata( + new Metadata.Builder(state.metadata()).remove(request.index()).build() + ).build(); String matchingTemplate = findV2Template(state.metadata(), request.index(), false); if (matchingTemplate != null) { final Template template = TransportSimulateIndexTemplateAction.resolveTemplate( matchingTemplate, request.index(), - state, + simulatedState, isDataStreamsLifecycleOnlyMode(clusterService.getSettings()), xContentRegistry, indicesService, systemIndices, - indexSettingProviders + indexSettingProviders, + componentTemplateSubstitutions ); CompressedXContent mappings = template.mappings(); if (mappings != null) { diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java index 905c3078b3c9c..4192162891865 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java @@ -1824,7 +1824,7 @@ public Builder() { this(Map.of(), 0); } - Builder(Metadata metadata) { + public Builder(Metadata metadata) { this.clusterUUID = metadata.clusterUUID; this.clusterUUIDCommitted = metadata.clusterUUIDCommitted; this.coordinationMetadata = metadata.coordinationMetadata; diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java index 17db4f9253824..44fb354c2a944 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java @@ -817,7 +817,23 @@ public static List collectV2Mappings( final NamedXContentRegistry xContentRegistry, final String indexName ) throws Exception { - List templateMappings = MetadataIndexTemplateService.collectMappings(currentState, templateName, indexName); + return collectV2Mappings(requestMappings, currentState, templateName, Map.of(), xContentRegistry, indexName); + } + + public static List collectV2Mappings( + @Nullable final String requestMappings, + final ClusterState currentState, + final String templateName, + Map componentTemplateSubstitutions, + final NamedXContentRegistry xContentRegistry, + final String indexName + ) throws Exception { + List templateMappings = MetadataIndexTemplateService.collectMappings( + currentState, + templateName, + componentTemplateSubstitutions, + indexName + ); return collectV2Mappings(requestMappings, templateMappings, xContentRegistry); } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java index ff23f50ef7afe..05dfec83682be 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java @@ -1348,6 +1348,15 @@ private static boolean isGlobalAndHasIndexHiddenSetting(Metadata metadata, Compo * Collect the given v2 template into an ordered list of mappings. */ public static List collectMappings(final ClusterState state, final String templateName, final String indexName) { + return collectMappings(state, templateName, Map.of(), indexName); + } + + public static List collectMappings( + final ClusterState state, + final String templateName, + Map componentTemplateSubstitutions, + final String indexName + ) { final ComposableIndexTemplate template = state.metadata().templatesV2().get(templateName); assert template != null : "attempted to resolve mappings for a template [" + templateName + "] that did not exist in the cluster state"; @@ -1356,7 +1365,7 @@ public static List collectMappings(final ClusterState state, } final Map componentTemplates = state.metadata().componentTemplates(); - return collectMappings(template, componentTemplates, indexName); + return collectMappings(template, componentTemplates, componentTemplateSubstitutions, indexName); } /** @@ -1366,6 +1375,15 @@ public static List collectMappings( final ComposableIndexTemplate template, final Map componentTemplates, final String indexName + ) { + return collectMappings(template, componentTemplates, Map.of(), indexName); + } + + private static List collectMappings( + final ComposableIndexTemplate template, + final Map componentTemplates, + final Map componentTemplateSubstitutions, + final String indexName ) { Objects.requireNonNull(template, "Composable index template must be provided"); // Check if this is a failure store index, and if it is, discard any template mappings. Failure store mappings are predefined. @@ -1375,9 +1393,12 @@ public static List collectMappings( ComposableIndexTemplate.DataStreamTemplate.DATA_STREAM_MAPPING_SNIPPET ); } + final Map combinedComponentTemplates = new HashMap<>(); + combinedComponentTemplates.putAll(componentTemplates); + combinedComponentTemplates.putAll(componentTemplateSubstitutions); List mappings = template.composedOf() .stream() - .map(componentTemplates::get) + .map(combinedComponentTemplates::get) .filter(Objects::nonNull) .map(ComponentTemplate::template) .map(Template::mappings) @@ -1427,24 +1448,43 @@ public static Settings resolveSettings(final List templat * Resolve the given v2 template into a collected {@link Settings} object */ public static Settings resolveSettings(final Metadata metadata, final String templateName) { + return resolveSettings(metadata, templateName, Map.of()); + } + + public static Settings resolveSettings( + final Metadata metadata, + final String templateName, + Map templateSubstitutions + ) { final ComposableIndexTemplate template = metadata.templatesV2().get(templateName); assert template != null : "attempted to resolve settings for a template [" + templateName + "] that did not exist in the cluster state"; if (template == null) { return Settings.EMPTY; } - return resolveSettings(template, metadata.componentTemplates()); + return resolveSettings(template, metadata.componentTemplates(), templateSubstitutions); } /** * Resolve the provided v2 template and component templates into a collected {@link Settings} object */ public static Settings resolveSettings(ComposableIndexTemplate template, Map componentTemplates) { + return resolveSettings(template, componentTemplates, Map.of()); + } + + public static Settings resolveSettings( + ComposableIndexTemplate template, + Map componentTemplates, + Map templateSubstitutions + ) { Objects.requireNonNull(template, "attempted to resolve settings for a null template"); Objects.requireNonNull(componentTemplates, "attempted to resolve settings with null component templates"); + Map combinedComponentTemplates = new HashMap<>(); + combinedComponentTemplates.putAll(componentTemplates); + combinedComponentTemplates.putAll(templateSubstitutions); List componentSettings = template.composedOf() .stream() - .map(componentTemplates::get) + .map(combinedComponentTemplates::get) .filter(Objects::nonNull) .map(ComponentTemplate::template) .map(Template::settings) diff --git a/server/src/main/java/org/elasticsearch/ingest/IngestService.java b/server/src/main/java/org/elasticsearch/ingest/IngestService.java index 20f97e1871483..4fe7ff9a3265b 100644 --- a/server/src/main/java/org/elasticsearch/ingest/IngestService.java +++ b/server/src/main/java/org/elasticsearch/ingest/IngestService.java @@ -32,6 +32,7 @@ import org.elasticsearch.cluster.ClusterStateApplier; import org.elasticsearch.cluster.ClusterStateTaskExecutor; import org.elasticsearch.cluster.ClusterStateTaskListener; +import org.elasticsearch.cluster.metadata.ComponentTemplate; import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.metadata.IndexAbstraction; import org.elasticsearch.cluster.metadata.IndexMetadata; @@ -269,14 +270,24 @@ public static void resolvePipelinesAndUpdateIndexRequest( final IndexRequest indexRequest, final Metadata metadata ) { - resolvePipelinesAndUpdateIndexRequest(originalRequest, indexRequest, metadata, System.currentTimeMillis()); + resolvePipelinesAndUpdateIndexRequest(originalRequest, indexRequest, metadata, Map.of()); + } + + public static void resolvePipelinesAndUpdateIndexRequest( + final DocWriteRequest originalRequest, + final IndexRequest indexRequest, + final Metadata metadata, + Map templateSubstitutions + ) { + resolvePipelinesAndUpdateIndexRequest(originalRequest, indexRequest, metadata, System.currentTimeMillis(), templateSubstitutions); } static void resolvePipelinesAndUpdateIndexRequest( final DocWriteRequest originalRequest, final IndexRequest indexRequest, final Metadata metadata, - final long epochMillis + final long epochMillis, + final Map templateSubstitutions ) { if (indexRequest.isPipelineResolved()) { return; @@ -284,9 +295,21 @@ static void resolvePipelinesAndUpdateIndexRequest( String requestPipeline = indexRequest.getPipeline(); - Pipelines pipelines = resolvePipelinesFromMetadata(originalRequest, indexRequest, metadata, epochMillis) // - .or(() -> resolvePipelinesFromIndexTemplates(indexRequest, metadata)) - .orElse(Pipelines.NO_PIPELINES_DEFINED); + Optional pipelinesFromMetadata = resolvePipelinesFromMetadata(originalRequest, indexRequest, metadata, epochMillis); + Optional pipelinesFromTemplates; + if (templateSubstitutions.isEmpty() == false || pipelinesFromMetadata.isEmpty()) { + pipelinesFromTemplates = resolvePipelinesFromIndexTemplates(indexRequest, metadata, templateSubstitutions); + } else { + pipelinesFromTemplates = Optional.empty(); + } + Pipelines pipelines; + if (templateSubstitutions.isEmpty() == false) { + logger.info("There were template substitutions"); + pipelines = pipelinesFromTemplates.or(() -> pipelinesFromMetadata).orElse(Pipelines.NO_PIPELINES_DEFINED); + } else { + logger.info("There were no template substitutions"); + pipelines = pipelinesFromMetadata.or(() -> pipelinesFromTemplates).orElse(Pipelines.NO_PIPELINES_DEFINED); + } // The pipeline coming as part of the request always has priority over the resolved one from metadata or templates if (requestPipeline != null) { @@ -1442,7 +1465,11 @@ private static Optional resolvePipelinesFromMetadata( return Optional.of(new Pipelines(IndexSettings.DEFAULT_PIPELINE.get(settings), IndexSettings.FINAL_PIPELINE.get(settings))); } - private static Optional resolvePipelinesFromIndexTemplates(IndexRequest indexRequest, Metadata metadata) { + private static Optional resolvePipelinesFromIndexTemplates( + IndexRequest indexRequest, + Metadata metadata, + Map templateSubstitutions + ) { if (indexRequest.index() == null) { return Optional.empty(); } @@ -1452,7 +1479,7 @@ private static Optional resolvePipelinesFromIndexTemplates(IndexReque // precedence), or if a V2 template does not match, any V1 templates String v2Template = MetadataIndexTemplateService.findV2Template(metadata, indexRequest.index(), false); if (v2Template != null) { - final Settings settings = MetadataIndexTemplateService.resolveSettings(metadata, v2Template); + final Settings settings = MetadataIndexTemplateService.resolveSettings(metadata, v2Template, templateSubstitutions); return Optional.of(new Pipelines(IndexSettings.DEFAULT_PIPELINE.get(settings), IndexSettings.FINAL_PIPELINE.get(settings))); } diff --git a/server/src/main/java/org/elasticsearch/ingest/SimulateIngestService.java b/server/src/main/java/org/elasticsearch/ingest/SimulateIngestService.java index 2f9da248b2afb..e069e7fe650ad 100644 --- a/server/src/main/java/org/elasticsearch/ingest/SimulateIngestService.java +++ b/server/src/main/java/org/elasticsearch/ingest/SimulateIngestService.java @@ -10,6 +10,7 @@ import org.elasticsearch.action.bulk.BulkRequest; import org.elasticsearch.action.bulk.SimulateBulkRequest; +import org.elasticsearch.cluster.metadata.IndexTemplateMetadata; import java.util.HashMap; import java.util.Map; @@ -63,6 +64,10 @@ private Map getPipelineSubstitutions( return parsedPipelineSubstitutions; } + private Map getTemplateSubstitutions() { + return Map.of(); + } + /** * This method returns the Pipeline for the given pipelineId. If a substitute definition of the pipeline has been defined for the * current simulate, then that pipeline is returned. Otherwise, the pipeline stored in the cluster state is returned. diff --git a/server/src/main/java/org/elasticsearch/rest/action/ingest/RestSimulateIngestAction.java b/server/src/main/java/org/elasticsearch/rest/action/ingest/RestSimulateIngestAction.java index d13c39f112878..88b1d29f0c405 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/ingest/RestSimulateIngestAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/ingest/RestSimulateIngestAction.java @@ -74,7 +74,8 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC Tuple sourceTuple = request.contentOrSourceParam(); Map sourceMap = XContentHelper.convertToMap(sourceTuple.v2(), false, sourceTuple.v1()).v2(); SimulateBulkRequest bulkRequest = new SimulateBulkRequest( - (Map>) sourceMap.remove("pipeline_substitutions") + (Map>) sourceMap.remove("pipeline_substitutions"), + (Map>) sourceMap.remove("template_substitutions") ); BytesReference transformedData = convertToBulkRequestXContentBytes(sourceMap); bulkRequest.add( diff --git a/server/src/test/java/org/elasticsearch/action/bulk/SimulateBulkRequestTests.java b/server/src/test/java/org/elasticsearch/action/bulk/SimulateBulkRequestTests.java index e7e922c47acbe..a03fcfa9f9e8e 100644 --- a/server/src/test/java/org/elasticsearch/action/bulk/SimulateBulkRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/bulk/SimulateBulkRequestTests.java @@ -19,13 +19,18 @@ public class SimulateBulkRequestTests extends ESTestCase { public void testSerialization() throws Exception { - testSerialization(getTestPipelineSubstitutions()); - testSerialization(null); - testSerialization(Map.of()); + testSerialization(getTestPipelineSubstitutions(), getTestTemplateSubstitutions()); + testSerialization(getTestPipelineSubstitutions(), null); + testSerialization(null, getTestTemplateSubstitutions()); + testSerialization(null, null); + testSerialization(Map.of(), Map.of()); } - private void testSerialization(Map> pipelineSubstitutions) throws IOException { - SimulateBulkRequest simulateBulkRequest = new SimulateBulkRequest(pipelineSubstitutions); + private void testSerialization( + Map> pipelineSubstitutions, + Map> templateSubstitutions + ) throws IOException { + SimulateBulkRequest simulateBulkRequest = new SimulateBulkRequest(pipelineSubstitutions, templateSubstitutions); /* * Note: SimulateBulkRequest does not implement equals or hashCode, so we can't test serialization in the usual way for a * Writable @@ -42,4 +47,13 @@ private static Map> getTestPipelineSubstitutions() { Map.of("processors", List.of(Map.of("processor3", Map.of()))) ); } + + private static Map> getTestTemplateSubstitutions() { + return Map.of( + "template1", + Map.of("mappings", Map.of("_source", Map.of("enabled", false), "properties", Map.of()), "settings", Map.of()), + "template2", + Map.of("mappings", Map.of(), "settings", Map.of()) + ); + } } diff --git a/server/src/test/java/org/elasticsearch/action/bulk/TransportSimulateBulkActionTests.java b/server/src/test/java/org/elasticsearch/action/bulk/TransportSimulateBulkActionTests.java index 3a066ab85147d..a6044c10b7f4c 100644 --- a/server/src/test/java/org/elasticsearch/action/bulk/TransportSimulateBulkActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/bulk/TransportSimulateBulkActionTests.java @@ -134,7 +134,7 @@ public void tearDown() throws Exception { public void testIndexData() { Task task = mock(Task.class); // unused - BulkRequest bulkRequest = new SimulateBulkRequest((Map>) null); + BulkRequest bulkRequest = new SimulateBulkRequest((Map>) null, (Map>) null); int bulkItemCount = randomIntBetween(0, 200); for (int i = 0; i < bulkItemCount; i++) { Map source = Map.of(randomAlphaOfLength(10), randomAlphaOfLength(5)); @@ -217,7 +217,7 @@ public void testIndexDataWithValidation() throws IOException { * (7) An indexing request to a nonexistent index that matches no templates */ Task task = mock(Task.class); // unused - BulkRequest bulkRequest = new SimulateBulkRequest((Map>) null); + BulkRequest bulkRequest = new SimulateBulkRequest((Map>) null, (Map>) null); int bulkItemCount = randomIntBetween(0, 200); Map indicesMap = new HashMap<>(); Map v1Templates = new HashMap<>(); diff --git a/server/src/test/java/org/elasticsearch/ingest/SimulateIngestServiceTests.java b/server/src/test/java/org/elasticsearch/ingest/SimulateIngestServiceTests.java index 18f66676cfd1f..08ab5d83d27c7 100644 --- a/server/src/test/java/org/elasticsearch/ingest/SimulateIngestServiceTests.java +++ b/server/src/test/java/org/elasticsearch/ingest/SimulateIngestServiceTests.java @@ -64,7 +64,10 @@ public void testGetPipeline() { ingestService.innerUpdatePipelines(ingestMetadata); { // First we make sure that if there are no substitutions that we get our original pipeline back: - SimulateBulkRequest simulateBulkRequest = new SimulateBulkRequest((Map>) null); + SimulateBulkRequest simulateBulkRequest = new SimulateBulkRequest( + (Map>) null, + (Map>) null + ); SimulateIngestService simulateIngestService = new SimulateIngestService(ingestService, simulateBulkRequest); Pipeline pipeline = simulateIngestService.getPipeline("pipeline1"); assertThat(pipeline.getProcessors(), contains(transformedMatch(Processor::getType, equalTo("processor1")))); @@ -82,7 +85,8 @@ public void testGetPipeline() { ); pipelineSubstitutions.put("pipeline2", newHashMap("processors", List.of(newHashMap("processor3", Collections.emptyMap())))); - SimulateBulkRequest simulateBulkRequest = new SimulateBulkRequest(pipelineSubstitutions); + // TODO + SimulateBulkRequest simulateBulkRequest = new SimulateBulkRequest(pipelineSubstitutions, null); SimulateIngestService simulateIngestService = new SimulateIngestService(ingestService, simulateBulkRequest); Pipeline pipeline1 = simulateIngestService.getPipeline("pipeline1"); assertThat( @@ -102,7 +106,7 @@ public void testGetPipeline() { */ Map> pipelineSubstitutions = new HashMap<>(); pipelineSubstitutions.put("pipeline2", newHashMap("processors", List.of(newHashMap("processor3", Collections.emptyMap())))); - SimulateBulkRequest simulateBulkRequest = new SimulateBulkRequest(pipelineSubstitutions); + SimulateBulkRequest simulateBulkRequest = new SimulateBulkRequest(pipelineSubstitutions, null); SimulateIngestService simulateIngestService = new SimulateIngestService(ingestService, simulateBulkRequest); Pipeline pipeline1 = simulateIngestService.getPipeline("pipeline1"); assertThat(pipeline1.getProcessors(), contains(transformedMatch(Processor::getType, equalTo("processor1")))); From 0818e53c4fce1659247c49aafffb7a33618f76cb Mon Sep 17 00:00:00 2001 From: Keith Massey Date: Wed, 11 Sep 2024 14:37:57 -0500 Subject: [PATCH 02/28] Update docs/changelog/112762.yaml --- docs/changelog/112762.yaml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 docs/changelog/112762.yaml diff --git a/docs/changelog/112762.yaml b/docs/changelog/112762.yaml new file mode 100644 index 0000000000000..335530fecba0d --- /dev/null +++ b/docs/changelog/112762.yaml @@ -0,0 +1,5 @@ +pr: 112762 +summary: Simulate ingest API `template_substitutions` prototype +area: Ingest Node +type: enhancement +issues: [] From 1d5a6866295a640b2df78d0e57de72453a446170 Mon Sep 17 00:00:00 2001 From: Keith Massey Date: Wed, 11 Sep 2024 15:22:12 -0500 Subject: [PATCH 03/28] fixing compilation error --- .../java/org/elasticsearch/ingest/IngestService.java | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/server/src/main/java/org/elasticsearch/ingest/IngestService.java b/server/src/main/java/org/elasticsearch/ingest/IngestService.java index 4fe7ff9a3265b..92a36df535eea 100644 --- a/server/src/main/java/org/elasticsearch/ingest/IngestService.java +++ b/server/src/main/java/org/elasticsearch/ingest/IngestService.java @@ -282,6 +282,15 @@ public static void resolvePipelinesAndUpdateIndexRequest( resolvePipelinesAndUpdateIndexRequest(originalRequest, indexRequest, metadata, System.currentTimeMillis(), templateSubstitutions); } + static void resolvePipelinesAndUpdateIndexRequest( + final DocWriteRequest originalRequest, + final IndexRequest indexRequest, + final Metadata metadata, + final long epochMillis + ) { + resolvePipelinesAndUpdateIndexRequest(originalRequest, indexRequest, metadata, epochMillis, Map.of()); + } + static void resolvePipelinesAndUpdateIndexRequest( final DocWriteRequest originalRequest, final IndexRequest indexRequest, From f9b7717d41f4ebe95a3e338ac5892ccb0dbdc682 Mon Sep 17 00:00:00 2001 From: Keith Massey Date: Wed, 11 Sep 2024 16:53:22 -0500 Subject: [PATCH 04/28] Fixing TransportSimulateBulkActionIT to use the correct request type --- .../action/bulk/TransportSimulateBulkActionIT.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/TransportSimulateBulkActionIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/TransportSimulateBulkActionIT.java index 573d929ee30a9..f976a3ed80d58 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/TransportSimulateBulkActionIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/TransportSimulateBulkActionIT.java @@ -56,7 +56,7 @@ public void testMappingValidationIndexExists() { } """; indicesAdmin().create(new CreateIndexRequest(indexName).mapping(mapping)).actionGet(); - BulkRequest bulkRequest = new BulkRequest(); + BulkRequest bulkRequest = new SimulateBulkRequest(Map.of(), Map.of()); bulkRequest.add(new IndexRequest(indexName).source(""" { "foo1": "baz" @@ -92,7 +92,7 @@ public void testMappingValidationIndexDoesNotExistsNoTemplate() { * mapping-less "random-index-template" created by the parent class), so we expect no mapping validation failure. */ String indexName = randomAlphaOfLength(20).toLowerCase(Locale.ROOT); - BulkRequest bulkRequest = new BulkRequest(); + BulkRequest bulkRequest = new SimulateBulkRequest(Map.of(), Map.of()); bulkRequest.add(new IndexRequest(indexName).source(""" { "foo1": "baz" @@ -139,7 +139,7 @@ public void testMappingValidationIndexDoesNotExistsV2Template() throws IOExcepti request.indexTemplate(composableIndexTemplate); client().execute(TransportPutComposableIndexTemplateAction.TYPE, request).actionGet(); - BulkRequest bulkRequest = new BulkRequest(); + BulkRequest bulkRequest = new SimulateBulkRequest(Map.of(), Map.of()); bulkRequest.add(new IndexRequest(indexName).source(""" { "foo1": "baz" @@ -171,7 +171,7 @@ public void testMappingValidationIndexDoesNotExistsV1Template() { indicesAdmin().putTemplate( new PutIndexTemplateRequest("test-template").patterns(List.of("my-index-*")).mapping("foo1", "type=integer") ).actionGet(); - BulkRequest bulkRequest = new BulkRequest(); + BulkRequest bulkRequest = new SimulateBulkRequest(Map.of(), Map.of()); bulkRequest.add(new IndexRequest(indexName).source(""" { "foo1": "baz" @@ -225,7 +225,7 @@ public void testMappingValidationIndexDoesNotExistsDataStream() throws IOExcepti client().execute(TransportPutComposableIndexTemplateAction.TYPE, request).actionGet(); { // First, try with no @timestamp to make sure we're picking up data-stream-specific templates - BulkRequest bulkRequest = new BulkRequest(); + BulkRequest bulkRequest = new SimulateBulkRequest(Map.of(), Map.of()); bulkRequest.add(new IndexRequest(indexName).source(""" { "foo1": "baz" @@ -251,7 +251,7 @@ public void testMappingValidationIndexDoesNotExistsDataStream() throws IOExcepti } { // Now with @timestamp - BulkRequest bulkRequest = new BulkRequest(); + BulkRequest bulkRequest = new SimulateBulkRequest(Map.of(), Map.of()); bulkRequest.add(new IndexRequest(indexName).source(""" { "@timestamp": "2024-08-27", From f1dff593df84f99ac10fbaec46770d6ac154eb44 Mon Sep 17 00:00:00 2001 From: Keith Massey Date: Wed, 11 Sep 2024 16:54:30 -0500 Subject: [PATCH 05/28] removing duplicate method from TransportSimulateIndexTemplateAction --- .../TransportSimulateIndexTemplateAction.java | 26 ++----------------- .../post/TransportSimulateTemplateAction.java | 4 +-- ...sportSimulateIndexTemplateActionTests.java | 3 ++- 3 files changed, 6 insertions(+), 27 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java index d8780f49315f9..0dcaeccf55c78 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java @@ -156,7 +156,8 @@ protected void masterOperation( xContentRegistry, indicesService, systemIndices, - indexSettingProviders + indexSettingProviders, + Map.of() ); final Map> overlapping = new HashMap<>(); @@ -225,29 +226,6 @@ public static ClusterState resolveTemporaryState( * Take a template and index name as well as state where the template exists, and return a final * {@link Template} that represents all the resolved Settings, Mappings, Aliases and Lifecycle */ - public static Template resolveTemplate( - final String matchingTemplate, - final String indexName, - final ClusterState simulatedState, - final boolean isDslOnlyMode, - final NamedXContentRegistry xContentRegistry, - final IndicesService indicesService, - final SystemIndices systemIndices, - Set indexSettingProviders - ) throws Exception { - return resolveTemplate( - matchingTemplate, - indexName, - simulatedState, - isDslOnlyMode, - xContentRegistry, - indicesService, - systemIndices, - indexSettingProviders, - Map.of() - ); - } - public static Template resolveTemplate( final String matchingTemplate, final String indexName, diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateTemplateAction.java index ead00dc858a47..1ca9bc1b14ffb 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateTemplateAction.java @@ -160,7 +160,6 @@ protected void masterOperation( Map> overlapping = new HashMap<>(); overlapping.putAll(findConflictingV1Templates(tempClusterState, matchingTemplate, templateV2.indexPatterns())); overlapping.putAll(findConflictingV2Templates(tempClusterState, matchingTemplate, templateV2.indexPatterns())); - Template template = TransportSimulateIndexTemplateAction.resolveTemplate( matchingTemplate, temporaryIndexName, @@ -169,7 +168,8 @@ protected void masterOperation( xContentRegistry, indicesService, systemIndices, - indexSettingProviders + indexSettingProviders, + Map.of() ); if (request.includeDefaults()) { listener.onResponse( diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateActionTests.java index c8fb09b1e7177..8b16d415001ee 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateActionTests.java @@ -86,7 +86,8 @@ public Settings getAdditionalIndexSettings( xContentRegistry(), indicesService, systemIndices, - indexSettingsProviders + indexSettingsProviders, + Map.of() ); assertThat(resolvedTemplate.settings().getAsInt("test-setting", -1), is(1)); From 772b2f1e599a1964232916d94981188735e54c10 Mon Sep 17 00:00:00 2001 From: Keith Massey Date: Wed, 11 Sep 2024 17:26:07 -0500 Subject: [PATCH 06/28] removing if/else logic around simualte bulk request or normal bulk request --- .../action/bulk/BulkRequest.java | 21 ++++++++ .../action/bulk/BulkRequestModifier.java | 14 +----- .../action/bulk/SimulateBulkRequest.java | 48 +++++++++++++++++-- .../bulk/TransportAbstractBulkAction.java | 22 +++------ .../bulk/TransportSimulateBulkAction.java | 42 +--------------- .../TransportSimulateBulkActionTests.java | 2 +- 6 files changed, 76 insertions(+), 73 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/bulk/BulkRequest.java b/server/src/main/java/org/elasticsearch/action/bulk/BulkRequest.java index 1a8bdb1c885c6..ebffd451a262e 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/BulkRequest.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/BulkRequest.java @@ -20,6 +20,7 @@ import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.action.support.replication.ReplicationRequest; import org.elasticsearch.action.update.UpdateRequest; +import org.elasticsearch.cluster.metadata.ComponentTemplate; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; @@ -37,6 +38,7 @@ import java.util.Collections; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.Set; @@ -474,4 +476,23 @@ public Set getIndices() { public boolean isSimulated() { return false; // Always false, but may be overridden by a subclass } + + /* + * Returns any component template substitutions that are to be used as part of this bulk request. We would likely only have + * substitutions in the event of a simulated request. + */ + public Map getComponentTemplateSubstitutions() throws IOException { + return Map.of(); + } + + /* + * This copies this bulk request, but without all of its inner requests + */ + public BulkRequest shallowClone() { + BulkRequest bulkRequest = new BulkRequest(); + bulkRequest.setRefreshPolicy(bulkRequest.getRefreshPolicy()); + bulkRequest.waitForActiveShards(bulkRequest.waitForActiveShards()); + bulkRequest.timeout(bulkRequest.timeout()); + return bulkRequest; + } } diff --git a/server/src/main/java/org/elasticsearch/action/bulk/BulkRequestModifier.java b/server/src/main/java/org/elasticsearch/action/bulk/BulkRequestModifier.java index be0fd67263ae5..b4b34300bfa9c 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/BulkRequestModifier.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/BulkRequestModifier.java @@ -86,19 +86,7 @@ BulkRequest getBulkRequest() { if (itemResponses.isEmpty()) { return bulkRequest; } else { - BulkRequest modifiedBulkRequest; - if (bulkRequest instanceof SimulateBulkRequest simulateBulkRequest) { - modifiedBulkRequest = new SimulateBulkRequest( - simulateBulkRequest.getPipelineSubstitutions(), - simulateBulkRequest.getTemplateSubstitutions() - ); - } else { - modifiedBulkRequest = new BulkRequest(); - } - modifiedBulkRequest.setRefreshPolicy(bulkRequest.getRefreshPolicy()); - modifiedBulkRequest.waitForActiveShards(bulkRequest.waitForActiveShards()); - modifiedBulkRequest.timeout(bulkRequest.timeout()); - + BulkRequest modifiedBulkRequest = bulkRequest.shallowClone(); int slot = 0; List> requests = bulkRequest.requests(); for (int i = 0; i < requests.size(); i++) { diff --git a/server/src/main/java/org/elasticsearch/action/bulk/SimulateBulkRequest.java b/server/src/main/java/org/elasticsearch/action/bulk/SimulateBulkRequest.java index cf93003fd9f45..5d0b2f71919fd 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/SimulateBulkRequest.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/SimulateBulkRequest.java @@ -9,11 +9,18 @@ package org.elasticsearch.action.bulk; import org.elasticsearch.TransportVersions; +import org.elasticsearch.cluster.metadata.AliasMetadata; +import org.elasticsearch.cluster.metadata.ComponentTemplate; +import org.elasticsearch.cluster.metadata.DataStreamLifecycle; +import org.elasticsearch.cluster.metadata.Template; +import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.Nullable; import java.io.IOException; +import java.util.HashMap; import java.util.Map; /** @@ -114,12 +121,45 @@ public Map> getPipelineSubstitutions() { return pipelineSubstitutions; } - public Map> getTemplateSubstitutions() { - return templateSubstitutions; - } - @Override public boolean isSimulated() { return true; } + + @Override + public Map getComponentTemplateSubstitutions() throws IOException { + if (templateSubstitutions == null) { + return Map.of(); + } + Map result = new HashMap<>(templateSubstitutions.size()); + for (Map.Entry> rawEntry : templateSubstitutions.entrySet()) { + result.put(rawEntry.getKey(), convertRawTemplateToComponentTemplate(rawEntry.getValue())); + } + return result; + } + + @SuppressWarnings("unchecked") + private static ComponentTemplate convertRawTemplateToComponentTemplate(Map rawTemplate) throws IOException { + Settings settings = null; + CompressedXContent mappings = null; + if (rawTemplate.containsKey("mappings")) { + mappings = new CompressedXContent((Map) rawTemplate.get("mappings")); + } + if (rawTemplate.containsKey("settings")) { + settings = Settings.builder().loadFromMap((Map) rawTemplate.get("settings")).build(); + } + Map aliases = null; + DataStreamLifecycle lifecycle = null; + Template template = new Template(settings, mappings, aliases, lifecycle); + return new ComponentTemplate(template, null, null); + } + + @Override + public BulkRequest shallowClone() { + BulkRequest bulkRequest = new SimulateBulkRequest(pipelineSubstitutions, templateSubstitutions); + bulkRequest.setRefreshPolicy(bulkRequest.getRefreshPolicy()); + bulkRequest.waitForActiveShards(bulkRequest.waitForActiveShards()); + bulkRequest.timeout(bulkRequest.timeout()); + return bulkRequest; + } } diff --git a/server/src/main/java/org/elasticsearch/action/bulk/TransportAbstractBulkAction.java b/server/src/main/java/org/elasticsearch/action/bulk/TransportAbstractBulkAction.java index 42023f284e980..a66830d2b3c7d 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/TransportAbstractBulkAction.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/TransportAbstractBulkAction.java @@ -165,25 +165,17 @@ public void onTimeout(TimeValue timeout) { private void forkAndExecute(Task task, BulkRequest bulkRequest, Executor executor, ActionListener releasingListener) { executor.execute(new ActionRunnable<>(releasingListener) { @Override - protected void doRun() { + protected void doRun() throws IOException { applyPipelinesAndDoInternalExecute(task, bulkRequest, executor, releasingListener); } }); } - private boolean applyPipelines(Task task, BulkRequest bulkRequest, Executor executor, ActionListener listener) { + private boolean applyPipelines(Task task, BulkRequest bulkRequest, Executor executor, ActionListener listener) + throws IOException { boolean hasIndexRequestsWithPipelines = false; final Metadata metadata = clusterService.state().getMetadata(); - Map templateSubstitutions = Map.of(); - if (bulkRequest instanceof SimulateBulkRequest simulateBulkRequest) { - Map> rawTemplateSubstitutions = simulateBulkRequest.getTemplateSubstitutions(); - try { - templateSubstitutions = TransportSimulateBulkAction.getComponentTemplateSubstitutionsFromRaw(rawTemplateSubstitutions); - // TODO: fix the horribleness - } catch (IOException e) { - throw new RuntimeException(e); - } - } + Map templateSubstitutions = bulkRequest.getComponentTemplateSubstitutions(); for (DocWriteRequest actionRequest : bulkRequest.requests) { IndexRequest indexRequest = getIndexWriteRequest(actionRequest); if (indexRequest != null) { @@ -257,7 +249,7 @@ private void processBulkIndexIngestRequest( } else { ActionRunnable runnable = new ActionRunnable<>(actionListener) { @Override - protected void doRun() { + protected void doRun() throws IOException { applyPipelinesAndDoInternalExecute(task, bulkRequest, executor, actionListener); } @@ -335,7 +327,7 @@ private void applyPipelinesAndDoInternalExecute( BulkRequest bulkRequest, Executor executor, ActionListener listener - ) { + ) throws IOException { final long relativeStartTimeNanos = relativeTimeNanos(); if (applyPipelines(task, bulkRequest, executor, listener) == false) { doInternalExecute(task, bulkRequest, executor, listener, relativeStartTimeNanos); @@ -356,6 +348,6 @@ protected abstract void doInternalExecute( Executor executor, ActionListener listener, long relativeStartTimeNanos - ); + ) throws IOException; } diff --git a/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java b/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java index 64d39f29949ad..2172ee02b2d15 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java @@ -15,9 +15,7 @@ import org.elasticsearch.action.ingest.SimulateIndexResponse; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.metadata.AliasMetadata; import org.elasticsearch.cluster.metadata.ComponentTemplate; -import org.elasticsearch.cluster.metadata.DataStreamLifecycle; import org.elasticsearch.cluster.metadata.IndexAbstraction; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.IndexTemplateMetadata; @@ -52,7 +50,6 @@ import org.elasticsearch.xcontent.NamedXContentRegistry; import java.io.IOException; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; @@ -115,17 +112,11 @@ protected void doInternalExecute( Executor executor, ActionListener listener, long relativeStartTimeNanos - ) { + ) throws IOException { final AtomicArray responses = new AtomicArray<>(bulkRequest.requests.size()); assert bulkRequest instanceof SimulateBulkRequest : "TransportSimulateBulkAction should only ever be called with a SimulateBulkRequest but got a " + bulkRequest.getClass(); - Map> rawTemplateSubstitutions = ((SimulateBulkRequest) bulkRequest).getTemplateSubstitutions(); - Map componentTemplateSubstitutions; - try { - componentTemplateSubstitutions = getComponentTemplateSubstitutionsFromRaw(rawTemplateSubstitutions); - } catch (IOException e) { - throw new RuntimeException(e); - } + Map componentTemplateSubstitutions = bulkRequest.getComponentTemplateSubstitutions(); for (int i = 0; i < bulkRequest.requests.size(); i++) { DocWriteRequest docRequest = bulkRequest.requests.get(i); assert docRequest instanceof IndexRequest : "TransportSimulateBulkAction should only ever be called with IndexRequests"; @@ -153,35 +144,6 @@ protected void doInternalExecute( ); } - static Map getComponentTemplateSubstitutionsFromRaw( - Map> rawTemplateSubstitutions - ) throws IOException { - if (rawTemplateSubstitutions == null) { - return Map.of(); - } - Map result = new HashMap<>(rawTemplateSubstitutions.size()); - for (Map.Entry> rawEntry : rawTemplateSubstitutions.entrySet()) { - result.put(rawEntry.getKey(), convertRawTemplateToComponentTemplate(rawEntry.getValue())); - } - return result; - } - - @SuppressWarnings("unchecked") - private static ComponentTemplate convertRawTemplateToComponentTemplate(Map rawTemplate) throws IOException { - Settings settings = null; - CompressedXContent mappings = null; - if (rawTemplate.containsKey("mappings")) { - mappings = new CompressedXContent((Map) rawTemplate.get("mappings")); - } - if (rawTemplate.containsKey("settings")) { - settings = Settings.builder().loadFromMap((Map) rawTemplate.get("settings")).build(); - } - Map aliases = null; - DataStreamLifecycle lifecycle = null; - Template template = new Template(settings, mappings, aliases, lifecycle); - return new ComponentTemplate(template, null, null); - } - /** * This creates a temporary index with the mappings of the index in the request, and then attempts to index the source from the request * into it. If there is a mapping exception, that exception is returned. On success the returned exception is null. diff --git a/server/src/test/java/org/elasticsearch/action/bulk/TransportSimulateBulkActionTests.java b/server/src/test/java/org/elasticsearch/action/bulk/TransportSimulateBulkActionTests.java index a6044c10b7f4c..b0da1d918ac67 100644 --- a/server/src/test/java/org/elasticsearch/action/bulk/TransportSimulateBulkActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/bulk/TransportSimulateBulkActionTests.java @@ -132,7 +132,7 @@ public void tearDown() throws Exception { super.tearDown(); } - public void testIndexData() { + public void testIndexData() throws IOException { Task task = mock(Task.class); // unused BulkRequest bulkRequest = new SimulateBulkRequest((Map>) null, (Map>) null); int bulkItemCount = randomIntBetween(0, 200); From ae91dacdd16fc19cf65fa1df3cdc1aae1529b63b Mon Sep 17 00:00:00 2001 From: Keith Massey Date: Wed, 11 Sep 2024 17:27:33 -0500 Subject: [PATCH 07/28] adding back accidentally removed line --- .../indices/template/post/TransportSimulateTemplateAction.java | 1 + 1 file changed, 1 insertion(+) diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateTemplateAction.java index 1ca9bc1b14ffb..40ee8f68be13d 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateTemplateAction.java @@ -160,6 +160,7 @@ protected void masterOperation( Map> overlapping = new HashMap<>(); overlapping.putAll(findConflictingV1Templates(tempClusterState, matchingTemplate, templateV2.indexPatterns())); overlapping.putAll(findConflictingV2Templates(tempClusterState, matchingTemplate, templateV2.indexPatterns())); + Template template = TransportSimulateIndexTemplateAction.resolveTemplate( matchingTemplate, temporaryIndexName, From c65544e93261759ad6affe76af321d7d566e8f9a Mon Sep 17 00:00:00 2001 From: Keith Massey Date: Wed, 11 Sep 2024 17:34:38 -0500 Subject: [PATCH 08/28] adding comments to TransportSimulateBulkAction --- .../bulk/TransportSimulateBulkAction.java | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java b/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java index 2172ee02b2d15..f8be019c80f64 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java @@ -165,6 +165,10 @@ private Exception validateMappings(Map componentTempl IndexAbstraction indexAbstraction = state.metadata().getIndicesLookup().get(request.index()); try { if (indexAbstraction != null && componentTemplateSubstitutions.isEmpty()) { + /* + * In this case the index exists and we don't have any component template overrides. So we can just use withTempIndexService + * to do the mapping validation, using all the existing logic for validation. + */ IndexMetadata imd = state.metadata().getIndexSafe(indexAbstraction.getWriteIndex(request, state.metadata())); indicesService.withTempIndexService(imd, indexService -> { indexService.mapperService().updateMapping(null, imd); @@ -185,13 +189,14 @@ private Exception validateMappings(Map componentTempl }); } else { /* - * The index did not exist, so we put together the mappings from existing templates. - * This reproduces a lot of the mapping resolution logic in MetadataCreateIndexService.applyCreateIndexRequest(). However, - * it does not deal with aliases (since an alias cannot be created if an index does not exist, and this is the path for - * when the index does not exist). And it does not deal with system indices since we do not intend for users to simulate - * writing to system indices. + * The index did not exist, or we have component template substitutions, so we put together the mappings from existing + * templates This reproduces a lot of the mapping resolution logic in MetadataCreateIndexService.applyCreateIndexRequest(). + * However, it does not deal with aliases (since an alias cannot be created if an index does not exist, and this is the + * path for when the index does not exist). And it does not deal with system indices since we do not intend for users to + * simulate writing to system indices. */ - ClusterState simulatedState = new ClusterState.Builder(state).metadata( + // First, we remove the index from the cluster state if necessary (since we're going to use the templates) + ClusterState simulatedState = indexAbstraction == null ? state : new ClusterState.Builder(state).metadata( new Metadata.Builder(state.metadata()).remove(request.index()).build() ).build(); String matchingTemplate = findV2Template(state.metadata(), request.index(), false); From 06c9444b50d6af81634fbad4d0e8ff128c0528ce Mon Sep 17 00:00:00 2001 From: Keith Massey Date: Wed, 11 Sep 2024 17:50:04 -0500 Subject: [PATCH 09/28] removing duplicate methods --- .../bulk/TransportSimulateBulkAction.java | 7 +++--- .../metadata/MetadataCreateIndexService.java | 12 ++-------- .../MetadataIndexTemplateService.java | 16 ++------------ .../elasticsearch/ingest/IngestService.java | 9 -------- .../MetadataIndexTemplateServiceTests.java | 22 +++++++++++++++---- .../ingest/IngestServiceTests.java | 2 +- 6 files changed, 27 insertions(+), 41 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java b/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java index f8be019c80f64..01aa1200cf0f4 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java @@ -196,9 +196,10 @@ private Exception validateMappings(Map componentTempl * simulate writing to system indices. */ // First, we remove the index from the cluster state if necessary (since we're going to use the templates) - ClusterState simulatedState = indexAbstraction == null ? state : new ClusterState.Builder(state).metadata( - new Metadata.Builder(state.metadata()).remove(request.index()).build() - ).build(); + ClusterState simulatedState = indexAbstraction == null + ? state + : new ClusterState.Builder(state).metadata(new Metadata.Builder(state.metadata()).remove(request.index()).build()) + .build(); String matchingTemplate = findV2Template(state.metadata(), request.index(), false); if (matchingTemplate != null) { final Template template = TransportSimulateIndexTemplateAction.resolveTemplate( diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java index 44fb354c2a944..21cbd431e75f6 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java @@ -647,6 +647,7 @@ private ClusterState applyCreateIndexRequestWithV2Template( request.mappings(), currentState, templateName, + Map.of(), xContentRegistry, request.index() ); @@ -805,21 +806,12 @@ private static List collectSystemV2Mappings( List templateMappings = MetadataIndexTemplateService.collectMappings( composableIndexTemplate, componentTemplates, + Map.of(), indexName ); return collectV2Mappings(null, templateMappings, xContentRegistry); } - public static List collectV2Mappings( - @Nullable final String requestMappings, - final ClusterState currentState, - final String templateName, - final NamedXContentRegistry xContentRegistry, - final String indexName - ) throws Exception { - return collectV2Mappings(requestMappings, currentState, templateName, Map.of(), xContentRegistry, indexName); - } - public static List collectV2Mappings( @Nullable final String requestMappings, final ClusterState currentState, diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java index 05dfec83682be..a84b6ff0f9718 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java @@ -697,7 +697,7 @@ private void validateIndexTemplateV2(String name, ComposableIndexTemplate indexT final var now = Instant.now(); final var metadata = currentState.getMetadata(); - final var combinedMappings = collectMappings(indexTemplate, metadata.componentTemplates(), "tmp_idx"); + final var combinedMappings = collectMappings(indexTemplate, metadata.componentTemplates(), Map.of(), "tmp_idx"); final var combinedSettings = resolveSettings(indexTemplate, metadata.componentTemplates()); // First apply settings sourced from index setting providers: for (var provider : indexSettingProviders) { @@ -1347,10 +1347,6 @@ private static boolean isGlobalAndHasIndexHiddenSetting(Metadata metadata, Compo /** * Collect the given v2 template into an ordered list of mappings. */ - public static List collectMappings(final ClusterState state, final String templateName, final String indexName) { - return collectMappings(state, templateName, Map.of(), indexName); - } - public static List collectMappings( final ClusterState state, final String templateName, @@ -1372,14 +1368,6 @@ public static List collectMappings( * Collect the given v2 template into an ordered list of mappings. */ public static List collectMappings( - final ComposableIndexTemplate template, - final Map componentTemplates, - final String indexName - ) { - return collectMappings(template, componentTemplates, Map.of(), indexName); - } - - private static List collectMappings( final ComposableIndexTemplate template, final Map componentTemplates, final Map componentTemplateSubstitutions, @@ -1733,7 +1721,7 @@ private static void validateCompositeTemplate( String indexName = DataStream.BACKING_INDEX_PREFIX + temporaryIndexName; // Parse mappings to ensure they are valid after being composed - List mappings = collectMappings(stateWithIndex, templateName, indexName); + List mappings = collectMappings(stateWithIndex, templateName, Map.of(), indexName); try { MapperService mapperService = tempIndexService.mapperService(); mapperService.merge(MapperService.SINGLE_MAPPING_NAME, mappings, MapperService.MergeReason.INDEX_TEMPLATE); diff --git a/server/src/main/java/org/elasticsearch/ingest/IngestService.java b/server/src/main/java/org/elasticsearch/ingest/IngestService.java index 92a36df535eea..4fe7ff9a3265b 100644 --- a/server/src/main/java/org/elasticsearch/ingest/IngestService.java +++ b/server/src/main/java/org/elasticsearch/ingest/IngestService.java @@ -282,15 +282,6 @@ public static void resolvePipelinesAndUpdateIndexRequest( resolvePipelinesAndUpdateIndexRequest(originalRequest, indexRequest, metadata, System.currentTimeMillis(), templateSubstitutions); } - static void resolvePipelinesAndUpdateIndexRequest( - final DocWriteRequest originalRequest, - final IndexRequest indexRequest, - final Metadata metadata, - final long epochMillis - ) { - resolvePipelinesAndUpdateIndexRequest(originalRequest, indexRequest, metadata, epochMillis, Map.of()); - } - static void resolvePipelinesAndUpdateIndexRequest( final DocWriteRequest originalRequest, final IndexRequest indexRequest, diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java index c6eb65381fab6..129195263e079 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java @@ -1080,7 +1080,7 @@ public void testResolveConflictingMappings() throws Exception { .build(); state = service.addIndexTemplateV2(state, true, "my-template", it); - List mappings = MetadataIndexTemplateService.collectMappings(state, "my-template", "my-index"); + List mappings = MetadataIndexTemplateService.collectMappings(state, "my-template", Map.of(), "my-index"); assertNotNull(mappings); assertThat(mappings.size(), equalTo(3)); @@ -1142,7 +1142,7 @@ public void testResolveMappings() throws Exception { .build(); state = service.addIndexTemplateV2(state, true, "my-template", it); - List mappings = MetadataIndexTemplateService.collectMappings(state, "my-template", "my-index"); + List mappings = MetadataIndexTemplateService.collectMappings(state, "my-template", Map.of(), "my-index"); assertNotNull(mappings); assertThat(mappings.size(), equalTo(3)); @@ -1196,6 +1196,7 @@ public void testDefinedTimestampMappingIsAddedForDataStreamTemplates() throws Ex List mappings = MetadataIndexTemplateService.collectMappings( state, "logs-data-stream-template", + Map.of(), DataStream.getDefaultBackingIndexName("logs", 1L) ); @@ -1247,7 +1248,12 @@ public void testDefinedTimestampMappingIsAddedForDataStreamTemplates() throws Ex .build(); state = service.addIndexTemplateV2(state, true, "timeseries-template", it); - List mappings = MetadataIndexTemplateService.collectMappings(state, "timeseries-template", "timeseries"); + List mappings = MetadataIndexTemplateService.collectMappings( + state, + "timeseries-template", + Map.of(), + "timeseries" + ); assertNotNull(mappings); assertThat(mappings.size(), equalTo(2)); @@ -1269,6 +1275,7 @@ public void testDefinedTimestampMappingIsAddedForDataStreamTemplates() throws Ex mappings = MetadataIndexTemplateService.collectMappings( state, "timeseries-template", + Map.of(), DataStream.getDefaultBackingIndexName("timeseries", 1L) ); @@ -1317,6 +1324,7 @@ public void testUserDefinedMappingTakesPrecedenceOverDefault() throws Exception List mappings = MetadataIndexTemplateService.collectMappings( state, "logs-template", + Map.of(), DataStream.getDefaultBackingIndexName("logs", 1L) ); @@ -1373,6 +1381,7 @@ public void testUserDefinedMappingTakesPrecedenceOverDefault() throws Exception List mappings = MetadataIndexTemplateService.collectMappings( state, "timeseries-template", + Map.of(), DataStream.getDefaultBackingIndexName("timeseries-template", 1L) ); @@ -2439,7 +2448,12 @@ public void testComposableTemplateWithSubobjectsFalse() throws Exception { .build(); state = service.addIndexTemplateV2(state, true, "composable-template", it); - List mappings = MetadataIndexTemplateService.collectMappings(state, "composable-template", "test-index"); + List mappings = MetadataIndexTemplateService.collectMappings( + state, + "composable-template", + Map.of(), + "test-index" + ); assertNotNull(mappings); assertThat(mappings.size(), equalTo(2)); diff --git a/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java b/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java index 5c07c2344cf13..14d867d4bc844 100644 --- a/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java +++ b/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java @@ -2585,7 +2585,7 @@ public void testResolveFinalPipelineWithDateMathExpression() { // index name matches with IDM: IndexRequest indexRequest = new IndexRequest(""); - IngestService.resolvePipelinesAndUpdateIndexRequest(indexRequest, indexRequest, metadata, epochMillis); + IngestService.resolvePipelinesAndUpdateIndexRequest(indexRequest, indexRequest, metadata, epochMillis, Map.of()); assertTrue(hasPipeline(indexRequest)); assertTrue(indexRequest.isPipelineResolved()); assertThat(indexRequest.getPipeline(), equalTo("_none")); From fecb62d081b547ffe7d805d0977ce0b4b29bfa25 Mon Sep 17 00:00:00 2001 From: Keith Massey Date: Thu, 12 Sep 2024 13:31:16 -0500 Subject: [PATCH 10/28] fixing shallowClone methods --- .../java/org/elasticsearch/action/bulk/BulkRequest.java | 6 +++--- .../org/elasticsearch/action/bulk/SimulateBulkRequest.java | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/bulk/BulkRequest.java b/server/src/main/java/org/elasticsearch/action/bulk/BulkRequest.java index ebffd451a262e..4237fb268430e 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/BulkRequest.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/BulkRequest.java @@ -490,9 +490,9 @@ public Map getComponentTemplateSubstitutions() throws */ public BulkRequest shallowClone() { BulkRequest bulkRequest = new BulkRequest(); - bulkRequest.setRefreshPolicy(bulkRequest.getRefreshPolicy()); - bulkRequest.waitForActiveShards(bulkRequest.waitForActiveShards()); - bulkRequest.timeout(bulkRequest.timeout()); + bulkRequest.setRefreshPolicy(getRefreshPolicy()); + bulkRequest.waitForActiveShards(waitForActiveShards()); + bulkRequest.timeout(timeout()); return bulkRequest; } } diff --git a/server/src/main/java/org/elasticsearch/action/bulk/SimulateBulkRequest.java b/server/src/main/java/org/elasticsearch/action/bulk/SimulateBulkRequest.java index 5d0b2f71919fd..c7591e3c6e3ef 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/SimulateBulkRequest.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/SimulateBulkRequest.java @@ -157,9 +157,9 @@ private static ComponentTemplate convertRawTemplateToComponentTemplate(Map Date: Thu, 12 Sep 2024 13:34:46 -0500 Subject: [PATCH 11/28] Changing template_substitutions to component_template_substitutions --- .../rest-api-spec/test/ingest/80_ingest_simulate.yml | 4 ++-- .../org/elasticsearch/action/bulk/SimulateBulkRequest.java | 2 +- .../rest/action/ingest/RestSimulateIngestAction.java | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/80_ingest_simulate.yml b/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/80_ingest_simulate.yml index cb62e28fed97c..c7c67213d341a 100644 --- a/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/80_ingest_simulate.yml +++ b/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/80_ingest_simulate.yml @@ -401,7 +401,7 @@ setup: } } ], - "template_substitutions": { + "component_template_substitutions": { "mappings_template": { "mappings": { "dynamic": "true", @@ -450,7 +450,7 @@ setup: } } ], - "template_substitutions": { + "component_template_substitutions": { "mappings_template": { "mappings": { "dynamic": "true", diff --git a/server/src/main/java/org/elasticsearch/action/bulk/SimulateBulkRequest.java b/server/src/main/java/org/elasticsearch/action/bulk/SimulateBulkRequest.java index c7591e3c6e3ef..71090c14f29e2 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/SimulateBulkRequest.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/SimulateBulkRequest.java @@ -53,7 +53,7 @@ * ] * } * }, - * "template_substitutions": { + * "component_template_substitutions": { * "my-template-1": { * "settings": { * "number_of_shards": 1 diff --git a/server/src/main/java/org/elasticsearch/rest/action/ingest/RestSimulateIngestAction.java b/server/src/main/java/org/elasticsearch/rest/action/ingest/RestSimulateIngestAction.java index 88b1d29f0c405..91d5919fe6737 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/ingest/RestSimulateIngestAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/ingest/RestSimulateIngestAction.java @@ -75,7 +75,7 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC Map sourceMap = XContentHelper.convertToMap(sourceTuple.v2(), false, sourceTuple.v1()).v2(); SimulateBulkRequest bulkRequest = new SimulateBulkRequest( (Map>) sourceMap.remove("pipeline_substitutions"), - (Map>) sourceMap.remove("template_substitutions") + (Map>) sourceMap.remove("component_template_substitutions") ); BytesReference transformedData = convertToBulkRequestXContentBytes(sourceMap); bulkRequest.add( From d30bf927ed14f49476d84e679bfc31f6c61ab286 Mon Sep 17 00:00:00 2001 From: Keith Massey Date: Thu, 12 Sep 2024 13:39:05 -0500 Subject: [PATCH 12/28] removing verbose logging --- .../src/main/java/org/elasticsearch/ingest/IngestService.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/ingest/IngestService.java b/server/src/main/java/org/elasticsearch/ingest/IngestService.java index 4fe7ff9a3265b..2d34e5526f12a 100644 --- a/server/src/main/java/org/elasticsearch/ingest/IngestService.java +++ b/server/src/main/java/org/elasticsearch/ingest/IngestService.java @@ -304,10 +304,8 @@ static void resolvePipelinesAndUpdateIndexRequest( } Pipelines pipelines; if (templateSubstitutions.isEmpty() == false) { - logger.info("There were template substitutions"); pipelines = pipelinesFromTemplates.or(() -> pipelinesFromMetadata).orElse(Pipelines.NO_PIPELINES_DEFINED); } else { - logger.info("There were no template substitutions"); pipelines = pipelinesFromMetadata.or(() -> pipelinesFromTemplates).orElse(Pipelines.NO_PIPELINES_DEFINED); } From 13ad4343d228da0e2e5ac2e883ebadfe065a2a58 Mon Sep 17 00:00:00 2001 From: Keith Massey Date: Thu, 12 Sep 2024 17:22:29 -0500 Subject: [PATCH 13/28] adding an integration test --- .../bulk/TransportSimulateBulkActionIT.java | 103 ++++++++++++++++++ .../org/elasticsearch/TransportVersions.java | 2 +- .../action/bulk/SimulateBulkRequest.java | 26 +++-- .../action/bulk/TransportBulkAction.java | 2 + 4 files changed, 120 insertions(+), 13 deletions(-) diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/TransportSimulateBulkActionIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/TransportSimulateBulkActionIT.java index f976a3ed80d58..fc726fea4d067 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/TransportSimulateBulkActionIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/TransportSimulateBulkActionIT.java @@ -14,12 +14,14 @@ import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; +import org.elasticsearch.action.admin.indices.template.put.PutComponentTemplateAction; import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest; import org.elasticsearch.action.admin.indices.template.put.TransportPutComposableIndexTemplateAction; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.ingest.SimulateIndexResponse; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.cluster.metadata.ComponentTemplate; import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; import org.elasticsearch.cluster.metadata.Template; import org.elasticsearch.common.compress.CompressedXContent; @@ -86,6 +88,107 @@ public void testMappingValidationIndexExists() { assertThat(fields.size(), equalTo(1)); } + @SuppressWarnings("unchecked") + public void testMappingValidationIndexExistsWithComponentTemplate() throws IOException { + /* + * This test simulates a BulkRequest of two documents into an existing index. Then we make sure the index contains no documents, and + * that the index's mapping in the cluster state has not been updated with the two new field. With the mapping from the template + * that was used to create the index, we would expect the second document to throw an exception because it uses a field that does + * not exist. But we substitute a new version of the component template named "test-component-template" that allows for the new + * field. + */ + String originalComponentTemplateMappingString = """ + { + "_doc":{ + "dynamic":"strict", + "properties":{ + "foo1":{ + "type":"text" + } + } + } + } + """; + CompressedXContent mapping = CompressedXContent.fromJSON(originalComponentTemplateMappingString); + Template template = new Template(Settings.EMPTY, mapping, null); + PutComponentTemplateAction.Request componentTemplateActionRequest = new PutComponentTemplateAction.Request( + "test-component-template" + ); + ComponentTemplate componentTemplate = new ComponentTemplate(template, null, null); + componentTemplateActionRequest.componentTemplate(componentTemplate); + client().execute(PutComponentTemplateAction.INSTANCE, componentTemplateActionRequest).actionGet(); + ComposableIndexTemplate composableIndexTemplate = ComposableIndexTemplate.builder() + .indexPatterns(List.of("my-index-*")) + .componentTemplates(List.of("test-component-template")) + .build(); + TransportPutComposableIndexTemplateAction.Request request = new TransportPutComposableIndexTemplateAction.Request("test"); + request.indexTemplate(composableIndexTemplate); + client().execute(TransportPutComposableIndexTemplateAction.TYPE, request).actionGet(); + + String indexName = "my-index-1"; + // First, run before the index is created: + assertMappingsUpdatedFromComponentTemplateSubstitutions(indexName); + // Now, create the index and make sure the component template substitutions work the same: + indicesAdmin().create(new CreateIndexRequest(indexName)).actionGet(); + assertMappingsUpdatedFromComponentTemplateSubstitutions(indexName); + // Now make sure nothing was actually changed: + indicesAdmin().refresh(new RefreshRequest(indexName)).actionGet(); + SearchResponse searchResponse = client().search(new SearchRequest(indexName)).actionGet(); + assertThat(searchResponse.getHits().getTotalHits().value, equalTo(0L)); + searchResponse.decRef(); + ClusterStateResponse clusterStateResponse = admin().cluster().state(new ClusterStateRequest(TEST_REQUEST_TIMEOUT)).actionGet(); + Map indexMapping = clusterStateResponse.getState().metadata().index(indexName).mapping().sourceAsMap(); + Map fields = (Map) indexMapping.get("properties"); + assertThat(fields.size(), equalTo(1)); + } + + private void assertMappingsUpdatedFromComponentTemplateSubstitutions(String indexName) { + IndexRequest indexRequest1 = new IndexRequest(indexName).source(""" + { + "foo1": "baz" + } + """, XContentType.JSON).id(randomUUID()); + IndexRequest indexRequest2 = new IndexRequest(indexName).source(""" + { + "foo3": "baz" + } + """, XContentType.JSON).id(randomUUID()); + { + // First we use the original component template, and expect a failure in the second document: + BulkRequest bulkRequest = new SimulateBulkRequest(Map.of(), Map.of()); + bulkRequest.add(indexRequest1); + bulkRequest.add(indexRequest2); + BulkResponse response = client().execute(new ActionType(SimulateBulkAction.NAME), bulkRequest).actionGet(); + assertThat(response.getItems().length, equalTo(2)); + assertThat(response.getItems()[0].getResponse().getResult(), equalTo(DocWriteResponse.Result.CREATED)); + assertNull(((SimulateIndexResponse) response.getItems()[0].getResponse()).getException()); + assertThat(response.getItems()[1].getResponse().getResult(), equalTo(DocWriteResponse.Result.CREATED)); + assertThat( + ((SimulateIndexResponse) response.getItems()[1].getResponse()).getException().getMessage(), + containsString("mapping set to strict, dynamic introduction of") + ); + } + + { + // Now we substitute a "test-component-template" that defines both fields, so we expect no exception: + BulkRequest bulkRequest = new SimulateBulkRequest( + Map.of(), + Map.of( + "test-component-template", + Map.of("dynamic", "strict", "properties", Map.of("foo1", Map.of("type", "text"), "foo3", Map.of("type", "text"))) + ) + ); + bulkRequest.add(indexRequest1); + bulkRequest.add(indexRequest2); + BulkResponse response = client().execute(new ActionType(SimulateBulkAction.NAME), bulkRequest).actionGet(); + assertThat(response.getItems().length, equalTo(2)); + assertThat(response.getItems()[0].getResponse().getResult(), equalTo(DocWriteResponse.Result.CREATED)); + assertNull(((SimulateIndexResponse) response.getItems()[0].getResponse()).getException()); + assertThat(response.getItems()[1].getResponse().getResult(), equalTo(DocWriteResponse.Result.CREATED)); + assertNull(((SimulateIndexResponse) response.getItems()[1].getResponse()).getException()); + } + } + public void testMappingValidationIndexDoesNotExistsNoTemplate() { /* * This test simulates a BulkRequest of two documents into an index that does not exist. There is no template (other than the diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java index 70c5ef1d11688..580ae4a0c5a33 100644 --- a/server/src/main/java/org/elasticsearch/TransportVersions.java +++ b/server/src/main/java/org/elasticsearch/TransportVersions.java @@ -208,7 +208,7 @@ static TransportVersion def(int id) { public static final TransportVersion ESQL_AGGREGATE_EXEC_TRACKS_INTERMEDIATE_ATTRS = def(8_738_00_0); public static final TransportVersion CCS_TELEMETRY_STATS = def(8_739_00_0); public static final TransportVersion GLOBAL_RETENTION_TELEMETRY = def(8_740_00_0); - public static final TransportVersion SIMULATE_TEMPLATES_SUBSTITUTIONS = def(8_741_00_0); + public static final TransportVersion SIMULATE_COMPONENT_TEMPLATES_SUBSTITUTIONS = def(8_741_00_0); /* * STOP! READ THIS FIRST! No, really, diff --git a/server/src/main/java/org/elasticsearch/action/bulk/SimulateBulkRequest.java b/server/src/main/java/org/elasticsearch/action/bulk/SimulateBulkRequest.java index 71090c14f29e2..00be5cc078956 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/SimulateBulkRequest.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/SimulateBulkRequest.java @@ -81,30 +81,32 @@ */ public class SimulateBulkRequest extends BulkRequest { private final Map> pipelineSubstitutions; - private final Map> templateSubstitutions; + private final Map> componentTemplateSubstitutions; /** * @param pipelineSubstitutions The pipeline definitions that are to be used in place of any pre-existing pipeline definitions with * the same pipelineId. The key of the map is the pipelineId, and the value the pipeline definition as * parsed by XContentHelper.convertToMap(). + * @param componentTemplateSubstitutions The component template definitions that are to be used in place of any pre-existing + * component template definitions with the same name. */ public SimulateBulkRequest( @Nullable Map> pipelineSubstitutions, - @Nullable Map> templateSubstitutions + @Nullable Map> componentTemplateSubstitutions ) { super(); this.pipelineSubstitutions = pipelineSubstitutions; - this.templateSubstitutions = templateSubstitutions; + this.componentTemplateSubstitutions = componentTemplateSubstitutions; } @SuppressWarnings("unchecked") public SimulateBulkRequest(StreamInput in) throws IOException { super(in); this.pipelineSubstitutions = (Map>) in.readGenericValue(); - if (in.getTransportVersion().onOrAfter(TransportVersions.SIMULATE_TEMPLATES_SUBSTITUTIONS)) { - this.templateSubstitutions = (Map>) in.readGenericValue(); + if (in.getTransportVersion().onOrAfter(TransportVersions.SIMULATE_COMPONENT_TEMPLATES_SUBSTITUTIONS)) { + this.componentTemplateSubstitutions = (Map>) in.readGenericValue(); } else { - templateSubstitutions = Map.of(); + componentTemplateSubstitutions = Map.of(); } } @@ -112,8 +114,8 @@ public SimulateBulkRequest(StreamInput in) throws IOException { public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); out.writeGenericValue(pipelineSubstitutions); - if (out.getTransportVersion().onOrAfter(TransportVersions.SIMULATE_TEMPLATES_SUBSTITUTIONS)) { - out.writeGenericValue(templateSubstitutions); + if (out.getTransportVersion().onOrAfter(TransportVersions.SIMULATE_COMPONENT_TEMPLATES_SUBSTITUTIONS)) { + out.writeGenericValue(componentTemplateSubstitutions); } } @@ -128,11 +130,11 @@ public boolean isSimulated() { @Override public Map getComponentTemplateSubstitutions() throws IOException { - if (templateSubstitutions == null) { + if (componentTemplateSubstitutions == null) { return Map.of(); } - Map result = new HashMap<>(templateSubstitutions.size()); - for (Map.Entry> rawEntry : templateSubstitutions.entrySet()) { + Map result = new HashMap<>(componentTemplateSubstitutions.size()); + for (Map.Entry> rawEntry : componentTemplateSubstitutions.entrySet()) { result.put(rawEntry.getKey(), convertRawTemplateToComponentTemplate(rawEntry.getValue())); } return result; @@ -156,7 +158,7 @@ private static ComponentTemplate convertRawTemplateToComponentTemplate(Map listener, long relativeStartTimeNanos ) { + assert (bulkRequest instanceof SimulateBulkRequest) == false + : "TransportBulkAction should never be called with a SimulateBulkRequest"; trackIndexRequests(bulkRequest); Map indicesToAutoCreate = new HashMap<>(); From c7dd111d13e7383d71609eae0e0073abf1cae2f6 Mon Sep 17 00:00:00 2001 From: Keith Massey Date: Fri, 13 Sep 2024 09:17:11 -0500 Subject: [PATCH 14/28] fixing behavior when component template overrides remove pipeline config --- .../test/ingest/80_ingest_simulate.yml | 117 ++++++++++++++++++ .../elasticsearch/ingest/IngestService.java | 31 ++--- 2 files changed, 134 insertions(+), 14 deletions(-) diff --git a/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/80_ingest_simulate.yml b/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/80_ingest_simulate.yml index c7c67213d341a..692231b07bd33 100644 --- a/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/80_ingest_simulate.yml +++ b/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/80_ingest_simulate.yml @@ -477,3 +477,120 @@ setup: - match: { docs.0.doc._source.foo: "FOO" } - match: { docs.0.doc.executed_pipelines: ["bar-pipeline"] } - not_exists: docs.0.doc.error + +--- +"Test ingest simulate with template substitutions for component templates removing pipelines": + + - skip: + features: + - headers + - allowed_warnings + + - requires: + cluster_features: ["simulate.component.template.substitutions"] + reason: "ingest simulate component template substitutions added in 8.16" + + - do: + headers: + Content-Type: application/json + ingest.put_pipeline: + id: "foo-pipeline" + body: > + { + "processors": [ + { + "set": { + "field": "foo", + "value": true + } + } + ] + } + - match: { acknowledged: true } + + - do: + cluster.put_component_template: + name: settings_template + body: + template: + settings: + index: + default_pipeline: "foo_pipeline" + + - do: + allowed_warnings: + - "index template [test-composable-1] has index patterns [tsdb_templated_*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [test-composable-1] will take precedence during new index creation" + indices.put_index_template: + name: test-composable-1 + body: + index_patterns: + - foo* + composed_of: + - settings_template + + - do: + headers: + Content-Type: application/json + simulate.ingest: + index: foo-1 + body: > + { + "docs": [ + { + "_id": "asdf", + "_source": { + "foo": "FOO" + } + } + ], + "component_template_substitutions": { + "settings_template": { + "settings": { + "index": { + "default_pipeline": null + } + } + } + } + } + - length: { docs: 1 } + - match: { docs.0.doc._index: "foo-1" } + - match: { docs.0.doc._source.foo: "FOO" } + - match: { docs.0.doc.executed_pipelines: [] } + - not_exists: docs.0.doc.error + + - do: + indices.create: + index: foo-1 + - match: { acknowledged: true } + + - do: + headers: + Content-Type: application/json + simulate.ingest: + index: foo-1 + body: > + { + "docs": [ + { + "_id": "asdf", + "_source": { + "foo": "FOO" + } + } + ], + "component_template_substitutions": { + "settings_template": { + "settings": { + "index": { + "default_pipeline": null + } + } + } + } + } + - length: { docs: 1 } + - match: { docs.0.doc._index: "foo-1" } + - match: { docs.0.doc._source.foo: "FOO" } + - match: { docs.0.doc.executed_pipelines: [] } + - not_exists: docs.0.doc.error diff --git a/server/src/main/java/org/elasticsearch/ingest/IngestService.java b/server/src/main/java/org/elasticsearch/ingest/IngestService.java index 9221cf08c496f..ea2dd9ae4cbc7 100644 --- a/server/src/main/java/org/elasticsearch/ingest/IngestService.java +++ b/server/src/main/java/org/elasticsearch/ingest/IngestService.java @@ -287,29 +287,32 @@ static void resolvePipelinesAndUpdateIndexRequest( final IndexRequest indexRequest, final Metadata metadata, final long epochMillis, - final Map templateSubstitutions + final Map componentTemplateSubstitutions ) { if (indexRequest.isPipelineResolved()) { return; } - String requestPipeline = indexRequest.getPipeline(); - - Optional pipelinesFromMetadata = resolvePipelinesFromMetadata(originalRequest, indexRequest, metadata, epochMillis); - Optional pipelinesFromTemplates; - if (templateSubstitutions.isEmpty() == false || pipelinesFromMetadata.isEmpty()) { - pipelinesFromTemplates = resolvePipelinesFromIndexTemplates(indexRequest, metadata, templateSubstitutions); - } else { - pipelinesFromTemplates = Optional.empty(); - } - Pipelines pipelines; - if (templateSubstitutions.isEmpty() == false) { - pipelines = pipelinesFromTemplates.or(() -> pipelinesFromMetadata).orElse(Pipelines.NO_PIPELINES_DEFINED); + /* + * Here we look for the pipelines associated with the index if the index exists. If the index does not exist we fall back to using + * templates to find the pipelines. But if a user has passed in component template substitutions, they want the settings from those + * used in place of the settings used to create any previous indices. So in that case we use the templates to find the pipelines -- + * we don't fall back to the existing index if we don't find any because it is possible the user has intentionally removed the + * pipeline. + */ + final Pipelines pipelines; + if (componentTemplateSubstitutions.isEmpty()) { + pipelines = resolvePipelinesFromMetadata(originalRequest, indexRequest, metadata, epochMillis) // + .or(() -> resolvePipelinesFromIndexTemplates(indexRequest, metadata, Map.of())) + .orElse(Pipelines.NO_PIPELINES_DEFINED); } else { - pipelines = pipelinesFromMetadata.or(() -> pipelinesFromTemplates).orElse(Pipelines.NO_PIPELINES_DEFINED); + pipelines = resolvePipelinesFromIndexTemplates(indexRequest, metadata, componentTemplateSubstitutions).orElse( + Pipelines.NO_PIPELINES_DEFINED + ); } // The pipeline coming as part of the request always has priority over the resolved one from metadata or templates + String requestPipeline = indexRequest.getPipeline(); if (requestPipeline != null) { indexRequest.setPipeline(requestPipeline); } else { From 43558678015b1f0ecf8e6a20de2c3e500e23f63c Mon Sep 17 00:00:00 2001 From: Keith Massey Date: Fri, 13 Sep 2024 12:56:17 -0500 Subject: [PATCH 15/28] unit testing SimulateBulkRequestTests --- .../lib/NativeLibraryProvider.java | 2 +- .../action/bulk/SimulateBulkRequestTests.java | 74 +++++++++++++++++++ 2 files changed, 75 insertions(+), 1 deletion(-) diff --git a/libs/native/src/main/java/org/elasticsearch/nativeaccess/lib/NativeLibraryProvider.java b/libs/native/src/main/java/org/elasticsearch/nativeaccess/lib/NativeLibraryProvider.java index 737d5aecd0d0a..43c45c82e7d85 100644 --- a/libs/native/src/main/java/org/elasticsearch/nativeaccess/lib/NativeLibraryProvider.java +++ b/libs/native/src/main/java/org/elasticsearch/nativeaccess/lib/NativeLibraryProvider.java @@ -64,7 +64,7 @@ public T getLibrary(Class cls) { private static NativeLibraryProvider loadProvider() { final int runtimeVersion = Runtime.version().feature(); if (runtimeVersion >= 21) { - return loadJdkImpl(runtimeVersion); + // return loadJdkImpl(runtimeVersion); } return loadJnaImpl(); } diff --git a/server/src/test/java/org/elasticsearch/action/bulk/SimulateBulkRequestTests.java b/server/src/test/java/org/elasticsearch/action/bulk/SimulateBulkRequestTests.java index a03fcfa9f9e8e..72a01adc20a3d 100644 --- a/server/src/test/java/org/elasticsearch/action/bulk/SimulateBulkRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/bulk/SimulateBulkRequestTests.java @@ -8,13 +8,19 @@ package org.elasticsearch.action.bulk; +import org.elasticsearch.cluster.metadata.ComponentTemplate; +import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xcontent.XContentType; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.util.List; import java.util.Map; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.instanceOf; public class SimulateBulkRequestTests extends ESTestCase { @@ -39,6 +45,74 @@ private void testSerialization( assertThat(copy.getPipelineSubstitutions(), equalTo(simulateBulkRequest.getPipelineSubstitutions())); } + @SuppressWarnings("unchecked") + public void testGetComponentTemplateSubstitutions() throws IOException { + SimulateBulkRequest simulateBulkRequest = new SimulateBulkRequest(Map.of(), Map.of()); + assertThat(simulateBulkRequest.getComponentTemplateSubstitutions(), equalTo(Map.of())); + String substituteComponentTemplatesString = """ + { + "mappings_template": { + "mappings": { + "dynamic": "true", + "properties": { + "foo": { + "type": "keyword" + } + } + } + }, + "settings_template": { + "settings": { + "index": { + "default_pipeline": "bar-pipeline" + } + } + } + } + """; + + Map tempMap = XContentHelper.convertToMap( + new BytesArray(substituteComponentTemplatesString.getBytes(StandardCharsets.UTF_8)), + randomBoolean(), + XContentType.JSON + ).v2(); + Map> substituteComponentTemplates = (Map>) tempMap; + simulateBulkRequest = new SimulateBulkRequest(Map.of(), substituteComponentTemplates); + Map componentTemplateSubstitutions = simulateBulkRequest.getComponentTemplateSubstitutions(); + assertThat(componentTemplateSubstitutions.size(), equalTo(2)); + assertThat( + XContentHelper.convertToMap( + componentTemplateSubstitutions.get("mappings_template").template().mappings().uncompressed(), + randomBoolean(), + XContentType.JSON + ).v2(), + equalTo(substituteComponentTemplates.get("mappings_template").get("mappings")) + ); + assertNull(componentTemplateSubstitutions.get("mappings_template").template().settings()); + assertNull(componentTemplateSubstitutions.get("settings_template").template().mappings()); + assertThat(componentTemplateSubstitutions.get("settings_template").template().settings().size(), equalTo(1)); + assertThat( + componentTemplateSubstitutions.get("settings_template").template().settings().get("index.default_pipeline"), + equalTo("bar-pipeline") + ); + } + + public void testShallowClone() throws IOException { + SimulateBulkRequest simulateBulkRequest = new SimulateBulkRequest(getTestPipelineSubstitutions(), getTestTemplateSubstitutions()); + BulkRequest shallowCopy = simulateBulkRequest.shallowClone(); + assertThat(shallowCopy, instanceOf(SimulateBulkRequest.class)); + SimulateBulkRequest simulateBulkRequestCopy = (SimulateBulkRequest) shallowCopy; + assertThat(simulateBulkRequestCopy.requests, equalTo(List.of())); + assertThat( + simulateBulkRequestCopy.getComponentTemplateSubstitutions(), + equalTo(simulateBulkRequest.getComponentTemplateSubstitutions()) + ); + assertThat(simulateBulkRequestCopy.getPipelineSubstitutions(), equalTo(simulateBulkRequest.getPipelineSubstitutions())); + assertThat(simulateBulkRequestCopy.getRefreshPolicy(), equalTo(simulateBulkRequest.getRefreshPolicy())); + assertThat(simulateBulkRequestCopy.waitForActiveShards(), equalTo(simulateBulkRequest.waitForActiveShards())); + assertThat(simulateBulkRequestCopy.timeout(), equalTo(simulateBulkRequest.timeout())); + } + private static Map> getTestPipelineSubstitutions() { return Map.of( "pipeline1", From b7b2429ff03570b8e274dbeed948ef0898b06858 Mon Sep 17 00:00:00 2001 From: Keith Massey Date: Fri, 13 Sep 2024 13:02:14 -0500 Subject: [PATCH 16/28] imporving the changelog --- docs/changelog/112762.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog/112762.yaml b/docs/changelog/112762.yaml index 335530fecba0d..33366289cb777 100644 --- a/docs/changelog/112762.yaml +++ b/docs/changelog/112762.yaml @@ -1,5 +1,5 @@ pr: 112762 -summary: Simulate ingest API `template_substitutions` prototype +summary: Adding the ability to include component template substitutions in simulate ingest API requests area: Ingest Node type: enhancement issues: [] From 15f7445a61388b2ec9dc01b7d5b2aa4ca8fb5506 Mon Sep 17 00:00:00 2001 From: Keith Massey Date: Fri, 13 Sep 2024 13:07:59 -0500 Subject: [PATCH 17/28] cleanup --- .../org/elasticsearch/ingest/SimulateIngestService.java | 5 ----- .../action/bulk/TransportSimulateBulkActionTests.java | 4 ++-- .../elasticsearch/ingest/SimulateIngestServiceTests.java | 6 +----- 3 files changed, 3 insertions(+), 12 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/ingest/SimulateIngestService.java b/server/src/main/java/org/elasticsearch/ingest/SimulateIngestService.java index e069e7fe650ad..2f9da248b2afb 100644 --- a/server/src/main/java/org/elasticsearch/ingest/SimulateIngestService.java +++ b/server/src/main/java/org/elasticsearch/ingest/SimulateIngestService.java @@ -10,7 +10,6 @@ import org.elasticsearch.action.bulk.BulkRequest; import org.elasticsearch.action.bulk.SimulateBulkRequest; -import org.elasticsearch.cluster.metadata.IndexTemplateMetadata; import java.util.HashMap; import java.util.Map; @@ -64,10 +63,6 @@ private Map getPipelineSubstitutions( return parsedPipelineSubstitutions; } - private Map getTemplateSubstitutions() { - return Map.of(); - } - /** * This method returns the Pipeline for the given pipelineId. If a substitute definition of the pipeline has been defined for the * current simulate, then that pipeline is returned. Otherwise, the pipeline stored in the cluster state is returned. diff --git a/server/src/test/java/org/elasticsearch/action/bulk/TransportSimulateBulkActionTests.java b/server/src/test/java/org/elasticsearch/action/bulk/TransportSimulateBulkActionTests.java index b0da1d918ac67..f1bb9baec59ff 100644 --- a/server/src/test/java/org/elasticsearch/action/bulk/TransportSimulateBulkActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/bulk/TransportSimulateBulkActionTests.java @@ -134,7 +134,7 @@ public void tearDown() throws Exception { public void testIndexData() throws IOException { Task task = mock(Task.class); // unused - BulkRequest bulkRequest = new SimulateBulkRequest((Map>) null, (Map>) null); + BulkRequest bulkRequest = new SimulateBulkRequest(null, null); int bulkItemCount = randomIntBetween(0, 200); for (int i = 0; i < bulkItemCount; i++) { Map source = Map.of(randomAlphaOfLength(10), randomAlphaOfLength(5)); @@ -217,7 +217,7 @@ public void testIndexDataWithValidation() throws IOException { * (7) An indexing request to a nonexistent index that matches no templates */ Task task = mock(Task.class); // unused - BulkRequest bulkRequest = new SimulateBulkRequest((Map>) null, (Map>) null); + BulkRequest bulkRequest = new SimulateBulkRequest(null, null); int bulkItemCount = randomIntBetween(0, 200); Map indicesMap = new HashMap<>(); Map v1Templates = new HashMap<>(); diff --git a/server/src/test/java/org/elasticsearch/ingest/SimulateIngestServiceTests.java b/server/src/test/java/org/elasticsearch/ingest/SimulateIngestServiceTests.java index 08ab5d83d27c7..84c45bc54cec7 100644 --- a/server/src/test/java/org/elasticsearch/ingest/SimulateIngestServiceTests.java +++ b/server/src/test/java/org/elasticsearch/ingest/SimulateIngestServiceTests.java @@ -64,10 +64,7 @@ public void testGetPipeline() { ingestService.innerUpdatePipelines(ingestMetadata); { // First we make sure that if there are no substitutions that we get our original pipeline back: - SimulateBulkRequest simulateBulkRequest = new SimulateBulkRequest( - (Map>) null, - (Map>) null - ); + SimulateBulkRequest simulateBulkRequest = new SimulateBulkRequest(null, null); SimulateIngestService simulateIngestService = new SimulateIngestService(ingestService, simulateBulkRequest); Pipeline pipeline = simulateIngestService.getPipeline("pipeline1"); assertThat(pipeline.getProcessors(), contains(transformedMatch(Processor::getType, equalTo("processor1")))); @@ -85,7 +82,6 @@ public void testGetPipeline() { ); pipelineSubstitutions.put("pipeline2", newHashMap("processors", List.of(newHashMap("processor3", Collections.emptyMap())))); - // TODO SimulateBulkRequest simulateBulkRequest = new SimulateBulkRequest(pipelineSubstitutions, null); SimulateIngestService simulateIngestService = new SimulateIngestService(ingestService, simulateBulkRequest); Pipeline pipeline1 = simulateIngestService.getPipeline("pipeline1"); From 3e74db4b51f32a1721e819ef830096cfa132a519 Mon Sep 17 00:00:00 2001 From: Keith Massey Date: Fri, 13 Sep 2024 13:08:37 -0500 Subject: [PATCH 18/28] reverting accidental commit --- .../elasticsearch/nativeaccess/lib/NativeLibraryProvider.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/native/src/main/java/org/elasticsearch/nativeaccess/lib/NativeLibraryProvider.java b/libs/native/src/main/java/org/elasticsearch/nativeaccess/lib/NativeLibraryProvider.java index 43c45c82e7d85..737d5aecd0d0a 100644 --- a/libs/native/src/main/java/org/elasticsearch/nativeaccess/lib/NativeLibraryProvider.java +++ b/libs/native/src/main/java/org/elasticsearch/nativeaccess/lib/NativeLibraryProvider.java @@ -64,7 +64,7 @@ public T getLibrary(Class cls) { private static NativeLibraryProvider loadProvider() { final int runtimeVersion = Runtime.version().feature(); if (runtimeVersion >= 21) { - // return loadJdkImpl(runtimeVersion); + return loadJdkImpl(runtimeVersion); } return loadJnaImpl(); } From 0812fcaf4c1c5c66f49424642e76263f4e9a8eee Mon Sep 17 00:00:00 2001 From: Keith Massey Date: Fri, 13 Sep 2024 14:19:03 -0500 Subject: [PATCH 19/28] adding documentation --- .../ingest/apis/simulate-ingest.asciidoc | 118 +++++++++++++++++- .../action/bulk/SimulateBulkRequest.java | 4 +- .../action/bulk/SimulateBulkRequestTests.java | 2 +- 3 files changed, 120 insertions(+), 4 deletions(-) diff --git a/docs/reference/ingest/apis/simulate-ingest.asciidoc b/docs/reference/ingest/apis/simulate-ingest.asciidoc index ee84a39ee6f65..456b54c27ad42 100644 --- a/docs/reference/ingest/apis/simulate-ingest.asciidoc +++ b/docs/reference/ingest/apis/simulate-ingest.asciidoc @@ -83,11 +83,30 @@ POST /_ingest/_simulate } ] } + }, + "component_template_substitutions": { <2> + "my-component-template": { + "mappings": { + "dynamic": "true", + "properties": { + "field3": { + "type": "keyword" + } + } + }, + "settings": { + "index": { + "default_pipeline": "my-pipeline" + } + } + } } } ---- <1> This replaces the existing `my-pipeline` pipeline with the contents given here for the duration of this request. +<2> This replaces the existing `my-component-template` component template with the contents given here for the duration of this request. +These templates can be used to change the pipeline(s) used, or to modify the mapping that will be used to validate the result. [[simulate-ingest-api-request]] ==== {api-request-title} @@ -191,6 +210,18 @@ Map of pipeline IDs to substitute pipeline definition objects. include::put-pipeline.asciidoc[tag=pipeline-object] ==== +`component_template_substitutions`:: +(Optional, map of strings to objects) +Map of component template names to substitute component template definition objects. ++ +.Properties of component template definition objects +[%collapsible%open] +==== +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=mappings] + +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=settings] +==== + [[simulate-ingest-api-example]] ==== {api-examples-title} @@ -268,7 +299,7 @@ The API returns the following response: [[simulate-ingest-api-request-body-ex]] ===== Specify a pipeline substitution in the request body -In this example the index `index` has a default pipeline called `my-pipeline` and a final +In this example the index `my-index` has a default pipeline called `my-pipeline` and a final pipeline called `my-final-pipeline`. But a substitute definition of `my-pipeline` is provided in `pipeline_substitutions`. The substitute `my-pipeline` will be used in place of the `my-pipeline` that is in the system, and then the `my-final-pipeline` that is already @@ -348,6 +379,90 @@ The API returns the following response: } ---- +[[simulate-ingest-api-substitute-component-templates-ex]] +===== Specify a component template substitution in the request body +In this example, imagine that the index `my-index` has a strict mapping with only the `foo` +keyword field defined. Say that field mapping came from a component template named +`my-mappings-template`. We want to test adding a new field, `bar`. So a substitute definition of +`my-mappings-template` is provided in `component_template_substitutions`. The substitute +`my-mappings-template` will be used in place of the existing mapping for `my-index` and in place +of the `my-mappings-template` that is in the system. + +[source,console] +---- +POST /_ingest/_simulate +{ + "docs": [ + { + "_index": "my-index", + "_id": "123", + "_source": { + "foo": "foo" + } + }, + { + "_index": "my-index", + "_id": "456", + "_source": { + "bar": "rab" + } + } + ], + "component_template_substitutions": { + "my-mappings_template": { + "mappings": { + "dynamic": "strict", + "properties": { + "foo": { + "type": "keyword" + }, + "bar": { + "type": "keyword" + } + } + } + } + } +} +---- + +The API returns the following response: + +[source,console-result] +---- +{ + "docs": [ + { + "doc": { + "_id": "123", + "_index": "my-index", + "_version": -3, + "_source": { + "field2": "value2", + "foo": "BAR" + }, + "executed_pipelines": [ + "my-pipeline", + "my-final-pipeline" + ] + } + }, + { + "doc": { + "_id": "456", + "_index": "my-index", + "_version": -3, + "_source": { + "foo": "foo", + "bar": "rab" + }, + "executed_pipelines": [] + } + } + ] +} +---- + //// [source,console] ---- @@ -363,3 +478,4 @@ DELETE /_ingest/pipeline/* } ---- //// + diff --git a/server/src/main/java/org/elasticsearch/action/bulk/SimulateBulkRequest.java b/server/src/main/java/org/elasticsearch/action/bulk/SimulateBulkRequest.java index 00be5cc078956..63b3759056857 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/SimulateBulkRequest.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/SimulateBulkRequest.java @@ -24,8 +24,8 @@ import java.util.Map; /** - * This extends BulkRequest with support for providing substitute pipeline definitions and template definitions. In a user request, the - * substitutions will look something like this: + * This extends BulkRequest with support for providing substitute pipeline definitions and component template definitions. In a user + * request, the substitutions will look something like this: * * "pipeline_substitutions": { * "my-pipeline-1": { diff --git a/server/src/test/java/org/elasticsearch/action/bulk/SimulateBulkRequestTests.java b/server/src/test/java/org/elasticsearch/action/bulk/SimulateBulkRequestTests.java index 72a01adc20a3d..4f79bd8fe1bbb 100644 --- a/server/src/test/java/org/elasticsearch/action/bulk/SimulateBulkRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/bulk/SimulateBulkRequestTests.java @@ -45,7 +45,7 @@ private void testSerialization( assertThat(copy.getPipelineSubstitutions(), equalTo(simulateBulkRequest.getPipelineSubstitutions())); } - @SuppressWarnings("unchecked") + @SuppressWarnings("unchecked,rawtypes") public void testGetComponentTemplateSubstitutions() throws IOException { SimulateBulkRequest simulateBulkRequest = new SimulateBulkRequest(Map.of(), Map.of()); assertThat(simulateBulkRequest.getComponentTemplateSubstitutions(), equalTo(Map.of())); From 651bd946392130eb721338495a50501d6415361b Mon Sep 17 00:00:00 2001 From: Keith Massey Date: Fri, 13 Sep 2024 14:36:15 -0500 Subject: [PATCH 20/28] fixing SuppressWarnings --- .../org/elasticsearch/action/bulk/SimulateBulkRequestTests.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/test/java/org/elasticsearch/action/bulk/SimulateBulkRequestTests.java b/server/src/test/java/org/elasticsearch/action/bulk/SimulateBulkRequestTests.java index 4f79bd8fe1bbb..5a6e458ff3ee4 100644 --- a/server/src/test/java/org/elasticsearch/action/bulk/SimulateBulkRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/bulk/SimulateBulkRequestTests.java @@ -45,7 +45,7 @@ private void testSerialization( assertThat(copy.getPipelineSubstitutions(), equalTo(simulateBulkRequest.getPipelineSubstitutions())); } - @SuppressWarnings("unchecked,rawtypes") + @SuppressWarnings({ "unchecked", "rawtypes" }) public void testGetComponentTemplateSubstitutions() throws IOException { SimulateBulkRequest simulateBulkRequest = new SimulateBulkRequest(Map.of(), Map.of()); assertThat(simulateBulkRequest.getComponentTemplateSubstitutions(), equalTo(Map.of())); From 3bea3e28240436073aa992b1beef0cc66bc6062c Mon Sep 17 00:00:00 2001 From: Keith Massey Date: Fri, 13 Sep 2024 15:45:05 -0500 Subject: [PATCH 21/28] fixing docs --- docs/reference/ingest/apis/simulate-ingest.asciidoc | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/docs/reference/ingest/apis/simulate-ingest.asciidoc b/docs/reference/ingest/apis/simulate-ingest.asciidoc index 456b54c27ad42..85cdd70111559 100644 --- a/docs/reference/ingest/apis/simulate-ingest.asciidoc +++ b/docs/reference/ingest/apis/simulate-ingest.asciidoc @@ -438,13 +438,9 @@ The API returns the following response: "_index": "my-index", "_version": -3, "_source": { - "field2": "value2", - "foo": "BAR" + "foo": "foo" }, - "executed_pipelines": [ - "my-pipeline", - "my-final-pipeline" - ] + "executed_pipelines": [] } }, { @@ -453,7 +449,6 @@ The API returns the following response: "_index": "my-index", "_version": -3, "_source": { - "foo": "foo", "bar": "rab" }, "executed_pipelines": [] From 92d8568304bac899a1c04d1e63ff734be47b45e3 Mon Sep 17 00:00:00 2001 From: Keith Massey Date: Tue, 17 Sep 2024 14:12:00 -0500 Subject: [PATCH 22/28] handling new component_template_substitutions format --- .../ingest/apis/simulate-ingest.asciidoc | 40 +++++++------ .../test/ingest/80_ingest_simulate.yml | 56 +++++++++++-------- 2 files changed, 56 insertions(+), 40 deletions(-) diff --git a/docs/reference/ingest/apis/simulate-ingest.asciidoc b/docs/reference/ingest/apis/simulate-ingest.asciidoc index 85cdd70111559..365870701db4b 100644 --- a/docs/reference/ingest/apis/simulate-ingest.asciidoc +++ b/docs/reference/ingest/apis/simulate-ingest.asciidoc @@ -86,17 +86,19 @@ POST /_ingest/_simulate }, "component_template_substitutions": { <2> "my-component-template": { - "mappings": { - "dynamic": "true", - "properties": { - "field3": { - "type": "keyword" + "template": { + "mappings": { + "dynamic": "true", + "properties": { + "field3": { + "type": "keyword" + } + } + }, + "settings": { + "index": { + "default_pipeline": "my-pipeline" } - } - }, - "settings": { - "index": { - "default_pipeline": "my-pipeline" } } } @@ -410,14 +412,16 @@ POST /_ingest/_simulate ], "component_template_substitutions": { "my-mappings_template": { - "mappings": { - "dynamic": "strict", - "properties": { - "foo": { - "type": "keyword" - }, - "bar": { - "type": "keyword" + "template": { + "mappings": { + "dynamic": "strict", + "properties": { + "foo": { + "type": "keyword" + }, + "bar": { + "type": "keyword" + } } } } diff --git a/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/80_ingest_simulate.yml b/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/80_ingest_simulate.yml index 692231b07bd33..f3a977cd96f62 100644 --- a/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/80_ingest_simulate.yml +++ b/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/80_ingest_simulate.yml @@ -403,19 +403,23 @@ setup: ], "component_template_substitutions": { "mappings_template": { - "mappings": { - "dynamic": "true", - "properties": { - "foo": { - "type": "keyword" + "template": { + "mappings": { + "dynamic": "true", + "properties": { + "foo": { + "type": "keyword" + } } } } }, "settings_template": { - "settings": { - "index": { - "default_pipeline": "bar-pipeline" + "template": { + "settings": { + "index": { + "default_pipeline": "bar-pipeline" + } } } } @@ -452,19 +456,23 @@ setup: ], "component_template_substitutions": { "mappings_template": { - "mappings": { - "dynamic": "true", - "properties": { - "foo": { - "type": "keyword" + "template": { + "mappings": { + "dynamic": "true", + "properties": { + "foo": { + "type": "keyword" + } } } } }, "settings_template": { - "settings": { - "index": { - "default_pipeline": "bar-pipeline" + "template": { + "settings": { + "index": { + "default_pipeline": "bar-pipeline" + } } } } @@ -545,9 +553,11 @@ setup: ], "component_template_substitutions": { "settings_template": { - "settings": { - "index": { - "default_pipeline": null + "template": { + "settings": { + "index": { + "default_pipeline": null + } } } } @@ -581,9 +591,11 @@ setup: ], "component_template_substitutions": { "settings_template": { - "settings": { - "index": { - "default_pipeline": null + "template": { + "settings": { + "index": { + "default_pipeline": null + } } } } From 3a73cfe74ccc33752a7eaf1e97b342d1a36e74b8 Mon Sep 17 00:00:00 2001 From: Keith Massey Date: Tue, 17 Sep 2024 16:21:04 -0500 Subject: [PATCH 23/28] Making transport changes to enable component template substitutions in the simulate ingest API --- .../lib/NativeLibraryProvider.java | 2 +- .../bulk/TransportSimulateBulkActionIT.java | 126 +++++++++++++++++- .../TransportSimulateIndexTemplateAction.java | 8 +- .../post/TransportSimulateTemplateAction.java | 3 +- .../bulk/TransportAbstractBulkAction.java | 17 ++- .../action/bulk/TransportBulkAction.java | 2 + .../bulk/TransportSimulateBulkAction.java | 38 ++++-- .../cluster/metadata/Metadata.java | 2 +- .../metadata/MetadataCreateIndexService.java | 10 +- .../MetadataIndexTemplateService.java | 43 +++++- .../elasticsearch/ingest/IngestService.java | 46 +++++-- ...sportSimulateIndexTemplateActionTests.java | 3 +- .../TransportSimulateBulkActionTests.java | 2 +- .../MetadataIndexTemplateServiceTests.java | 22 ++- .../ingest/IngestServiceTests.java | 84 +++++++++++- 15 files changed, 356 insertions(+), 52 deletions(-) diff --git a/libs/native/src/main/java/org/elasticsearch/nativeaccess/lib/NativeLibraryProvider.java b/libs/native/src/main/java/org/elasticsearch/nativeaccess/lib/NativeLibraryProvider.java index 5fc53d4d242d6..c3000982e4a6e 100644 --- a/libs/native/src/main/java/org/elasticsearch/nativeaccess/lib/NativeLibraryProvider.java +++ b/libs/native/src/main/java/org/elasticsearch/nativeaccess/lib/NativeLibraryProvider.java @@ -65,7 +65,7 @@ public T getLibrary(Class cls) { private static NativeLibraryProvider loadProvider() { final int runtimeVersion = Runtime.version().feature(); if (runtimeVersion >= 21) { - return loadJdkImpl(runtimeVersion); + // return loadJdkImpl(runtimeVersion); } return loadJnaImpl(); } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/TransportSimulateBulkActionIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/TransportSimulateBulkActionIT.java index 0ad254937d0dc..91674b7ce9050 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/TransportSimulateBulkActionIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/TransportSimulateBulkActionIT.java @@ -15,12 +15,14 @@ import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; +import org.elasticsearch.action.admin.indices.template.put.PutComponentTemplateAction; import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest; import org.elasticsearch.action.admin.indices.template.put.TransportPutComposableIndexTemplateAction; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.ingest.SimulateIndexResponse; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.cluster.metadata.ComponentTemplate; import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; import org.elasticsearch.cluster.metadata.Template; import org.elasticsearch.common.compress.CompressedXContent; @@ -57,7 +59,7 @@ public void testMappingValidationIndexExists() { } """; indicesAdmin().create(new CreateIndexRequest(indexName).mapping(mapping)).actionGet(); - BulkRequest bulkRequest = new BulkRequest(); + BulkRequest bulkRequest = new SimulateBulkRequest(Map.of(), Map.of()); bulkRequest.add(new IndexRequest(indexName).source(""" { "foo1": "baz" @@ -87,13 +89,125 @@ public void testMappingValidationIndexExists() { assertThat(fields.size(), equalTo(1)); } + @SuppressWarnings("unchecked") + public void testMappingValidationIndexExistsWithComponentTemplate() throws IOException { + /* + * This test simulates a BulkRequest of two documents into an existing index. Then we make sure the index contains no documents, and + * that the index's mapping in the cluster state has not been updated with the two new field. With the mapping from the template + * that was used to create the index, we would expect the second document to throw an exception because it uses a field that does + * not exist. But we substitute a new version of the component template named "test-component-template" that allows for the new + * field. + */ + String originalComponentTemplateMappingString = """ + { + "_doc":{ + "dynamic":"strict", + "properties":{ + "foo1":{ + "type":"text" + } + } + } + } + """; + CompressedXContent mapping = CompressedXContent.fromJSON(originalComponentTemplateMappingString); + Template template = new Template(Settings.EMPTY, mapping, null); + PutComponentTemplateAction.Request componentTemplateActionRequest = new PutComponentTemplateAction.Request( + "test-component-template" + ); + ComponentTemplate componentTemplate = new ComponentTemplate(template, null, null); + componentTemplateActionRequest.componentTemplate(componentTemplate); + client().execute(PutComponentTemplateAction.INSTANCE, componentTemplateActionRequest).actionGet(); + ComposableIndexTemplate composableIndexTemplate = ComposableIndexTemplate.builder() + .indexPatterns(List.of("my-index-*")) + .componentTemplates(List.of("test-component-template")) + .build(); + TransportPutComposableIndexTemplateAction.Request request = new TransportPutComposableIndexTemplateAction.Request("test"); + request.indexTemplate(composableIndexTemplate); + client().execute(TransportPutComposableIndexTemplateAction.TYPE, request).actionGet(); + + String indexName = "my-index-1"; + // First, run before the index is created: + assertMappingsUpdatedFromComponentTemplateSubstitutions(indexName); + // Now, create the index and make sure the component template substitutions work the same: + indicesAdmin().create(new CreateIndexRequest(indexName)).actionGet(); + assertMappingsUpdatedFromComponentTemplateSubstitutions(indexName); + // Now make sure nothing was actually changed: + indicesAdmin().refresh(new RefreshRequest(indexName)).actionGet(); + SearchResponse searchResponse = client().search(new SearchRequest(indexName)).actionGet(); + assertThat(searchResponse.getHits().getTotalHits().value, equalTo(0L)); + searchResponse.decRef(); + ClusterStateResponse clusterStateResponse = admin().cluster().state(new ClusterStateRequest(TEST_REQUEST_TIMEOUT)).actionGet(); + Map indexMapping = clusterStateResponse.getState().metadata().index(indexName).mapping().sourceAsMap(); + Map fields = (Map) indexMapping.get("properties"); + assertThat(fields.size(), equalTo(1)); + } + + private void assertMappingsUpdatedFromComponentTemplateSubstitutions(String indexName) { + IndexRequest indexRequest1 = new IndexRequest(indexName).source(""" + { + "foo1": "baz" + } + """, XContentType.JSON).id(randomUUID()); + IndexRequest indexRequest2 = new IndexRequest(indexName).source(""" + { + "foo3": "baz" + } + """, XContentType.JSON).id(randomUUID()); + { + // First we use the original component template, and expect a failure in the second document: + BulkRequest bulkRequest = new SimulateBulkRequest(Map.of(), Map.of()); + bulkRequest.add(indexRequest1); + bulkRequest.add(indexRequest2); + BulkResponse response = client().execute(new ActionType(SimulateBulkAction.NAME), bulkRequest).actionGet(); + assertThat(response.getItems().length, equalTo(2)); + assertThat(response.getItems()[0].getResponse().getResult(), equalTo(DocWriteResponse.Result.CREATED)); + assertNull(((SimulateIndexResponse) response.getItems()[0].getResponse()).getException()); + assertThat(response.getItems()[1].getResponse().getResult(), equalTo(DocWriteResponse.Result.CREATED)); + assertThat( + ((SimulateIndexResponse) response.getItems()[1].getResponse()).getException().getMessage(), + containsString("mapping set to strict, dynamic introduction of") + ); + } + + { + // Now we substitute a "test-component-template" that defines both fields, so we expect no exception: + BulkRequest bulkRequest = new SimulateBulkRequest( + Map.of(), + Map.of( + "test-component-template", + Map.of( + "template", + Map.of( + "mappings", + Map.of( + "dynamic", + "strict", + "properties", + Map.of("foo1", Map.of("type", "text"), "foo3", Map.of("type", "text")) + ) + ) + ) + ) + ); + bulkRequest.add(indexRequest1); + bulkRequest.add(indexRequest2); + BulkResponse response = client().execute(new ActionType(SimulateBulkAction.NAME), bulkRequest).actionGet(); + assertThat(response.getItems().length, equalTo(2)); + assertThat(response.getItems()[0].getResponse().getResult(), equalTo(DocWriteResponse.Result.CREATED)); + assertNull(((SimulateIndexResponse) response.getItems()[0].getResponse()).getException()); + assertThat(response.getItems()[1].getResponse().getResult(), equalTo(DocWriteResponse.Result.CREATED)); + assertNull(((SimulateIndexResponse) response.getItems()[1].getResponse()).getException()); + } + } + public void testMappingValidationIndexDoesNotExistsNoTemplate() { /* * This test simulates a BulkRequest of two documents into an index that does not exist. There is no template (other than the * mapping-less "random-index-template" created by the parent class), so we expect no mapping validation failure. */ String indexName = randomAlphaOfLength(20).toLowerCase(Locale.ROOT); - BulkRequest bulkRequest = new BulkRequest(); + BulkRequest bulkRequest = new SimulateBulkRequest(Map.of(), Map.of()); bulkRequest.add(new IndexRequest(indexName).source(""" { "foo1": "baz" @@ -140,7 +254,7 @@ public void testMappingValidationIndexDoesNotExistsV2Template() throws IOExcepti request.indexTemplate(composableIndexTemplate); client().execute(TransportPutComposableIndexTemplateAction.TYPE, request).actionGet(); - BulkRequest bulkRequest = new BulkRequest(); + BulkRequest bulkRequest = new SimulateBulkRequest(Map.of(), Map.of()); bulkRequest.add(new IndexRequest(indexName).source(""" { "foo1": "baz" @@ -172,7 +286,7 @@ public void testMappingValidationIndexDoesNotExistsV1Template() { indicesAdmin().putTemplate( new PutIndexTemplateRequest("test-template").patterns(List.of("my-index-*")).mapping("foo1", "type=integer") ).actionGet(); - BulkRequest bulkRequest = new BulkRequest(); + BulkRequest bulkRequest = new SimulateBulkRequest(Map.of(), Map.of()); bulkRequest.add(new IndexRequest(indexName).source(""" { "foo1": "baz" @@ -226,7 +340,7 @@ public void testMappingValidationIndexDoesNotExistsDataStream() throws IOExcepti client().execute(TransportPutComposableIndexTemplateAction.TYPE, request).actionGet(); { // First, try with no @timestamp to make sure we're picking up data-stream-specific templates - BulkRequest bulkRequest = new BulkRequest(); + BulkRequest bulkRequest = new SimulateBulkRequest(Map.of(), Map.of()); bulkRequest.add(new IndexRequest(indexName).source(""" { "foo1": "baz" @@ -252,7 +366,7 @@ public void testMappingValidationIndexDoesNotExistsDataStream() throws IOExcepti } { // Now with @timestamp - BulkRequest bulkRequest = new BulkRequest(); + BulkRequest bulkRequest = new SimulateBulkRequest(Map.of(), Map.of()); bulkRequest.add(new IndexRequest(indexName).source(""" { "@timestamp": "2024-08-27", diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java index fdced5fc18ac9..3561a4d0e2cb4 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java @@ -16,6 +16,7 @@ import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.metadata.AliasMetadata; +import org.elasticsearch.cluster.metadata.ComponentTemplate; import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.metadata.DataStreamLifecycle; @@ -156,7 +157,8 @@ protected void masterOperation( xContentRegistry, indicesService, systemIndices, - indexSettingProviders + indexSettingProviders, + Map.of() ); final Map> overlapping = new HashMap<>(); @@ -233,7 +235,8 @@ public static Template resolveTemplate( final NamedXContentRegistry xContentRegistry, final IndicesService indicesService, final SystemIndices systemIndices, - Set indexSettingProviders + Set indexSettingProviders, + Map componentTemplateSubstitutions ) throws Exception { var metadata = simulatedState.getMetadata(); Settings templateSettings = resolveSettings(simulatedState.metadata(), matchingTemplate); @@ -263,6 +266,7 @@ public static Template resolveTemplate( null, // empty request mapping as the user can't specify any explicit mappings via the simulate api simulatedState, matchingTemplate, + componentTemplateSubstitutions, xContentRegistry, simulatedIndexName ); diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateTemplateAction.java index 30bbad0b57df0..af7a253b5a042 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateTemplateAction.java @@ -170,7 +170,8 @@ protected void masterOperation( xContentRegistry, indicesService, systemIndices, - indexSettingProviders + indexSettingProviders, + Map.of() ); if (request.includeDefaults()) { listener.onResponse( diff --git a/server/src/main/java/org/elasticsearch/action/bulk/TransportAbstractBulkAction.java b/server/src/main/java/org/elasticsearch/action/bulk/TransportAbstractBulkAction.java index d306299645d64..6119e73118202 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/TransportAbstractBulkAction.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/TransportAbstractBulkAction.java @@ -24,6 +24,7 @@ import org.elasticsearch.cluster.ClusterStateObserver; import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.block.ClusterBlockLevel; +import org.elasticsearch.cluster.metadata.ComponentTemplate; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.io.stream.Writeable; @@ -39,6 +40,8 @@ import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; +import java.io.IOException; +import java.util.Map; import java.util.Objects; import java.util.concurrent.Executor; import java.util.concurrent.TimeUnit; @@ -163,19 +166,21 @@ public void onTimeout(TimeValue timeout) { private void forkAndExecute(Task task, BulkRequest bulkRequest, Executor executor, ActionListener releasingListener) { executor.execute(new ActionRunnable<>(releasingListener) { @Override - protected void doRun() { + protected void doRun() throws IOException { applyPipelinesAndDoInternalExecute(task, bulkRequest, executor, releasingListener); } }); } - private boolean applyPipelines(Task task, BulkRequest bulkRequest, Executor executor, ActionListener listener) { + private boolean applyPipelines(Task task, BulkRequest bulkRequest, Executor executor, ActionListener listener) + throws IOException { boolean hasIndexRequestsWithPipelines = false; final Metadata metadata = clusterService.state().getMetadata(); + Map templateSubstitutions = bulkRequest.getComponentTemplateSubstitutions(); for (DocWriteRequest actionRequest : bulkRequest.requests) { IndexRequest indexRequest = getIndexWriteRequest(actionRequest); if (indexRequest != null) { - IngestService.resolvePipelinesAndUpdateIndexRequest(actionRequest, indexRequest, metadata); + IngestService.resolvePipelinesAndUpdateIndexRequest(actionRequest, indexRequest, metadata, templateSubstitutions); hasIndexRequestsWithPipelines |= IngestService.hasPipeline(indexRequest); } @@ -245,7 +250,7 @@ private void processBulkIndexIngestRequest( } else { ActionRunnable runnable = new ActionRunnable<>(actionListener) { @Override - protected void doRun() { + protected void doRun() throws IOException { applyPipelinesAndDoInternalExecute(task, bulkRequest, executor, actionListener); } @@ -323,7 +328,7 @@ private void applyPipelinesAndDoInternalExecute( BulkRequest bulkRequest, Executor executor, ActionListener listener - ) { + ) throws IOException { final long relativeStartTimeNanos = relativeTimeNanos(); if (applyPipelines(task, bulkRequest, executor, listener) == false) { doInternalExecute(task, bulkRequest, executor, listener, relativeStartTimeNanos); @@ -344,6 +349,6 @@ protected abstract void doInternalExecute( Executor executor, ActionListener listener, long relativeStartTimeNanos - ); + ) throws IOException; } diff --git a/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java b/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java index d35688b410822..37ade67bf7d6d 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java @@ -205,6 +205,8 @@ protected void doInternalExecute( ActionListener listener, long relativeStartTimeNanos ) { + assert (bulkRequest instanceof SimulateBulkRequest) == false + : "TransportBulkAction should never be called with a SimulateBulkRequest"; trackIndexRequests(bulkRequest); Map indicesToAutoCreate = new HashMap<>(); diff --git a/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java b/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java index ada47f9de098c..51858b236fa52 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java @@ -16,6 +16,7 @@ import org.elasticsearch.action.ingest.SimulateIndexResponse; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.ComponentTemplate; import org.elasticsearch.cluster.metadata.IndexAbstraction; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.IndexTemplateMetadata; @@ -49,6 +50,7 @@ import org.elasticsearch.transport.TransportService; import org.elasticsearch.xcontent.NamedXContentRegistry; +import java.io.IOException; import java.util.List; import java.util.Map; import java.util.Set; @@ -108,13 +110,16 @@ protected void doInternalExecute( Executor executor, ActionListener listener, long relativeStartTimeNanos - ) { + ) throws IOException { + assert bulkRequest instanceof SimulateBulkRequest + : "TransportSimulateBulkAction should only ever be called with a SimulateBulkRequest but got a " + bulkRequest.getClass(); final AtomicArray responses = new AtomicArray<>(bulkRequest.requests.size()); + Map componentTemplateSubstitutions = bulkRequest.getComponentTemplateSubstitutions(); for (int i = 0; i < bulkRequest.requests.size(); i++) { DocWriteRequest docRequest = bulkRequest.requests.get(i); assert docRequest instanceof IndexRequest : "TransportSimulateBulkAction should only ever be called with IndexRequests"; IndexRequest request = (IndexRequest) docRequest; - Exception mappingValidationException = validateMappings(request); + Exception mappingValidationException = validateMappings(componentTemplateSubstitutions, request); responses.set( i, BulkItemResponse.success( @@ -140,10 +145,11 @@ protected void doInternalExecute( /** * This creates a temporary index with the mappings of the index in the request, and then attempts to index the source from the request * into it. If there is a mapping exception, that exception is returned. On success the returned exception is null. + * @parem componentTemplateSubstitutions The component template definitions to use in place of existing ones for validation * @param request The IndexRequest whose source will be validated against the mapping (if it exists) of its index * @return a mapping exception if the source does not match the mappings, otherwise null */ - private Exception validateMappings(IndexRequest request) { + private Exception validateMappings(Map componentTemplateSubstitutions, IndexRequest request) { final SourceToParse sourceToParse = new SourceToParse( request.id(), request.source(), @@ -157,7 +163,11 @@ private Exception validateMappings(IndexRequest request) { Exception mappingValidationException = null; IndexAbstraction indexAbstraction = state.metadata().getIndicesLookup().get(request.index()); try { - if (indexAbstraction != null) { + if (indexAbstraction != null && componentTemplateSubstitutions.isEmpty()) { + /* + * In this case the index exists and we don't have any component template overrides. So we can just use withTempIndexService + * to do the mapping validation, using all the existing logic for validation. + */ IndexMetadata imd = state.metadata().getIndexSafe(indexAbstraction.getWriteIndex(request, state.metadata())); indicesService.withTempIndexService(imd, indexService -> { indexService.mapperService().updateMapping(null, imd); @@ -178,23 +188,29 @@ private Exception validateMappings(IndexRequest request) { }); } else { /* - * The index did not exist, so we put together the mappings from existing templates. - * This reproduces a lot of the mapping resolution logic in MetadataCreateIndexService.applyCreateIndexRequest(). However, - * it does not deal with aliases (since an alias cannot be created if an index does not exist, and this is the path for - * when the index does not exist). And it does not deal with system indices since we do not intend for users to simulate - * writing to system indices. + * The index did not exist, or we have component template substitutions, so we put together the mappings from existing + * templates This reproduces a lot of the mapping resolution logic in MetadataCreateIndexService.applyCreateIndexRequest(). + * However, it does not deal with aliases (since an alias cannot be created if an index does not exist, and this is the + * path for when the index does not exist). And it does not deal with system indices since we do not intend for users to + * simulate writing to system indices. */ + // First, we remove the index from the cluster state if necessary (since we're going to use the templates) + ClusterState simulatedState = indexAbstraction == null + ? state + : new ClusterState.Builder(state).metadata(new Metadata.Builder(state.metadata()).remove(request.index()).build()) + .build(); String matchingTemplate = findV2Template(state.metadata(), request.index(), false); if (matchingTemplate != null) { final Template template = TransportSimulateIndexTemplateAction.resolveTemplate( matchingTemplate, request.index(), - state, + simulatedState, isDataStreamsLifecycleOnlyMode(clusterService.getSettings()), xContentRegistry, indicesService, systemIndices, - indexSettingProviders + indexSettingProviders, + componentTemplateSubstitutions ); CompressedXContent mappings = template.mappings(); if (mappings != null) { diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java index c1a2025271891..2014e18a8e016 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java @@ -1825,7 +1825,7 @@ public Builder() { this(Map.of(), 0); } - Builder(Metadata metadata) { + public Builder(Metadata metadata) { this.clusterUUID = metadata.clusterUUID; this.clusterUUIDCommitted = metadata.clusterUUIDCommitted; this.coordinationMetadata = metadata.coordinationMetadata; diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java index 07cfcddac9c93..275c186a1ea85 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java @@ -648,6 +648,7 @@ private ClusterState applyCreateIndexRequestWithV2Template( request.mappings(), currentState, templateName, + Map.of(), xContentRegistry, request.index() ); @@ -806,6 +807,7 @@ private static List collectSystemV2Mappings( List templateMappings = MetadataIndexTemplateService.collectMappings( composableIndexTemplate, componentTemplates, + Map.of(), indexName ); return collectV2Mappings(null, templateMappings, xContentRegistry); @@ -815,10 +817,16 @@ public static List collectV2Mappings( @Nullable final String requestMappings, final ClusterState currentState, final String templateName, + Map componentTemplateSubstitutions, final NamedXContentRegistry xContentRegistry, final String indexName ) throws Exception { - List templateMappings = MetadataIndexTemplateService.collectMappings(currentState, templateName, indexName); + List templateMappings = MetadataIndexTemplateService.collectMappings( + currentState, + templateName, + componentTemplateSubstitutions, + indexName + ); return collectV2Mappings(requestMappings, templateMappings, xContentRegistry); } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java index 207f773b54f34..9888059af9686 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java @@ -698,7 +698,7 @@ private void validateIndexTemplateV2(String name, ComposableIndexTemplate indexT final var now = Instant.now(); final var metadata = currentState.getMetadata(); - final var combinedMappings = collectMappings(indexTemplate, metadata.componentTemplates(), "tmp_idx"); + final var combinedMappings = collectMappings(indexTemplate, metadata.componentTemplates(), Map.of(), "tmp_idx"); final var combinedSettings = resolveSettings(indexTemplate, metadata.componentTemplates()); // First apply settings sourced from index setting providers: for (var provider : indexSettingProviders) { @@ -1348,7 +1348,12 @@ private static boolean isGlobalAndHasIndexHiddenSetting(Metadata metadata, Compo /** * Collect the given v2 template into an ordered list of mappings. */ - public static List collectMappings(final ClusterState state, final String templateName, final String indexName) { + public static List collectMappings( + final ClusterState state, + final String templateName, + Map componentTemplateSubstitutions, + final String indexName + ) { final ComposableIndexTemplate template = state.metadata().templatesV2().get(templateName); assert template != null : "attempted to resolve mappings for a template [" + templateName + "] that did not exist in the cluster state"; @@ -1357,7 +1362,7 @@ public static List collectMappings(final ClusterState state, } final Map componentTemplates = state.metadata().componentTemplates(); - return collectMappings(template, componentTemplates, indexName); + return collectMappings(template, componentTemplates, componentTemplateSubstitutions, indexName); } /** @@ -1366,6 +1371,7 @@ public static List collectMappings(final ClusterState state, public static List collectMappings( final ComposableIndexTemplate template, final Map componentTemplates, + final Map componentTemplateSubstitutions, final String indexName ) { Objects.requireNonNull(template, "Composable index template must be provided"); @@ -1376,9 +1382,12 @@ public static List collectMappings( ComposableIndexTemplate.DataStreamTemplate.DATA_STREAM_MAPPING_SNIPPET ); } + final Map combinedComponentTemplates = new HashMap<>(); + combinedComponentTemplates.putAll(componentTemplates); + combinedComponentTemplates.putAll(componentTemplateSubstitutions); List mappings = template.composedOf() .stream() - .map(componentTemplates::get) + .map(combinedComponentTemplates::get) .filter(Objects::nonNull) .map(ComponentTemplate::template) .map(Template::mappings) @@ -1428,24 +1437,44 @@ public static Settings resolveSettings(final List templat * Resolve the given v2 template into a collected {@link Settings} object */ public static Settings resolveSettings(final Metadata metadata, final String templateName) { + return resolveSettings(metadata, templateName, Map.of()); + } + + public static Settings resolveSettings( + final Metadata metadata, + final String templateName, + Map templateSubstitutions + ) { final ComposableIndexTemplate template = metadata.templatesV2().get(templateName); assert template != null : "attempted to resolve settings for a template [" + templateName + "] that did not exist in the cluster state"; if (template == null) { return Settings.EMPTY; } - return resolveSettings(template, metadata.componentTemplates()); + return resolveSettings(template, metadata.componentTemplates(), templateSubstitutions); } /** * Resolve the provided v2 template and component templates into a collected {@link Settings} object */ public static Settings resolveSettings(ComposableIndexTemplate template, Map componentTemplates) { + return resolveSettings(template, componentTemplates, Map.of()); + } + + public static Settings resolveSettings( + ComposableIndexTemplate template, + Map componentTemplates, + Map templateSubstitutions + ) { Objects.requireNonNull(template, "attempted to resolve settings for a null template"); Objects.requireNonNull(componentTemplates, "attempted to resolve settings with null component templates"); + Map combinedComponentTemplates = new HashMap<>(); + combinedComponentTemplates.putAll(componentTemplates); + // We want any substitutions to take precedence: + combinedComponentTemplates.putAll(templateSubstitutions); List componentSettings = template.composedOf() .stream() - .map(componentTemplates::get) + .map(combinedComponentTemplates::get) .filter(Objects::nonNull) .map(ComponentTemplate::template) .map(Template::settings) @@ -1694,7 +1723,7 @@ private static void validateCompositeTemplate( String indexName = DataStream.BACKING_INDEX_PREFIX + temporaryIndexName; // Parse mappings to ensure they are valid after being composed - List mappings = collectMappings(stateWithIndex, templateName, indexName); + List mappings = collectMappings(stateWithIndex, templateName, Map.of(), indexName); try { MapperService mapperService = tempIndexService.mapperService(); mapperService.merge(MapperService.SINGLE_MAPPING_NAME, mappings, MapperService.MergeReason.INDEX_TEMPLATE); diff --git a/server/src/main/java/org/elasticsearch/ingest/IngestService.java b/server/src/main/java/org/elasticsearch/ingest/IngestService.java index 40832cbdfc34f..4d857194d6634 100644 --- a/server/src/main/java/org/elasticsearch/ingest/IngestService.java +++ b/server/src/main/java/org/elasticsearch/ingest/IngestService.java @@ -33,6 +33,7 @@ import org.elasticsearch.cluster.ClusterStateApplier; import org.elasticsearch.cluster.ClusterStateTaskExecutor; import org.elasticsearch.cluster.ClusterStateTaskListener; +import org.elasticsearch.cluster.metadata.ComponentTemplate; import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.metadata.IndexAbstraction; import org.elasticsearch.cluster.metadata.IndexMetadata; @@ -270,26 +271,49 @@ public static void resolvePipelinesAndUpdateIndexRequest( final IndexRequest indexRequest, final Metadata metadata ) { - resolvePipelinesAndUpdateIndexRequest(originalRequest, indexRequest, metadata, System.currentTimeMillis()); + resolvePipelinesAndUpdateIndexRequest(originalRequest, indexRequest, metadata, Map.of()); + } + + public static void resolvePipelinesAndUpdateIndexRequest( + final DocWriteRequest originalRequest, + final IndexRequest indexRequest, + final Metadata metadata, + Map templateSubstitutions + ) { + resolvePipelinesAndUpdateIndexRequest(originalRequest, indexRequest, metadata, System.currentTimeMillis(), templateSubstitutions); } static void resolvePipelinesAndUpdateIndexRequest( final DocWriteRequest originalRequest, final IndexRequest indexRequest, final Metadata metadata, - final long epochMillis + final long epochMillis, + final Map componentTemplateSubstitutions ) { if (indexRequest.isPipelineResolved()) { return; } - String requestPipeline = indexRequest.getPipeline(); - - Pipelines pipelines = resolvePipelinesFromMetadata(originalRequest, indexRequest, metadata, epochMillis) // - .or(() -> resolvePipelinesFromIndexTemplates(indexRequest, metadata)) - .orElse(Pipelines.NO_PIPELINES_DEFINED); + /* + * Here we look for the pipelines associated with the index if the index exists. If the index does not exist we fall back to using + * templates to find the pipelines. But if a user has passed in component template substitutions, they want the settings from those + * used in place of the settings used to create any previous indices. So in that case we use the templates to find the pipelines -- + * we don't fall back to the existing index if we don't find any because it is possible the user has intentionally removed the + * pipeline. + */ + final Pipelines pipelines; + if (componentTemplateSubstitutions.isEmpty()) { + pipelines = resolvePipelinesFromMetadata(originalRequest, indexRequest, metadata, epochMillis) // + .or(() -> resolvePipelinesFromIndexTemplates(indexRequest, metadata, Map.of())) + .orElse(Pipelines.NO_PIPELINES_DEFINED); + } else { + pipelines = resolvePipelinesFromIndexTemplates(indexRequest, metadata, componentTemplateSubstitutions).orElse( + Pipelines.NO_PIPELINES_DEFINED + ); + } // The pipeline coming as part of the request always has priority over the resolved one from metadata or templates + String requestPipeline = indexRequest.getPipeline(); if (requestPipeline != null) { indexRequest.setPipeline(requestPipeline); } else { @@ -1436,7 +1460,11 @@ private static Optional resolvePipelinesFromMetadata( return Optional.of(new Pipelines(IndexSettings.DEFAULT_PIPELINE.get(settings), IndexSettings.FINAL_PIPELINE.get(settings))); } - private static Optional resolvePipelinesFromIndexTemplates(IndexRequest indexRequest, Metadata metadata) { + private static Optional resolvePipelinesFromIndexTemplates( + IndexRequest indexRequest, + Metadata metadata, + Map templateSubstitutions + ) { if (indexRequest.index() == null) { return Optional.empty(); } @@ -1446,7 +1474,7 @@ private static Optional resolvePipelinesFromIndexTemplates(IndexReque // precedence), or if a V2 template does not match, any V1 templates String v2Template = MetadataIndexTemplateService.findV2Template(metadata, indexRequest.index(), false); if (v2Template != null) { - final Settings settings = MetadataIndexTemplateService.resolveSettings(metadata, v2Template); + final Settings settings = MetadataIndexTemplateService.resolveSettings(metadata, v2Template, templateSubstitutions); return Optional.of(new Pipelines(IndexSettings.DEFAULT_PIPELINE.get(settings), IndexSettings.FINAL_PIPELINE.get(settings))); } diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateActionTests.java index 8f0ff82beab4b..9b1d8c15619ad 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateActionTests.java @@ -87,7 +87,8 @@ public Settings getAdditionalIndexSettings( xContentRegistry(), indicesService, systemIndices, - indexSettingsProviders + indexSettingsProviders, + Map.of() ); assertThat(resolvedTemplate.settings().getAsInt("test-setting", -1), is(1)); diff --git a/server/src/test/java/org/elasticsearch/action/bulk/TransportSimulateBulkActionTests.java b/server/src/test/java/org/elasticsearch/action/bulk/TransportSimulateBulkActionTests.java index e3c863ee69985..f4e53912d09a7 100644 --- a/server/src/test/java/org/elasticsearch/action/bulk/TransportSimulateBulkActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/bulk/TransportSimulateBulkActionTests.java @@ -133,7 +133,7 @@ public void tearDown() throws Exception { super.tearDown(); } - public void testIndexData() { + public void testIndexData() throws IOException { Task task = mock(Task.class); // unused BulkRequest bulkRequest = new SimulateBulkRequest(null, null); int bulkItemCount = randomIntBetween(0, 200); diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java index 38de29010a371..7a1d4b5b1ddf4 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java @@ -1081,7 +1081,7 @@ public void testResolveConflictingMappings() throws Exception { .build(); state = service.addIndexTemplateV2(state, true, "my-template", it); - List mappings = MetadataIndexTemplateService.collectMappings(state, "my-template", "my-index"); + List mappings = MetadataIndexTemplateService.collectMappings(state, "my-template", Map.of(), "my-index"); assertNotNull(mappings); assertThat(mappings.size(), equalTo(3)); @@ -1143,7 +1143,7 @@ public void testResolveMappings() throws Exception { .build(); state = service.addIndexTemplateV2(state, true, "my-template", it); - List mappings = MetadataIndexTemplateService.collectMappings(state, "my-template", "my-index"); + List mappings = MetadataIndexTemplateService.collectMappings(state, "my-template", Map.of(), "my-index"); assertNotNull(mappings); assertThat(mappings.size(), equalTo(3)); @@ -1197,6 +1197,7 @@ public void testDefinedTimestampMappingIsAddedForDataStreamTemplates() throws Ex List mappings = MetadataIndexTemplateService.collectMappings( state, "logs-data-stream-template", + Map.of(), DataStream.getDefaultBackingIndexName("logs", 1L) ); @@ -1248,7 +1249,12 @@ public void testDefinedTimestampMappingIsAddedForDataStreamTemplates() throws Ex .build(); state = service.addIndexTemplateV2(state, true, "timeseries-template", it); - List mappings = MetadataIndexTemplateService.collectMappings(state, "timeseries-template", "timeseries"); + List mappings = MetadataIndexTemplateService.collectMappings( + state, + "timeseries-template", + Map.of(), + "timeseries" + ); assertNotNull(mappings); assertThat(mappings.size(), equalTo(2)); @@ -1270,6 +1276,7 @@ public void testDefinedTimestampMappingIsAddedForDataStreamTemplates() throws Ex mappings = MetadataIndexTemplateService.collectMappings( state, "timeseries-template", + Map.of(), DataStream.getDefaultBackingIndexName("timeseries", 1L) ); @@ -1318,6 +1325,7 @@ public void testUserDefinedMappingTakesPrecedenceOverDefault() throws Exception List mappings = MetadataIndexTemplateService.collectMappings( state, "logs-template", + Map.of(), DataStream.getDefaultBackingIndexName("logs", 1L) ); @@ -1374,6 +1382,7 @@ public void testUserDefinedMappingTakesPrecedenceOverDefault() throws Exception List mappings = MetadataIndexTemplateService.collectMappings( state, "timeseries-template", + Map.of(), DataStream.getDefaultBackingIndexName("timeseries-template", 1L) ); @@ -2440,7 +2449,12 @@ public void testComposableTemplateWithSubobjectsFalse() throws Exception { .build(); state = service.addIndexTemplateV2(state, true, "composable-template", it); - List mappings = MetadataIndexTemplateService.collectMappings(state, "composable-template", "test-index"); + List mappings = MetadataIndexTemplateService.collectMappings( + state, + "composable-template", + Map.of(), + "test-index" + ); assertNotNull(mappings); assertThat(mappings.size(), equalTo(2)); diff --git a/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java b/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java index 3adaf398624de..49e75a71aa7f7 100644 --- a/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java +++ b/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java @@ -32,16 +32,20 @@ import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.AliasMetadata; +import org.elasticsearch.cluster.metadata.ComponentTemplate; +import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.IndexTemplateMetadata; import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.cluster.metadata.Template; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodeUtils; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.cluster.service.ClusterStateTaskExecutorUtils; import org.elasticsearch.common.TriConsumer; import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.time.DateFormatter; import org.elasticsearch.common.util.Maps; @@ -73,6 +77,7 @@ import org.mockito.ArgumentMatcher; import org.mockito.invocation.InvocationOnMock; +import java.io.IOException; import java.io.OutputStream; import java.nio.charset.StandardCharsets; import java.util.ArrayList; @@ -2488,7 +2493,7 @@ public void testResolveFinalPipelineWithDateMathExpression() { // index name matches with IDM: IndexRequest indexRequest = new IndexRequest(""); - IngestService.resolvePipelinesAndUpdateIndexRequest(indexRequest, indexRequest, metadata, epochMillis); + IngestService.resolvePipelinesAndUpdateIndexRequest(indexRequest, indexRequest, metadata, epochMillis, Map.of()); assertTrue(hasPipeline(indexRequest)); assertTrue(indexRequest.isPipelineResolved()); assertThat(indexRequest.getPipeline(), equalTo("_none")); @@ -2853,6 +2858,83 @@ public void testResolvePipelinesWithNonePipeline() { } } + public void testResolvePipelinesAndUpdateIndexRequestWithComponentTemplateSubstitutions() throws IOException { + final String componentTemplateName = "test-component-template"; + final String indexName = "my-index-1"; + final String indexPipeline = "index-pipeline"; + final String realTemplatePipeline = "template-pipeline"; + final String substitutePipeline = "substitute-pipeline"; + + Metadata metadata; + { + // Build up cluster state metadata + IndexMetadata.Builder builder = IndexMetadata.builder(indexName) + .settings(settings(IndexVersion.current())) + .numberOfShards(1) + .numberOfReplicas(0); + ComponentTemplate realComponentTemplate = new ComponentTemplate( + new Template( + Settings.builder().put("index.default_pipeline", realTemplatePipeline).build(), + CompressedXContent.fromJSON("{}"), + null + ), + null, + null + ); + ComposableIndexTemplate composableIndexTemplate = ComposableIndexTemplate.builder() + .indexPatterns(List.of("my-index-*")) + .componentTemplates(List.of(componentTemplateName)) + .build(); + metadata = Metadata.builder() + .put(builder) + .indexTemplates(Map.of("my-index-template", composableIndexTemplate)) + .componentTemplates(Map.of("test-component-template", realComponentTemplate)) + .build(); + } + + Map componentTemplateSubstitutions; + { + ComponentTemplate simulatedComponentTemplate = new ComponentTemplate( + new Template( + Settings.builder().put("index.default_pipeline", substitutePipeline).build(), + CompressedXContent.fromJSON("{}"), + null + ), + null, + null + ); + componentTemplateSubstitutions = Map.of(componentTemplateName, simulatedComponentTemplate); + } + + { + /* + * Here there is a pipeline in the request. This takes precedence over anything in the index or templates or component template + * substitutions. + */ + IndexRequest indexRequest = new IndexRequest(indexName).setPipeline(indexPipeline); + IngestService.resolvePipelinesAndUpdateIndexRequest(indexRequest, indexRequest, metadata, 0, componentTemplateSubstitutions); + assertThat(indexRequest.getPipeline(), equalTo(indexPipeline)); + } + { + /* + * Here there is no pipeline in the request, but there is one in the substitute component template. So it takes precedence. + */ + IndexRequest indexRequest = new IndexRequest(indexName); + IngestService.resolvePipelinesAndUpdateIndexRequest(indexRequest, indexRequest, metadata, 0, componentTemplateSubstitutions); + assertThat(indexRequest.getPipeline(), equalTo(substitutePipeline)); + } + { + /* + * This one is tricky. Since the index exists and there are no component template substitutions, we're going to use the actual + * index in this case rather than its template. The index does not have a default pipeline set, so it's "_none" instead of + * realTemplatePipeline. + */ + IndexRequest indexRequest = new IndexRequest(indexName); + IngestService.resolvePipelinesAndUpdateIndexRequest(indexRequest, indexRequest, metadata, 0, Map.of()); + assertThat(indexRequest.getPipeline(), equalTo("_none")); + } + } + private static Tuple randomMapEntry() { return tuple(randomAlphaOfLength(5), randomObject()); } From d738c64cad761d51cde5ace70b1a43d228e7a3f9 Mon Sep 17 00:00:00 2001 From: Keith Massey Date: Tue, 17 Sep 2024 16:28:10 -0500 Subject: [PATCH 24/28] removing native library hack --- .../elasticsearch/nativeaccess/lib/NativeLibraryProvider.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/native/src/main/java/org/elasticsearch/nativeaccess/lib/NativeLibraryProvider.java b/libs/native/src/main/java/org/elasticsearch/nativeaccess/lib/NativeLibraryProvider.java index c3000982e4a6e..5fc53d4d242d6 100644 --- a/libs/native/src/main/java/org/elasticsearch/nativeaccess/lib/NativeLibraryProvider.java +++ b/libs/native/src/main/java/org/elasticsearch/nativeaccess/lib/NativeLibraryProvider.java @@ -65,7 +65,7 @@ public T getLibrary(Class cls) { private static NativeLibraryProvider loadProvider() { final int runtimeVersion = Runtime.version().feature(); if (runtimeVersion >= 21) { - // return loadJdkImpl(runtimeVersion); + return loadJdkImpl(runtimeVersion); } return loadJnaImpl(); } From fc2ceba2d0e97a9eb8b5fe0d5b699242e4d0df1c Mon Sep 17 00:00:00 2001 From: Keith Massey Date: Thu, 19 Sep 2024 08:40:38 -0500 Subject: [PATCH 25/28] Adding an assertion about component template substitutions to TransportBulkAction --- .../org/elasticsearch/action/bulk/TransportBulkAction.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java b/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java index 37ade67bf7d6d..9a79644e7cffd 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java @@ -53,6 +53,7 @@ import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; +import java.io.IOException; import java.util.HashMap; import java.util.HashSet; import java.util.Map; @@ -204,9 +205,11 @@ protected void doInternalExecute( Executor executor, ActionListener listener, long relativeStartTimeNanos - ) { + ) throws IOException { assert (bulkRequest instanceof SimulateBulkRequest) == false : "TransportBulkAction should never be called with a SimulateBulkRequest"; + assert bulkRequest.getComponentTemplateSubstitutions().isEmpty() + : "Component template substitutions are not allowed in a non-simulated bulk"; trackIndexRequests(bulkRequest); Map indicesToAutoCreate = new HashMap<>(); From c36a44814720e424895e4097666b24ff46ddee08 Mon Sep 17 00:00:00 2001 From: Keith Massey Date: Thu, 19 Sep 2024 08:45:32 -0500 Subject: [PATCH 26/28] reverting change to Metadata --- .../action/bulk/TransportSimulateBulkAction.java | 4 ++-- .../java/org/elasticsearch/cluster/metadata/Metadata.java | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java b/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java index 51858b236fa52..0ea763c215959 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java @@ -197,8 +197,8 @@ private Exception validateMappings(Map componentTempl // First, we remove the index from the cluster state if necessary (since we're going to use the templates) ClusterState simulatedState = indexAbstraction == null ? state - : new ClusterState.Builder(state).metadata(new Metadata.Builder(state.metadata()).remove(request.index()).build()) - .build(); + : new ClusterState.Builder(state).metadata(Metadata.builder(state.metadata()).remove(request.index()).build()).build(); + String matchingTemplate = findV2Template(state.metadata(), request.index(), false); if (matchingTemplate != null) { final Template template = TransportSimulateIndexTemplateAction.resolveTemplate( diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java index 5ff1986f9531b..d2f5ab5eabaee 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java @@ -1818,7 +1818,7 @@ public Builder() { this(Map.of(), 0); } - public Builder(Metadata metadata) { + Builder(Metadata metadata) { this.clusterUUID = metadata.clusterUUID; this.clusterUUIDCommitted = metadata.clusterUUIDCommitted; this.coordinationMetadata = metadata.coordinationMetadata; From 88c6edfa296709f26ca9a5ebf75b97872e5d1d1a Mon Sep 17 00:00:00 2001 From: Keith Massey Date: Thu, 19 Sep 2024 08:47:18 -0500 Subject: [PATCH 27/28] renaming templateSubstitutions -> componentTemplateSubstitutions in IngestService --- .../org/elasticsearch/ingest/IngestService.java | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/ingest/IngestService.java b/server/src/main/java/org/elasticsearch/ingest/IngestService.java index 4d857194d6634..0275e988ce39d 100644 --- a/server/src/main/java/org/elasticsearch/ingest/IngestService.java +++ b/server/src/main/java/org/elasticsearch/ingest/IngestService.java @@ -278,9 +278,15 @@ public static void resolvePipelinesAndUpdateIndexRequest( final DocWriteRequest originalRequest, final IndexRequest indexRequest, final Metadata metadata, - Map templateSubstitutions + Map componentTemplateSubstitutions ) { - resolvePipelinesAndUpdateIndexRequest(originalRequest, indexRequest, metadata, System.currentTimeMillis(), templateSubstitutions); + resolvePipelinesAndUpdateIndexRequest( + originalRequest, + indexRequest, + metadata, + System.currentTimeMillis(), + componentTemplateSubstitutions + ); } static void resolvePipelinesAndUpdateIndexRequest( @@ -1463,7 +1469,7 @@ private static Optional resolvePipelinesFromMetadata( private static Optional resolvePipelinesFromIndexTemplates( IndexRequest indexRequest, Metadata metadata, - Map templateSubstitutions + Map componentTemplateSubstitutions ) { if (indexRequest.index() == null) { return Optional.empty(); @@ -1474,7 +1480,7 @@ private static Optional resolvePipelinesFromIndexTemplates( // precedence), or if a V2 template does not match, any V1 templates String v2Template = MetadataIndexTemplateService.findV2Template(metadata, indexRequest.index(), false); if (v2Template != null) { - final Settings settings = MetadataIndexTemplateService.resolveSettings(metadata, v2Template, templateSubstitutions); + final Settings settings = MetadataIndexTemplateService.resolveSettings(metadata, v2Template, componentTemplateSubstitutions); return Optional.of(new Pipelines(IndexSettings.DEFAULT_PIPELINE.get(settings), IndexSettings.FINAL_PIPELINE.get(settings))); } From b2eac9536f26ed16ffd84ca6c3782ddf697a3aa3 Mon Sep 17 00:00:00 2001 From: Keith Massey Date: Fri, 20 Sep 2024 09:40:13 -0500 Subject: [PATCH 28/28] fixing documentation --- docs/reference/indices/put-component-template.asciidoc | 7 +++++-- docs/reference/ingest/apis/simulate-ingest.asciidoc | 5 +++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/docs/reference/indices/put-component-template.asciidoc b/docs/reference/indices/put-component-template.asciidoc index 0a0e36b63e6cd..d880edfe42b8c 100644 --- a/docs/reference/indices/put-component-template.asciidoc +++ b/docs/reference/indices/put-component-template.asciidoc @@ -129,6 +129,8 @@ include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=master-timeout] [[put-component-template-api-request-body]] ==== {api-request-body-title} +// tag::template[] + `template`:: (Required, object) This is the template to be applied, may optionally include a `mappings`, @@ -136,7 +138,7 @@ This is the template to be applied, may optionally include a `mappings`, + .Properties of `template` [%collapsible%open] -==== +===== `aliases`:: (Optional, object of objects) Aliases to add. + @@ -147,7 +149,7 @@ include::{es-ref-dir}/indices/create-index.asciidoc[tag=aliases-props] include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=mappings] include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=settings] -==== +===== `version`:: (Optional, integer) @@ -174,6 +176,7 @@ This map is not automatically generated by {es}. Marks this component template as deprecated. When a deprecated component template is referenced when creating or updating a non-deprecated index template, {es} will emit a deprecation warning. +end::template[] [[put-component-template-api-example]] ==== {api-examples-title} diff --git a/docs/reference/ingest/apis/simulate-ingest.asciidoc b/docs/reference/ingest/apis/simulate-ingest.asciidoc index 365870701db4b..de7527991e964 100644 --- a/docs/reference/ingest/apis/simulate-ingest.asciidoc +++ b/docs/reference/ingest/apis/simulate-ingest.asciidoc @@ -218,10 +218,11 @@ Map of component template names to substitute component template definition obje + .Properties of component template definition objects [%collapsible%open] + ==== -include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=mappings] -include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=settings] +include::{es-ref-dir}/indices/put-component-template.asciidoc[tag=template] + ==== [[simulate-ingest-api-example]]