From 3a73cfe74ccc33752a7eaf1e97b342d1a36e74b8 Mon Sep 17 00:00:00 2001 From: Keith Massey Date: Tue, 17 Sep 2024 16:21:04 -0500 Subject: [PATCH 1/5] Making transport changes to enable component template substitutions in the simulate ingest API --- .../lib/NativeLibraryProvider.java | 2 +- .../bulk/TransportSimulateBulkActionIT.java | 126 +++++++++++++++++- .../TransportSimulateIndexTemplateAction.java | 8 +- .../post/TransportSimulateTemplateAction.java | 3 +- .../bulk/TransportAbstractBulkAction.java | 17 ++- .../action/bulk/TransportBulkAction.java | 2 + .../bulk/TransportSimulateBulkAction.java | 38 ++++-- .../cluster/metadata/Metadata.java | 2 +- .../metadata/MetadataCreateIndexService.java | 10 +- .../MetadataIndexTemplateService.java | 43 +++++- .../elasticsearch/ingest/IngestService.java | 46 +++++-- ...sportSimulateIndexTemplateActionTests.java | 3 +- .../TransportSimulateBulkActionTests.java | 2 +- .../MetadataIndexTemplateServiceTests.java | 22 ++- .../ingest/IngestServiceTests.java | 84 +++++++++++- 15 files changed, 356 insertions(+), 52 deletions(-) diff --git a/libs/native/src/main/java/org/elasticsearch/nativeaccess/lib/NativeLibraryProvider.java b/libs/native/src/main/java/org/elasticsearch/nativeaccess/lib/NativeLibraryProvider.java index 5fc53d4d242d6..c3000982e4a6e 100644 --- a/libs/native/src/main/java/org/elasticsearch/nativeaccess/lib/NativeLibraryProvider.java +++ b/libs/native/src/main/java/org/elasticsearch/nativeaccess/lib/NativeLibraryProvider.java @@ -65,7 +65,7 @@ public T getLibrary(Class cls) { private static NativeLibraryProvider loadProvider() { final int runtimeVersion = Runtime.version().feature(); if (runtimeVersion >= 21) { - return loadJdkImpl(runtimeVersion); + // return loadJdkImpl(runtimeVersion); } return loadJnaImpl(); } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/TransportSimulateBulkActionIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/TransportSimulateBulkActionIT.java index 0ad254937d0dc..91674b7ce9050 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/TransportSimulateBulkActionIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/TransportSimulateBulkActionIT.java @@ -15,12 +15,14 @@ import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; +import org.elasticsearch.action.admin.indices.template.put.PutComponentTemplateAction; import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest; import org.elasticsearch.action.admin.indices.template.put.TransportPutComposableIndexTemplateAction; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.ingest.SimulateIndexResponse; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.cluster.metadata.ComponentTemplate; import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; import org.elasticsearch.cluster.metadata.Template; import org.elasticsearch.common.compress.CompressedXContent; @@ -57,7 +59,7 @@ public void testMappingValidationIndexExists() { } """; indicesAdmin().create(new CreateIndexRequest(indexName).mapping(mapping)).actionGet(); - BulkRequest bulkRequest = new BulkRequest(); + BulkRequest bulkRequest = new SimulateBulkRequest(Map.of(), Map.of()); bulkRequest.add(new IndexRequest(indexName).source(""" { "foo1": "baz" @@ -87,13 +89,125 @@ public void testMappingValidationIndexExists() { assertThat(fields.size(), equalTo(1)); } + @SuppressWarnings("unchecked") + public void testMappingValidationIndexExistsWithComponentTemplate() throws IOException { + /* + * This test simulates a BulkRequest of two documents into an existing index. Then we make sure the index contains no documents, and + * that the index's mapping in the cluster state has not been updated with the two new field. With the mapping from the template + * that was used to create the index, we would expect the second document to throw an exception because it uses a field that does + * not exist. But we substitute a new version of the component template named "test-component-template" that allows for the new + * field. + */ + String originalComponentTemplateMappingString = """ + { + "_doc":{ + "dynamic":"strict", + "properties":{ + "foo1":{ + "type":"text" + } + } + } + } + """; + CompressedXContent mapping = CompressedXContent.fromJSON(originalComponentTemplateMappingString); + Template template = new Template(Settings.EMPTY, mapping, null); + PutComponentTemplateAction.Request componentTemplateActionRequest = new PutComponentTemplateAction.Request( + "test-component-template" + ); + ComponentTemplate componentTemplate = new ComponentTemplate(template, null, null); + componentTemplateActionRequest.componentTemplate(componentTemplate); + client().execute(PutComponentTemplateAction.INSTANCE, componentTemplateActionRequest).actionGet(); + ComposableIndexTemplate composableIndexTemplate = ComposableIndexTemplate.builder() + .indexPatterns(List.of("my-index-*")) + .componentTemplates(List.of("test-component-template")) + .build(); + TransportPutComposableIndexTemplateAction.Request request = new TransportPutComposableIndexTemplateAction.Request("test"); + request.indexTemplate(composableIndexTemplate); + client().execute(TransportPutComposableIndexTemplateAction.TYPE, request).actionGet(); + + String indexName = "my-index-1"; + // First, run before the index is created: + assertMappingsUpdatedFromComponentTemplateSubstitutions(indexName); + // Now, create the index and make sure the component template substitutions work the same: + indicesAdmin().create(new CreateIndexRequest(indexName)).actionGet(); + assertMappingsUpdatedFromComponentTemplateSubstitutions(indexName); + // Now make sure nothing was actually changed: + indicesAdmin().refresh(new RefreshRequest(indexName)).actionGet(); + SearchResponse searchResponse = client().search(new SearchRequest(indexName)).actionGet(); + assertThat(searchResponse.getHits().getTotalHits().value, equalTo(0L)); + searchResponse.decRef(); + ClusterStateResponse clusterStateResponse = admin().cluster().state(new ClusterStateRequest(TEST_REQUEST_TIMEOUT)).actionGet(); + Map indexMapping = clusterStateResponse.getState().metadata().index(indexName).mapping().sourceAsMap(); + Map fields = (Map) indexMapping.get("properties"); + assertThat(fields.size(), equalTo(1)); + } + + private void assertMappingsUpdatedFromComponentTemplateSubstitutions(String indexName) { + IndexRequest indexRequest1 = new IndexRequest(indexName).source(""" + { + "foo1": "baz" + } + """, XContentType.JSON).id(randomUUID()); + IndexRequest indexRequest2 = new IndexRequest(indexName).source(""" + { + "foo3": "baz" + } + """, XContentType.JSON).id(randomUUID()); + { + // First we use the original component template, and expect a failure in the second document: + BulkRequest bulkRequest = new SimulateBulkRequest(Map.of(), Map.of()); + bulkRequest.add(indexRequest1); + bulkRequest.add(indexRequest2); + BulkResponse response = client().execute(new ActionType(SimulateBulkAction.NAME), bulkRequest).actionGet(); + assertThat(response.getItems().length, equalTo(2)); + assertThat(response.getItems()[0].getResponse().getResult(), equalTo(DocWriteResponse.Result.CREATED)); + assertNull(((SimulateIndexResponse) response.getItems()[0].getResponse()).getException()); + assertThat(response.getItems()[1].getResponse().getResult(), equalTo(DocWriteResponse.Result.CREATED)); + assertThat( + ((SimulateIndexResponse) response.getItems()[1].getResponse()).getException().getMessage(), + containsString("mapping set to strict, dynamic introduction of") + ); + } + + { + // Now we substitute a "test-component-template" that defines both fields, so we expect no exception: + BulkRequest bulkRequest = new SimulateBulkRequest( + Map.of(), + Map.of( + "test-component-template", + Map.of( + "template", + Map.of( + "mappings", + Map.of( + "dynamic", + "strict", + "properties", + Map.of("foo1", Map.of("type", "text"), "foo3", Map.of("type", "text")) + ) + ) + ) + ) + ); + bulkRequest.add(indexRequest1); + bulkRequest.add(indexRequest2); + BulkResponse response = client().execute(new ActionType(SimulateBulkAction.NAME), bulkRequest).actionGet(); + assertThat(response.getItems().length, equalTo(2)); + assertThat(response.getItems()[0].getResponse().getResult(), equalTo(DocWriteResponse.Result.CREATED)); + assertNull(((SimulateIndexResponse) response.getItems()[0].getResponse()).getException()); + assertThat(response.getItems()[1].getResponse().getResult(), equalTo(DocWriteResponse.Result.CREATED)); + assertNull(((SimulateIndexResponse) response.getItems()[1].getResponse()).getException()); + } + } + public void testMappingValidationIndexDoesNotExistsNoTemplate() { /* * This test simulates a BulkRequest of two documents into an index that does not exist. There is no template (other than the * mapping-less "random-index-template" created by the parent class), so we expect no mapping validation failure. */ String indexName = randomAlphaOfLength(20).toLowerCase(Locale.ROOT); - BulkRequest bulkRequest = new BulkRequest(); + BulkRequest bulkRequest = new SimulateBulkRequest(Map.of(), Map.of()); bulkRequest.add(new IndexRequest(indexName).source(""" { "foo1": "baz" @@ -140,7 +254,7 @@ public void testMappingValidationIndexDoesNotExistsV2Template() throws IOExcepti request.indexTemplate(composableIndexTemplate); client().execute(TransportPutComposableIndexTemplateAction.TYPE, request).actionGet(); - BulkRequest bulkRequest = new BulkRequest(); + BulkRequest bulkRequest = new SimulateBulkRequest(Map.of(), Map.of()); bulkRequest.add(new IndexRequest(indexName).source(""" { "foo1": "baz" @@ -172,7 +286,7 @@ public void testMappingValidationIndexDoesNotExistsV1Template() { indicesAdmin().putTemplate( new PutIndexTemplateRequest("test-template").patterns(List.of("my-index-*")).mapping("foo1", "type=integer") ).actionGet(); - BulkRequest bulkRequest = new BulkRequest(); + BulkRequest bulkRequest = new SimulateBulkRequest(Map.of(), Map.of()); bulkRequest.add(new IndexRequest(indexName).source(""" { "foo1": "baz" @@ -226,7 +340,7 @@ public void testMappingValidationIndexDoesNotExistsDataStream() throws IOExcepti client().execute(TransportPutComposableIndexTemplateAction.TYPE, request).actionGet(); { // First, try with no @timestamp to make sure we're picking up data-stream-specific templates - BulkRequest bulkRequest = new BulkRequest(); + BulkRequest bulkRequest = new SimulateBulkRequest(Map.of(), Map.of()); bulkRequest.add(new IndexRequest(indexName).source(""" { "foo1": "baz" @@ -252,7 +366,7 @@ public void testMappingValidationIndexDoesNotExistsDataStream() throws IOExcepti } { // Now with @timestamp - BulkRequest bulkRequest = new BulkRequest(); + BulkRequest bulkRequest = new SimulateBulkRequest(Map.of(), Map.of()); bulkRequest.add(new IndexRequest(indexName).source(""" { "@timestamp": "2024-08-27", diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java index fdced5fc18ac9..3561a4d0e2cb4 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java @@ -16,6 +16,7 @@ import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.metadata.AliasMetadata; +import org.elasticsearch.cluster.metadata.ComponentTemplate; import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.metadata.DataStreamLifecycle; @@ -156,7 +157,8 @@ protected void masterOperation( xContentRegistry, indicesService, systemIndices, - indexSettingProviders + indexSettingProviders, + Map.of() ); final Map> overlapping = new HashMap<>(); @@ -233,7 +235,8 @@ public static Template resolveTemplate( final NamedXContentRegistry xContentRegistry, final IndicesService indicesService, final SystemIndices systemIndices, - Set indexSettingProviders + Set indexSettingProviders, + Map componentTemplateSubstitutions ) throws Exception { var metadata = simulatedState.getMetadata(); Settings templateSettings = resolveSettings(simulatedState.metadata(), matchingTemplate); @@ -263,6 +266,7 @@ public static Template resolveTemplate( null, // empty request mapping as the user can't specify any explicit mappings via the simulate api simulatedState, matchingTemplate, + componentTemplateSubstitutions, xContentRegistry, simulatedIndexName ); diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateTemplateAction.java index 30bbad0b57df0..af7a253b5a042 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateTemplateAction.java @@ -170,7 +170,8 @@ protected void masterOperation( xContentRegistry, indicesService, systemIndices, - indexSettingProviders + indexSettingProviders, + Map.of() ); if (request.includeDefaults()) { listener.onResponse( diff --git a/server/src/main/java/org/elasticsearch/action/bulk/TransportAbstractBulkAction.java b/server/src/main/java/org/elasticsearch/action/bulk/TransportAbstractBulkAction.java index d306299645d64..6119e73118202 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/TransportAbstractBulkAction.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/TransportAbstractBulkAction.java @@ -24,6 +24,7 @@ import org.elasticsearch.cluster.ClusterStateObserver; import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.block.ClusterBlockLevel; +import org.elasticsearch.cluster.metadata.ComponentTemplate; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.io.stream.Writeable; @@ -39,6 +40,8 @@ import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; +import java.io.IOException; +import java.util.Map; import java.util.Objects; import java.util.concurrent.Executor; import java.util.concurrent.TimeUnit; @@ -163,19 +166,21 @@ public void onTimeout(TimeValue timeout) { private void forkAndExecute(Task task, BulkRequest bulkRequest, Executor executor, ActionListener releasingListener) { executor.execute(new ActionRunnable<>(releasingListener) { @Override - protected void doRun() { + protected void doRun() throws IOException { applyPipelinesAndDoInternalExecute(task, bulkRequest, executor, releasingListener); } }); } - private boolean applyPipelines(Task task, BulkRequest bulkRequest, Executor executor, ActionListener listener) { + private boolean applyPipelines(Task task, BulkRequest bulkRequest, Executor executor, ActionListener listener) + throws IOException { boolean hasIndexRequestsWithPipelines = false; final Metadata metadata = clusterService.state().getMetadata(); + Map templateSubstitutions = bulkRequest.getComponentTemplateSubstitutions(); for (DocWriteRequest actionRequest : bulkRequest.requests) { IndexRequest indexRequest = getIndexWriteRequest(actionRequest); if (indexRequest != null) { - IngestService.resolvePipelinesAndUpdateIndexRequest(actionRequest, indexRequest, metadata); + IngestService.resolvePipelinesAndUpdateIndexRequest(actionRequest, indexRequest, metadata, templateSubstitutions); hasIndexRequestsWithPipelines |= IngestService.hasPipeline(indexRequest); } @@ -245,7 +250,7 @@ private void processBulkIndexIngestRequest( } else { ActionRunnable runnable = new ActionRunnable<>(actionListener) { @Override - protected void doRun() { + protected void doRun() throws IOException { applyPipelinesAndDoInternalExecute(task, bulkRequest, executor, actionListener); } @@ -323,7 +328,7 @@ private void applyPipelinesAndDoInternalExecute( BulkRequest bulkRequest, Executor executor, ActionListener listener - ) { + ) throws IOException { final long relativeStartTimeNanos = relativeTimeNanos(); if (applyPipelines(task, bulkRequest, executor, listener) == false) { doInternalExecute(task, bulkRequest, executor, listener, relativeStartTimeNanos); @@ -344,6 +349,6 @@ protected abstract void doInternalExecute( Executor executor, ActionListener listener, long relativeStartTimeNanos - ); + ) throws IOException; } diff --git a/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java b/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java index d35688b410822..37ade67bf7d6d 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java @@ -205,6 +205,8 @@ protected void doInternalExecute( ActionListener listener, long relativeStartTimeNanos ) { + assert (bulkRequest instanceof SimulateBulkRequest) == false + : "TransportBulkAction should never be called with a SimulateBulkRequest"; trackIndexRequests(bulkRequest); Map indicesToAutoCreate = new HashMap<>(); diff --git a/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java b/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java index ada47f9de098c..51858b236fa52 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java @@ -16,6 +16,7 @@ import org.elasticsearch.action.ingest.SimulateIndexResponse; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.ComponentTemplate; import org.elasticsearch.cluster.metadata.IndexAbstraction; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.IndexTemplateMetadata; @@ -49,6 +50,7 @@ import org.elasticsearch.transport.TransportService; import org.elasticsearch.xcontent.NamedXContentRegistry; +import java.io.IOException; import java.util.List; import java.util.Map; import java.util.Set; @@ -108,13 +110,16 @@ protected void doInternalExecute( Executor executor, ActionListener listener, long relativeStartTimeNanos - ) { + ) throws IOException { + assert bulkRequest instanceof SimulateBulkRequest + : "TransportSimulateBulkAction should only ever be called with a SimulateBulkRequest but got a " + bulkRequest.getClass(); final AtomicArray responses = new AtomicArray<>(bulkRequest.requests.size()); + Map componentTemplateSubstitutions = bulkRequest.getComponentTemplateSubstitutions(); for (int i = 0; i < bulkRequest.requests.size(); i++) { DocWriteRequest docRequest = bulkRequest.requests.get(i); assert docRequest instanceof IndexRequest : "TransportSimulateBulkAction should only ever be called with IndexRequests"; IndexRequest request = (IndexRequest) docRequest; - Exception mappingValidationException = validateMappings(request); + Exception mappingValidationException = validateMappings(componentTemplateSubstitutions, request); responses.set( i, BulkItemResponse.success( @@ -140,10 +145,11 @@ protected void doInternalExecute( /** * This creates a temporary index with the mappings of the index in the request, and then attempts to index the source from the request * into it. If there is a mapping exception, that exception is returned. On success the returned exception is null. + * @parem componentTemplateSubstitutions The component template definitions to use in place of existing ones for validation * @param request The IndexRequest whose source will be validated against the mapping (if it exists) of its index * @return a mapping exception if the source does not match the mappings, otherwise null */ - private Exception validateMappings(IndexRequest request) { + private Exception validateMappings(Map componentTemplateSubstitutions, IndexRequest request) { final SourceToParse sourceToParse = new SourceToParse( request.id(), request.source(), @@ -157,7 +163,11 @@ private Exception validateMappings(IndexRequest request) { Exception mappingValidationException = null; IndexAbstraction indexAbstraction = state.metadata().getIndicesLookup().get(request.index()); try { - if (indexAbstraction != null) { + if (indexAbstraction != null && componentTemplateSubstitutions.isEmpty()) { + /* + * In this case the index exists and we don't have any component template overrides. So we can just use withTempIndexService + * to do the mapping validation, using all the existing logic for validation. + */ IndexMetadata imd = state.metadata().getIndexSafe(indexAbstraction.getWriteIndex(request, state.metadata())); indicesService.withTempIndexService(imd, indexService -> { indexService.mapperService().updateMapping(null, imd); @@ -178,23 +188,29 @@ private Exception validateMappings(IndexRequest request) { }); } else { /* - * The index did not exist, so we put together the mappings from existing templates. - * This reproduces a lot of the mapping resolution logic in MetadataCreateIndexService.applyCreateIndexRequest(). However, - * it does not deal with aliases (since an alias cannot be created if an index does not exist, and this is the path for - * when the index does not exist). And it does not deal with system indices since we do not intend for users to simulate - * writing to system indices. + * The index did not exist, or we have component template substitutions, so we put together the mappings from existing + * templates This reproduces a lot of the mapping resolution logic in MetadataCreateIndexService.applyCreateIndexRequest(). + * However, it does not deal with aliases (since an alias cannot be created if an index does not exist, and this is the + * path for when the index does not exist). And it does not deal with system indices since we do not intend for users to + * simulate writing to system indices. */ + // First, we remove the index from the cluster state if necessary (since we're going to use the templates) + ClusterState simulatedState = indexAbstraction == null + ? state + : new ClusterState.Builder(state).metadata(new Metadata.Builder(state.metadata()).remove(request.index()).build()) + .build(); String matchingTemplate = findV2Template(state.metadata(), request.index(), false); if (matchingTemplate != null) { final Template template = TransportSimulateIndexTemplateAction.resolveTemplate( matchingTemplate, request.index(), - state, + simulatedState, isDataStreamsLifecycleOnlyMode(clusterService.getSettings()), xContentRegistry, indicesService, systemIndices, - indexSettingProviders + indexSettingProviders, + componentTemplateSubstitutions ); CompressedXContent mappings = template.mappings(); if (mappings != null) { diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java index c1a2025271891..2014e18a8e016 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java @@ -1825,7 +1825,7 @@ public Builder() { this(Map.of(), 0); } - Builder(Metadata metadata) { + public Builder(Metadata metadata) { this.clusterUUID = metadata.clusterUUID; this.clusterUUIDCommitted = metadata.clusterUUIDCommitted; this.coordinationMetadata = metadata.coordinationMetadata; diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java index 07cfcddac9c93..275c186a1ea85 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java @@ -648,6 +648,7 @@ private ClusterState applyCreateIndexRequestWithV2Template( request.mappings(), currentState, templateName, + Map.of(), xContentRegistry, request.index() ); @@ -806,6 +807,7 @@ private static List collectSystemV2Mappings( List templateMappings = MetadataIndexTemplateService.collectMappings( composableIndexTemplate, componentTemplates, + Map.of(), indexName ); return collectV2Mappings(null, templateMappings, xContentRegistry); @@ -815,10 +817,16 @@ public static List collectV2Mappings( @Nullable final String requestMappings, final ClusterState currentState, final String templateName, + Map componentTemplateSubstitutions, final NamedXContentRegistry xContentRegistry, final String indexName ) throws Exception { - List templateMappings = MetadataIndexTemplateService.collectMappings(currentState, templateName, indexName); + List templateMappings = MetadataIndexTemplateService.collectMappings( + currentState, + templateName, + componentTemplateSubstitutions, + indexName + ); return collectV2Mappings(requestMappings, templateMappings, xContentRegistry); } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java index 207f773b54f34..9888059af9686 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java @@ -698,7 +698,7 @@ private void validateIndexTemplateV2(String name, ComposableIndexTemplate indexT final var now = Instant.now(); final var metadata = currentState.getMetadata(); - final var combinedMappings = collectMappings(indexTemplate, metadata.componentTemplates(), "tmp_idx"); + final var combinedMappings = collectMappings(indexTemplate, metadata.componentTemplates(), Map.of(), "tmp_idx"); final var combinedSettings = resolveSettings(indexTemplate, metadata.componentTemplates()); // First apply settings sourced from index setting providers: for (var provider : indexSettingProviders) { @@ -1348,7 +1348,12 @@ private static boolean isGlobalAndHasIndexHiddenSetting(Metadata metadata, Compo /** * Collect the given v2 template into an ordered list of mappings. */ - public static List collectMappings(final ClusterState state, final String templateName, final String indexName) { + public static List collectMappings( + final ClusterState state, + final String templateName, + Map componentTemplateSubstitutions, + final String indexName + ) { final ComposableIndexTemplate template = state.metadata().templatesV2().get(templateName); assert template != null : "attempted to resolve mappings for a template [" + templateName + "] that did not exist in the cluster state"; @@ -1357,7 +1362,7 @@ public static List collectMappings(final ClusterState state, } final Map componentTemplates = state.metadata().componentTemplates(); - return collectMappings(template, componentTemplates, indexName); + return collectMappings(template, componentTemplates, componentTemplateSubstitutions, indexName); } /** @@ -1366,6 +1371,7 @@ public static List collectMappings(final ClusterState state, public static List collectMappings( final ComposableIndexTemplate template, final Map componentTemplates, + final Map componentTemplateSubstitutions, final String indexName ) { Objects.requireNonNull(template, "Composable index template must be provided"); @@ -1376,9 +1382,12 @@ public static List collectMappings( ComposableIndexTemplate.DataStreamTemplate.DATA_STREAM_MAPPING_SNIPPET ); } + final Map combinedComponentTemplates = new HashMap<>(); + combinedComponentTemplates.putAll(componentTemplates); + combinedComponentTemplates.putAll(componentTemplateSubstitutions); List mappings = template.composedOf() .stream() - .map(componentTemplates::get) + .map(combinedComponentTemplates::get) .filter(Objects::nonNull) .map(ComponentTemplate::template) .map(Template::mappings) @@ -1428,24 +1437,44 @@ public static Settings resolveSettings(final List templat * Resolve the given v2 template into a collected {@link Settings} object */ public static Settings resolveSettings(final Metadata metadata, final String templateName) { + return resolveSettings(metadata, templateName, Map.of()); + } + + public static Settings resolveSettings( + final Metadata metadata, + final String templateName, + Map templateSubstitutions + ) { final ComposableIndexTemplate template = metadata.templatesV2().get(templateName); assert template != null : "attempted to resolve settings for a template [" + templateName + "] that did not exist in the cluster state"; if (template == null) { return Settings.EMPTY; } - return resolveSettings(template, metadata.componentTemplates()); + return resolveSettings(template, metadata.componentTemplates(), templateSubstitutions); } /** * Resolve the provided v2 template and component templates into a collected {@link Settings} object */ public static Settings resolveSettings(ComposableIndexTemplate template, Map componentTemplates) { + return resolveSettings(template, componentTemplates, Map.of()); + } + + public static Settings resolveSettings( + ComposableIndexTemplate template, + Map componentTemplates, + Map templateSubstitutions + ) { Objects.requireNonNull(template, "attempted to resolve settings for a null template"); Objects.requireNonNull(componentTemplates, "attempted to resolve settings with null component templates"); + Map combinedComponentTemplates = new HashMap<>(); + combinedComponentTemplates.putAll(componentTemplates); + // We want any substitutions to take precedence: + combinedComponentTemplates.putAll(templateSubstitutions); List componentSettings = template.composedOf() .stream() - .map(componentTemplates::get) + .map(combinedComponentTemplates::get) .filter(Objects::nonNull) .map(ComponentTemplate::template) .map(Template::settings) @@ -1694,7 +1723,7 @@ private static void validateCompositeTemplate( String indexName = DataStream.BACKING_INDEX_PREFIX + temporaryIndexName; // Parse mappings to ensure they are valid after being composed - List mappings = collectMappings(stateWithIndex, templateName, indexName); + List mappings = collectMappings(stateWithIndex, templateName, Map.of(), indexName); try { MapperService mapperService = tempIndexService.mapperService(); mapperService.merge(MapperService.SINGLE_MAPPING_NAME, mappings, MapperService.MergeReason.INDEX_TEMPLATE); diff --git a/server/src/main/java/org/elasticsearch/ingest/IngestService.java b/server/src/main/java/org/elasticsearch/ingest/IngestService.java index 40832cbdfc34f..4d857194d6634 100644 --- a/server/src/main/java/org/elasticsearch/ingest/IngestService.java +++ b/server/src/main/java/org/elasticsearch/ingest/IngestService.java @@ -33,6 +33,7 @@ import org.elasticsearch.cluster.ClusterStateApplier; import org.elasticsearch.cluster.ClusterStateTaskExecutor; import org.elasticsearch.cluster.ClusterStateTaskListener; +import org.elasticsearch.cluster.metadata.ComponentTemplate; import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.metadata.IndexAbstraction; import org.elasticsearch.cluster.metadata.IndexMetadata; @@ -270,26 +271,49 @@ public static void resolvePipelinesAndUpdateIndexRequest( final IndexRequest indexRequest, final Metadata metadata ) { - resolvePipelinesAndUpdateIndexRequest(originalRequest, indexRequest, metadata, System.currentTimeMillis()); + resolvePipelinesAndUpdateIndexRequest(originalRequest, indexRequest, metadata, Map.of()); + } + + public static void resolvePipelinesAndUpdateIndexRequest( + final DocWriteRequest originalRequest, + final IndexRequest indexRequest, + final Metadata metadata, + Map templateSubstitutions + ) { + resolvePipelinesAndUpdateIndexRequest(originalRequest, indexRequest, metadata, System.currentTimeMillis(), templateSubstitutions); } static void resolvePipelinesAndUpdateIndexRequest( final DocWriteRequest originalRequest, final IndexRequest indexRequest, final Metadata metadata, - final long epochMillis + final long epochMillis, + final Map componentTemplateSubstitutions ) { if (indexRequest.isPipelineResolved()) { return; } - String requestPipeline = indexRequest.getPipeline(); - - Pipelines pipelines = resolvePipelinesFromMetadata(originalRequest, indexRequest, metadata, epochMillis) // - .or(() -> resolvePipelinesFromIndexTemplates(indexRequest, metadata)) - .orElse(Pipelines.NO_PIPELINES_DEFINED); + /* + * Here we look for the pipelines associated with the index if the index exists. If the index does not exist we fall back to using + * templates to find the pipelines. But if a user has passed in component template substitutions, they want the settings from those + * used in place of the settings used to create any previous indices. So in that case we use the templates to find the pipelines -- + * we don't fall back to the existing index if we don't find any because it is possible the user has intentionally removed the + * pipeline. + */ + final Pipelines pipelines; + if (componentTemplateSubstitutions.isEmpty()) { + pipelines = resolvePipelinesFromMetadata(originalRequest, indexRequest, metadata, epochMillis) // + .or(() -> resolvePipelinesFromIndexTemplates(indexRequest, metadata, Map.of())) + .orElse(Pipelines.NO_PIPELINES_DEFINED); + } else { + pipelines = resolvePipelinesFromIndexTemplates(indexRequest, metadata, componentTemplateSubstitutions).orElse( + Pipelines.NO_PIPELINES_DEFINED + ); + } // The pipeline coming as part of the request always has priority over the resolved one from metadata or templates + String requestPipeline = indexRequest.getPipeline(); if (requestPipeline != null) { indexRequest.setPipeline(requestPipeline); } else { @@ -1436,7 +1460,11 @@ private static Optional resolvePipelinesFromMetadata( return Optional.of(new Pipelines(IndexSettings.DEFAULT_PIPELINE.get(settings), IndexSettings.FINAL_PIPELINE.get(settings))); } - private static Optional resolvePipelinesFromIndexTemplates(IndexRequest indexRequest, Metadata metadata) { + private static Optional resolvePipelinesFromIndexTemplates( + IndexRequest indexRequest, + Metadata metadata, + Map templateSubstitutions + ) { if (indexRequest.index() == null) { return Optional.empty(); } @@ -1446,7 +1474,7 @@ private static Optional resolvePipelinesFromIndexTemplates(IndexReque // precedence), or if a V2 template does not match, any V1 templates String v2Template = MetadataIndexTemplateService.findV2Template(metadata, indexRequest.index(), false); if (v2Template != null) { - final Settings settings = MetadataIndexTemplateService.resolveSettings(metadata, v2Template); + final Settings settings = MetadataIndexTemplateService.resolveSettings(metadata, v2Template, templateSubstitutions); return Optional.of(new Pipelines(IndexSettings.DEFAULT_PIPELINE.get(settings), IndexSettings.FINAL_PIPELINE.get(settings))); } diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateActionTests.java index 8f0ff82beab4b..9b1d8c15619ad 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateActionTests.java @@ -87,7 +87,8 @@ public Settings getAdditionalIndexSettings( xContentRegistry(), indicesService, systemIndices, - indexSettingsProviders + indexSettingsProviders, + Map.of() ); assertThat(resolvedTemplate.settings().getAsInt("test-setting", -1), is(1)); diff --git a/server/src/test/java/org/elasticsearch/action/bulk/TransportSimulateBulkActionTests.java b/server/src/test/java/org/elasticsearch/action/bulk/TransportSimulateBulkActionTests.java index e3c863ee69985..f4e53912d09a7 100644 --- a/server/src/test/java/org/elasticsearch/action/bulk/TransportSimulateBulkActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/bulk/TransportSimulateBulkActionTests.java @@ -133,7 +133,7 @@ public void tearDown() throws Exception { super.tearDown(); } - public void testIndexData() { + public void testIndexData() throws IOException { Task task = mock(Task.class); // unused BulkRequest bulkRequest = new SimulateBulkRequest(null, null); int bulkItemCount = randomIntBetween(0, 200); diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java index 38de29010a371..7a1d4b5b1ddf4 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java @@ -1081,7 +1081,7 @@ public void testResolveConflictingMappings() throws Exception { .build(); state = service.addIndexTemplateV2(state, true, "my-template", it); - List mappings = MetadataIndexTemplateService.collectMappings(state, "my-template", "my-index"); + List mappings = MetadataIndexTemplateService.collectMappings(state, "my-template", Map.of(), "my-index"); assertNotNull(mappings); assertThat(mappings.size(), equalTo(3)); @@ -1143,7 +1143,7 @@ public void testResolveMappings() throws Exception { .build(); state = service.addIndexTemplateV2(state, true, "my-template", it); - List mappings = MetadataIndexTemplateService.collectMappings(state, "my-template", "my-index"); + List mappings = MetadataIndexTemplateService.collectMappings(state, "my-template", Map.of(), "my-index"); assertNotNull(mappings); assertThat(mappings.size(), equalTo(3)); @@ -1197,6 +1197,7 @@ public void testDefinedTimestampMappingIsAddedForDataStreamTemplates() throws Ex List mappings = MetadataIndexTemplateService.collectMappings( state, "logs-data-stream-template", + Map.of(), DataStream.getDefaultBackingIndexName("logs", 1L) ); @@ -1248,7 +1249,12 @@ public void testDefinedTimestampMappingIsAddedForDataStreamTemplates() throws Ex .build(); state = service.addIndexTemplateV2(state, true, "timeseries-template", it); - List mappings = MetadataIndexTemplateService.collectMappings(state, "timeseries-template", "timeseries"); + List mappings = MetadataIndexTemplateService.collectMappings( + state, + "timeseries-template", + Map.of(), + "timeseries" + ); assertNotNull(mappings); assertThat(mappings.size(), equalTo(2)); @@ -1270,6 +1276,7 @@ public void testDefinedTimestampMappingIsAddedForDataStreamTemplates() throws Ex mappings = MetadataIndexTemplateService.collectMappings( state, "timeseries-template", + Map.of(), DataStream.getDefaultBackingIndexName("timeseries", 1L) ); @@ -1318,6 +1325,7 @@ public void testUserDefinedMappingTakesPrecedenceOverDefault() throws Exception List mappings = MetadataIndexTemplateService.collectMappings( state, "logs-template", + Map.of(), DataStream.getDefaultBackingIndexName("logs", 1L) ); @@ -1374,6 +1382,7 @@ public void testUserDefinedMappingTakesPrecedenceOverDefault() throws Exception List mappings = MetadataIndexTemplateService.collectMappings( state, "timeseries-template", + Map.of(), DataStream.getDefaultBackingIndexName("timeseries-template", 1L) ); @@ -2440,7 +2449,12 @@ public void testComposableTemplateWithSubobjectsFalse() throws Exception { .build(); state = service.addIndexTemplateV2(state, true, "composable-template", it); - List mappings = MetadataIndexTemplateService.collectMappings(state, "composable-template", "test-index"); + List mappings = MetadataIndexTemplateService.collectMappings( + state, + "composable-template", + Map.of(), + "test-index" + ); assertNotNull(mappings); assertThat(mappings.size(), equalTo(2)); diff --git a/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java b/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java index 3adaf398624de..49e75a71aa7f7 100644 --- a/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java +++ b/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java @@ -32,16 +32,20 @@ import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.AliasMetadata; +import org.elasticsearch.cluster.metadata.ComponentTemplate; +import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.IndexTemplateMetadata; import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.cluster.metadata.Template; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodeUtils; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.cluster.service.ClusterStateTaskExecutorUtils; import org.elasticsearch.common.TriConsumer; import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.time.DateFormatter; import org.elasticsearch.common.util.Maps; @@ -73,6 +77,7 @@ import org.mockito.ArgumentMatcher; import org.mockito.invocation.InvocationOnMock; +import java.io.IOException; import java.io.OutputStream; import java.nio.charset.StandardCharsets; import java.util.ArrayList; @@ -2488,7 +2493,7 @@ public void testResolveFinalPipelineWithDateMathExpression() { // index name matches with IDM: IndexRequest indexRequest = new IndexRequest(""); - IngestService.resolvePipelinesAndUpdateIndexRequest(indexRequest, indexRequest, metadata, epochMillis); + IngestService.resolvePipelinesAndUpdateIndexRequest(indexRequest, indexRequest, metadata, epochMillis, Map.of()); assertTrue(hasPipeline(indexRequest)); assertTrue(indexRequest.isPipelineResolved()); assertThat(indexRequest.getPipeline(), equalTo("_none")); @@ -2853,6 +2858,83 @@ public void testResolvePipelinesWithNonePipeline() { } } + public void testResolvePipelinesAndUpdateIndexRequestWithComponentTemplateSubstitutions() throws IOException { + final String componentTemplateName = "test-component-template"; + final String indexName = "my-index-1"; + final String indexPipeline = "index-pipeline"; + final String realTemplatePipeline = "template-pipeline"; + final String substitutePipeline = "substitute-pipeline"; + + Metadata metadata; + { + // Build up cluster state metadata + IndexMetadata.Builder builder = IndexMetadata.builder(indexName) + .settings(settings(IndexVersion.current())) + .numberOfShards(1) + .numberOfReplicas(0); + ComponentTemplate realComponentTemplate = new ComponentTemplate( + new Template( + Settings.builder().put("index.default_pipeline", realTemplatePipeline).build(), + CompressedXContent.fromJSON("{}"), + null + ), + null, + null + ); + ComposableIndexTemplate composableIndexTemplate = ComposableIndexTemplate.builder() + .indexPatterns(List.of("my-index-*")) + .componentTemplates(List.of(componentTemplateName)) + .build(); + metadata = Metadata.builder() + .put(builder) + .indexTemplates(Map.of("my-index-template", composableIndexTemplate)) + .componentTemplates(Map.of("test-component-template", realComponentTemplate)) + .build(); + } + + Map componentTemplateSubstitutions; + { + ComponentTemplate simulatedComponentTemplate = new ComponentTemplate( + new Template( + Settings.builder().put("index.default_pipeline", substitutePipeline).build(), + CompressedXContent.fromJSON("{}"), + null + ), + null, + null + ); + componentTemplateSubstitutions = Map.of(componentTemplateName, simulatedComponentTemplate); + } + + { + /* + * Here there is a pipeline in the request. This takes precedence over anything in the index or templates or component template + * substitutions. + */ + IndexRequest indexRequest = new IndexRequest(indexName).setPipeline(indexPipeline); + IngestService.resolvePipelinesAndUpdateIndexRequest(indexRequest, indexRequest, metadata, 0, componentTemplateSubstitutions); + assertThat(indexRequest.getPipeline(), equalTo(indexPipeline)); + } + { + /* + * Here there is no pipeline in the request, but there is one in the substitute component template. So it takes precedence. + */ + IndexRequest indexRequest = new IndexRequest(indexName); + IngestService.resolvePipelinesAndUpdateIndexRequest(indexRequest, indexRequest, metadata, 0, componentTemplateSubstitutions); + assertThat(indexRequest.getPipeline(), equalTo(substitutePipeline)); + } + { + /* + * This one is tricky. Since the index exists and there are no component template substitutions, we're going to use the actual + * index in this case rather than its template. The index does not have a default pipeline set, so it's "_none" instead of + * realTemplatePipeline. + */ + IndexRequest indexRequest = new IndexRequest(indexName); + IngestService.resolvePipelinesAndUpdateIndexRequest(indexRequest, indexRequest, metadata, 0, Map.of()); + assertThat(indexRequest.getPipeline(), equalTo("_none")); + } + } + private static Tuple randomMapEntry() { return tuple(randomAlphaOfLength(5), randomObject()); } From d738c64cad761d51cde5ace70b1a43d228e7a3f9 Mon Sep 17 00:00:00 2001 From: Keith Massey Date: Tue, 17 Sep 2024 16:28:10 -0500 Subject: [PATCH 2/5] removing native library hack --- .../elasticsearch/nativeaccess/lib/NativeLibraryProvider.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/native/src/main/java/org/elasticsearch/nativeaccess/lib/NativeLibraryProvider.java b/libs/native/src/main/java/org/elasticsearch/nativeaccess/lib/NativeLibraryProvider.java index c3000982e4a6e..5fc53d4d242d6 100644 --- a/libs/native/src/main/java/org/elasticsearch/nativeaccess/lib/NativeLibraryProvider.java +++ b/libs/native/src/main/java/org/elasticsearch/nativeaccess/lib/NativeLibraryProvider.java @@ -65,7 +65,7 @@ public T getLibrary(Class cls) { private static NativeLibraryProvider loadProvider() { final int runtimeVersion = Runtime.version().feature(); if (runtimeVersion >= 21) { - // return loadJdkImpl(runtimeVersion); + return loadJdkImpl(runtimeVersion); } return loadJnaImpl(); } From fc2ceba2d0e97a9eb8b5fe0d5b699242e4d0df1c Mon Sep 17 00:00:00 2001 From: Keith Massey Date: Thu, 19 Sep 2024 08:40:38 -0500 Subject: [PATCH 3/5] Adding an assertion about component template substitutions to TransportBulkAction --- .../org/elasticsearch/action/bulk/TransportBulkAction.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java b/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java index 37ade67bf7d6d..9a79644e7cffd 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java @@ -53,6 +53,7 @@ import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; +import java.io.IOException; import java.util.HashMap; import java.util.HashSet; import java.util.Map; @@ -204,9 +205,11 @@ protected void doInternalExecute( Executor executor, ActionListener listener, long relativeStartTimeNanos - ) { + ) throws IOException { assert (bulkRequest instanceof SimulateBulkRequest) == false : "TransportBulkAction should never be called with a SimulateBulkRequest"; + assert bulkRequest.getComponentTemplateSubstitutions().isEmpty() + : "Component template substitutions are not allowed in a non-simulated bulk"; trackIndexRequests(bulkRequest); Map indicesToAutoCreate = new HashMap<>(); From c36a44814720e424895e4097666b24ff46ddee08 Mon Sep 17 00:00:00 2001 From: Keith Massey Date: Thu, 19 Sep 2024 08:45:32 -0500 Subject: [PATCH 4/5] reverting change to Metadata --- .../action/bulk/TransportSimulateBulkAction.java | 4 ++-- .../java/org/elasticsearch/cluster/metadata/Metadata.java | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java b/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java index 51858b236fa52..0ea763c215959 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java @@ -197,8 +197,8 @@ private Exception validateMappings(Map componentTempl // First, we remove the index from the cluster state if necessary (since we're going to use the templates) ClusterState simulatedState = indexAbstraction == null ? state - : new ClusterState.Builder(state).metadata(new Metadata.Builder(state.metadata()).remove(request.index()).build()) - .build(); + : new ClusterState.Builder(state).metadata(Metadata.builder(state.metadata()).remove(request.index()).build()).build(); + String matchingTemplate = findV2Template(state.metadata(), request.index(), false); if (matchingTemplate != null) { final Template template = TransportSimulateIndexTemplateAction.resolveTemplate( diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java index 5ff1986f9531b..d2f5ab5eabaee 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java @@ -1818,7 +1818,7 @@ public Builder() { this(Map.of(), 0); } - public Builder(Metadata metadata) { + Builder(Metadata metadata) { this.clusterUUID = metadata.clusterUUID; this.clusterUUIDCommitted = metadata.clusterUUIDCommitted; this.coordinationMetadata = metadata.coordinationMetadata; From 88c6edfa296709f26ca9a5ebf75b97872e5d1d1a Mon Sep 17 00:00:00 2001 From: Keith Massey Date: Thu, 19 Sep 2024 08:47:18 -0500 Subject: [PATCH 5/5] renaming templateSubstitutions -> componentTemplateSubstitutions in IngestService --- .../org/elasticsearch/ingest/IngestService.java | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/ingest/IngestService.java b/server/src/main/java/org/elasticsearch/ingest/IngestService.java index 4d857194d6634..0275e988ce39d 100644 --- a/server/src/main/java/org/elasticsearch/ingest/IngestService.java +++ b/server/src/main/java/org/elasticsearch/ingest/IngestService.java @@ -278,9 +278,15 @@ public static void resolvePipelinesAndUpdateIndexRequest( final DocWriteRequest originalRequest, final IndexRequest indexRequest, final Metadata metadata, - Map templateSubstitutions + Map componentTemplateSubstitutions ) { - resolvePipelinesAndUpdateIndexRequest(originalRequest, indexRequest, metadata, System.currentTimeMillis(), templateSubstitutions); + resolvePipelinesAndUpdateIndexRequest( + originalRequest, + indexRequest, + metadata, + System.currentTimeMillis(), + componentTemplateSubstitutions + ); } static void resolvePipelinesAndUpdateIndexRequest( @@ -1463,7 +1469,7 @@ private static Optional resolvePipelinesFromMetadata( private static Optional resolvePipelinesFromIndexTemplates( IndexRequest indexRequest, Metadata metadata, - Map templateSubstitutions + Map componentTemplateSubstitutions ) { if (indexRequest.index() == null) { return Optional.empty(); @@ -1474,7 +1480,7 @@ private static Optional resolvePipelinesFromIndexTemplates( // precedence), or if a V2 template does not match, any V1 templates String v2Template = MetadataIndexTemplateService.findV2Template(metadata, indexRequest.index(), false); if (v2Template != null) { - final Settings settings = MetadataIndexTemplateService.resolveSettings(metadata, v2Template, templateSubstitutions); + final Settings settings = MetadataIndexTemplateService.resolveSettings(metadata, v2Template, componentTemplateSubstitutions); return Optional.of(new Pipelines(IndexSettings.DEFAULT_PIPELINE.get(settings), IndexSettings.FINAL_PIPELINE.get(settings))); }