From cc2cabad88ce78fe574638050864233c77ecac3f Mon Sep 17 00:00:00 2001 From: Niels Bauman Date: Mon, 29 Sep 2025 11:36:49 -0300 Subject: [PATCH 1/2] Add small optimizations to `PUT _component_template` API By moving component template normalization to the transport action, we can perform an equality check on the component template before creating a cluster state update task. This reduces the load on the master/cluster state update thread for no-op update requests. We still need to perform the equality check in the cluster state update task, as the cluster state could have been updated after we performed the check in the transport action. But avoiding a cluster state update task altogether in some cases is worth adding the check to the transport action. Additionally, by making some small refactorings in `MetadataIndexTemplateService#addComponentTemplate` we avoid building some objects unnecessarily and improve readability. --- .../template/ComposableTemplateIT.java | 549 +++++++++++++++++- .../TransportPutComponentTemplateAction.java | 53 +- ...ReservedComposableIndexTemplateAction.java | 14 +- .../cluster/metadata/ComponentTemplate.java | 17 +- .../MetadataIndexTemplateService.java | 119 ++-- .../common/settings/Settings.java | 13 + .../elasticsearch/node/NodeConstruction.java | 5 +- ...vedComposableIndexTemplateActionTests.java | 22 +- .../MetadataIndexTemplateServiceTests.java | 507 ---------------- 9 files changed, 669 insertions(+), 630 deletions(-) diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/template/ComposableTemplateIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/template/ComposableTemplateIT.java index e23e342b91d65..8dd8561377d71 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/template/ComposableTemplateIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/template/ComposableTemplateIT.java @@ -13,11 +13,23 @@ import org.elasticsearch.action.admin.indices.template.put.TransportPutComposableIndexTemplateAction; import org.elasticsearch.cluster.metadata.ComponentTemplate; import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; +import org.elasticsearch.cluster.metadata.DataStream; +import org.elasticsearch.cluster.metadata.MetadataIndexTemplateService; +import org.elasticsearch.cluster.metadata.ProjectId; +import org.elasticsearch.cluster.metadata.ProjectMetadata; import org.elasticsearch.cluster.metadata.Template; import org.elasticsearch.common.compress.CompressedXContent; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.xcontent.NamedXContentRegistry; import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ExecutionException; + +import static org.hamcrest.Matchers.equalTo; public class ComposableTemplateIT extends ESIntegTestCase { @@ -32,7 +44,7 @@ public void testComponentTemplatesCanBeUpdatedAfterRestart() throws Exception { } } }"""), null), 3L, Collections.singletonMap("eggplant", "potato")); - client().execute(PutComponentTemplateAction.INSTANCE, new PutComponentTemplateAction.Request("my-ct").componentTemplate(ct)).get(); + addComponentTemplate("my-ct", ct); ComposableIndexTemplate cit = ComposableIndexTemplate.builder() .indexPatterns(Collections.singletonList("coleslaw")) @@ -50,10 +62,7 @@ public void testComponentTemplatesCanBeUpdatedAfterRestart() throws Exception { .version(5L) .metadata(Collections.singletonMap("egg", "bread")) .build(); - client().execute( - TransportPutComposableIndexTemplateAction.TYPE, - new TransportPutComposableIndexTemplateAction.Request("my-it").indexTemplate(cit) - ).get(); + addComposableTemplate("my-it", cit); internalCluster().fullRestart(); ensureGreen(); @@ -67,7 +76,7 @@ public void testComponentTemplatesCanBeUpdatedAfterRestart() throws Exception { } } }"""), null), 3L, Collections.singletonMap("eggplant", "potato")); - client().execute(PutComponentTemplateAction.INSTANCE, new PutComponentTemplateAction.Request("my-ct").componentTemplate(ct2)).get(); + addComponentTemplate("my-ct", ct2); ComposableIndexTemplate cit2 = ComposableIndexTemplate.builder() .indexPatterns(Collections.singletonList("coleslaw")) @@ -85,9 +94,535 @@ public void testComponentTemplatesCanBeUpdatedAfterRestart() throws Exception { .version(5L) .metadata(Collections.singletonMap("egg", "bread")) .build(); + addComposableTemplate("my-it", cit2); + } + + public void testComposableTemplateWithSubobjectsFalseObjectAndSubfield() throws Exception { + ComponentTemplate subobjects = new ComponentTemplate(new Template(null, new CompressedXContent(""" + { + "properties": { + "foo": { + "type": "object", + "subobjects": false + }, + "foo.bar": { + "type": "keyword" + } + } + } + """), null), null, null); + + addComponentTemplate("subobjects", subobjects); + ComposableIndexTemplate it = ComposableIndexTemplate.builder() + .indexPatterns(List.of("test-*")) + .template(new Template(null, null, null)) + .componentTemplates(List.of("subobjects")) + .priority(0L) + .version(1L) + .build(); + addComposableTemplate("composable-template", it); + + ProjectMetadata project = getProjectMetadata(); + List mappings = MetadataIndexTemplateService.collectMappings(project, "composable-template", "test-index"); + + assertNotNull(mappings); + assertThat(mappings.size(), equalTo(1)); + List> parsedMappings = mappings.stream().map(m -> { + try { + return MapperService.parseMapping(NamedXContentRegistry.EMPTY, m); + } catch (Exception e) { + logger.error(e); + fail("failed to parse mappings: " + m.string()); + return null; + } + }).toList(); + + assertThat( + parsedMappings.get(0), + equalTo( + Map.of( + "_doc", + Map.of("properties", Map.of("foo.bar", Map.of("type", "keyword"), "foo", Map.of("type", "object", "subobjects", false))) + ) + ) + ); + } + + public void testComposableTemplateWithSubobjectsFalse() throws Exception { + ComponentTemplate subobjects = new ComponentTemplate(new Template(null, new CompressedXContent(""" + { + "subobjects": false + } + """), null), null, null); + + ComponentTemplate fieldMapping = new ComponentTemplate(new Template(null, new CompressedXContent(""" + { + "properties": { + "parent.subfield": { + "type": "keyword" + } + } + } + """), null), null, null); + + addComponentTemplate("subobjects", subobjects); + addComponentTemplate("field_mapping", fieldMapping); + ComposableIndexTemplate it = ComposableIndexTemplate.builder() + .indexPatterns(List.of("test-*")) + .template(new Template(null, null, null)) + .componentTemplates(List.of("subobjects", "field_mapping")) + .priority(0L) + .version(1L) + .build(); + addComposableTemplate("composable-template", it); + + ProjectMetadata project = getProjectMetadata(); + List mappings = MetadataIndexTemplateService.collectMappings(project, "composable-template", "test-index"); + + assertNotNull(mappings); + assertThat(mappings.size(), equalTo(2)); + List> parsedMappings = mappings.stream().map(m -> { + try { + return MapperService.parseMapping(NamedXContentRegistry.EMPTY, m); + } catch (Exception e) { + logger.error(e); + fail("failed to parse mappings: " + m.string()); + return null; + } + }).toList(); + + assertThat(parsedMappings.get(0), equalTo(Map.of("_doc", Map.of("subobjects", false)))); + assertThat( + parsedMappings.get(1), + equalTo(Map.of("_doc", Map.of("properties", Map.of("parent.subfield", Map.of("type", "keyword"))))) + ); + } + + public void testResolveConflictingMappings() throws Exception { + ComponentTemplate ct1 = new ComponentTemplate(new Template(null, new CompressedXContent(""" + { + "properties": { + "field2": { + "type": "keyword" + } + } + }"""), null), null, null); + ComponentTemplate ct2 = new ComponentTemplate(new Template(null, new CompressedXContent(""" + { + "properties": { + "field2": { + "type": "text" + } + } + }"""), null), null, null); + addComponentTemplate("ct_high", ct1); + addComponentTemplate("ct_low", ct2); + ComposableIndexTemplate it = ComposableIndexTemplate.builder() + .indexPatterns(List.of("i*")) + .template(new Template(null, new CompressedXContent(""" + { + "properties": { + "field": { + "type": "keyword" + } + } + }"""), null)) + .componentTemplates(List.of("ct_low", "ct_high")) + .priority(0L) + .version(1L) + .build(); + addComposableTemplate("my-template", it); + + ProjectMetadata project = getProjectMetadata(); + List mappings = MetadataIndexTemplateService.collectMappings(project, "my-template", "my-index"); + + assertNotNull(mappings); + assertThat(mappings.size(), equalTo(3)); + List> parsedMappings = mappings.stream().map(m -> { + try { + return MapperService.parseMapping(NamedXContentRegistry.EMPTY, m); + } catch (Exception e) { + logger.error(e); + fail("failed to parse mappings: " + m.string()); + return null; + } + }).toList(); + + // The order of mappings should be: + // - ct_low + // - ct_high + // - index template + // Because the first elements when merging mappings have the lowest precedence + assertThat(parsedMappings.get(0), equalTo(Map.of("_doc", Map.of("properties", Map.of("field2", Map.of("type", "text")))))); + assertThat(parsedMappings.get(1), equalTo(Map.of("_doc", Map.of("properties", Map.of("field2", Map.of("type", "keyword")))))); + assertThat(parsedMappings.get(2), equalTo(Map.of("_doc", Map.of("properties", Map.of("field", Map.of("type", "keyword")))))); + } + + public void testResolveMappings() throws Exception { + ComponentTemplate ct1 = new ComponentTemplate(new Template(null, new CompressedXContent(""" + { + "properties": { + "field1": { + "type": "keyword" + } + } + }"""), null), null, null); + ComponentTemplate ct2 = new ComponentTemplate(new Template(null, new CompressedXContent(""" + { + "properties": { + "field2": { + "type": "text" + } + } + }"""), null), null, null); + addComponentTemplate("ct_high", ct1); + addComponentTemplate("ct_low", ct2); + ComposableIndexTemplate it = ComposableIndexTemplate.builder() + .indexPatterns(List.of("i*")) + .template(new Template(null, new CompressedXContent(""" + { + "properties": { + "field3": { + "type": "integer" + } + } + }"""), null)) + .componentTemplates(List.of("ct_low", "ct_high")) + .priority(0L) + .version(1L) + .build(); + addComposableTemplate("my-template", it); + + ProjectMetadata project = getProjectMetadata(); + List mappings = MetadataIndexTemplateService.collectMappings(project, "my-template", "my-index"); + + assertNotNull(mappings); + assertThat(mappings.size(), equalTo(3)); + List> parsedMappings = mappings.stream().map(m -> { + try { + return MapperService.parseMapping(NamedXContentRegistry.EMPTY, m); + } catch (Exception e) { + logger.error(e); + fail("failed to parse mappings: " + m.string()); + return null; + } + }).toList(); + assertThat(parsedMappings.get(0), equalTo(Map.of("_doc", Map.of("properties", Map.of("field2", Map.of("type", "text")))))); + assertThat(parsedMappings.get(1), equalTo(Map.of("_doc", Map.of("properties", Map.of("field1", Map.of("type", "keyword")))))); + assertThat(parsedMappings.get(2), equalTo(Map.of("_doc", Map.of("properties", Map.of("field3", Map.of("type", "integer")))))); + } + + public void testDefinedTimestampMappingIsAddedForDataStreamTemplates() throws Exception { + ComponentTemplate ct1 = new ComponentTemplate(new Template(null, new CompressedXContent(""" + { + "properties": { + "field1": { + "type": "keyword" + } + } + }"""), null), null, null); + + addComponentTemplate("ct1", ct1); + + { + ComposableIndexTemplate it = ComposableIndexTemplate.builder() + .indexPatterns(List.of("logs*")) + .template(new Template(null, new CompressedXContent(""" + { + "properties": { + "field2": { + "type": "integer" + } + } + }"""), null)) + .componentTemplates(List.of("ct1")) + .priority(0L) + .version(1L) + .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate()) + .build(); + addComposableTemplate("logs-data-stream-template", it); + + ProjectMetadata project = getProjectMetadata(); + List mappings = MetadataIndexTemplateService.collectMappings( + project, + "logs-data-stream-template", + DataStream.getDefaultBackingIndexName("logs", 1L) + ); + + assertNotNull(mappings); + assertThat(mappings.size(), equalTo(4)); + List> parsedMappings = mappings.stream().map(m -> { + try { + return MapperService.parseMapping(NamedXContentRegistry.EMPTY, m); + } catch (Exception e) { + logger.error(e); + fail("failed to parse mappings: " + m.string()); + return null; + } + }).toList(); + + assertThat( + parsedMappings.get(0), + equalTo( + Map.of( + "_doc", + Map.of( + "properties", + Map.of( + MetadataIndexTemplateService.DEFAULT_TIMESTAMP_FIELD, + Map.of("type", "date", "ignore_malformed", "false") + ), + "_routing", + Map.of("required", false) + ) + ) + ) + ); + assertThat(parsedMappings.get(1), equalTo(Map.of("_doc", Map.of("properties", Map.of("field1", Map.of("type", "keyword")))))); + assertThat(parsedMappings.get(2), equalTo(Map.of("_doc", Map.of("properties", Map.of("field2", Map.of("type", "integer")))))); + } + + { + // indices matched by templates without the data stream field defined don't get the default @timestamp mapping + ComposableIndexTemplate it = ComposableIndexTemplate.builder() + .indexPatterns(List.of("timeseries*")) + .template(new Template(null, new CompressedXContent(""" + { + "properties": { + "field2": { + "type": "integer" + } + } + }"""), null)) + .componentTemplates(List.of("ct1")) + .priority(0L) + .version(1L) + .build(); + addComposableTemplate("timeseries-template", it); + + ProjectMetadata project = getProjectMetadata(); + List mappings = MetadataIndexTemplateService.collectMappings(project, "timeseries-template", "timeseries"); + + assertNotNull(mappings); + assertThat(mappings.size(), equalTo(2)); + List> parsedMappings = mappings.stream().map(m -> { + try { + return MapperService.parseMapping(NamedXContentRegistry.EMPTY, m); + } catch (Exception e) { + logger.error(e); + fail("failed to parse mappings: " + m.string()); + return null; + } + }).toList(); + + assertThat(parsedMappings.get(0), equalTo(Map.of("_doc", Map.of("properties", Map.of("field1", Map.of("type", "keyword")))))); + assertThat(parsedMappings.get(1), equalTo(Map.of("_doc", Map.of("properties", Map.of("field2", Map.of("type", "integer")))))); + + // a default @timestamp mapping will not be added if the matching template doesn't have the data stream field configured, even + // if the index name matches that of a data stream backing index + mappings = MetadataIndexTemplateService.collectMappings( + project, + "timeseries-template", + DataStream.getDefaultBackingIndexName("timeseries", 1L) + ); + + assertNotNull(mappings); + assertThat(mappings.size(), equalTo(2)); + parsedMappings = mappings.stream().map(m -> { + try { + return MapperService.parseMapping(NamedXContentRegistry.EMPTY, m); + } catch (Exception e) { + logger.error(e); + fail("failed to parse mappings: " + m.string()); + return null; + } + }).toList(); + + assertThat(parsedMappings.get(0), equalTo(Map.of("_doc", Map.of("properties", Map.of("field1", Map.of("type", "keyword")))))); + assertThat(parsedMappings.get(1), equalTo(Map.of("_doc", Map.of("properties", Map.of("field2", Map.of("type", "integer")))))); + } + } + + public void testUserDefinedMappingTakesPrecedenceOverDefault() throws Exception { + { + // user defines a @timestamp mapping as part of a component template + ComponentTemplate ct1 = new ComponentTemplate(new Template(null, new CompressedXContent(""" + { + "properties": { + "@timestamp": { + "type": "date_nanos" + } + } + }"""), null), null, null); + addComponentTemplate("ct1", ct1); + + ComposableIndexTemplate it = ComposableIndexTemplate.builder() + .indexPatterns(List.of("logs*")) + .componentTemplates(List.of("ct1")) + .priority(0L) + .version(1L) + .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate()) + .build(); + addComposableTemplate("logs-template", it); + + ProjectMetadata project = getProjectMetadata(); + List mappings = MetadataIndexTemplateService.collectMappings( + project, + "logs-template", + DataStream.getDefaultBackingIndexName("logs", 1L) + ); + + assertNotNull(mappings); + assertThat(mappings.size(), equalTo(3)); + List> parsedMappings = mappings.stream().map(m -> { + try { + return MapperService.parseMapping(NamedXContentRegistry.EMPTY, m); + } catch (Exception e) { + logger.error(e); + fail("failed to parse mappings: " + m.string()); + return null; + } + }).toList(); + assertThat( + parsedMappings.get(0), + equalTo( + Map.of( + "_doc", + Map.of( + "properties", + Map.of( + MetadataIndexTemplateService.DEFAULT_TIMESTAMP_FIELD, + Map.of("type", "date", "ignore_malformed", "false") + ), + "_routing", + Map.of("required", false) + ) + ) + ) + ); + assertThat( + parsedMappings.get(1), + equalTo( + Map.of( + "_doc", + Map.of("properties", Map.of(MetadataIndexTemplateService.DEFAULT_TIMESTAMP_FIELD, Map.of("type", "date_nanos"))) + ) + ) + ); + } + + { + // user defines a @timestamp mapping as part of a composable index template + Template template = new Template(null, new CompressedXContent(""" + { + "properties": { + "@timestamp": { + "type": "date_nanos" + } + } + }"""), null); + ComposableIndexTemplate it = ComposableIndexTemplate.builder() + .indexPatterns(List.of("timeseries*")) + .template(template) + .priority(0L) + .version(1L) + .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate()) + .build(); + addComposableTemplate("timeseries-template", it); + + ProjectMetadata project = getProjectMetadata(); + List mappings = MetadataIndexTemplateService.collectMappings( + project, + "timeseries-template", + DataStream.getDefaultBackingIndexName("timeseries-template", 1L) + ); + + assertNotNull(mappings); + assertThat(mappings.size(), equalTo(3)); + List> parsedMappings = mappings.stream().map(m -> { + try { + return MapperService.parseMapping(NamedXContentRegistry.EMPTY, m); + } catch (Exception e) { + logger.error(e); + fail("failed to parse mappings: " + m.string()); + return null; + } + }).toList(); + assertThat( + parsedMappings.get(0), + equalTo( + Map.of( + "_doc", + Map.of( + "properties", + Map.of( + MetadataIndexTemplateService.DEFAULT_TIMESTAMP_FIELD, + Map.of("type", "date", "ignore_malformed", "false") + ), + "_routing", + Map.of("required", false) + ) + ) + ) + ); + assertThat( + parsedMappings.get(1), + equalTo( + Map.of( + "_doc", + Map.of("properties", Map.of(MetadataIndexTemplateService.DEFAULT_TIMESTAMP_FIELD, Map.of("type", "date_nanos"))) + ) + ) + ); + } + } + + public void testResolveSettings() throws Exception { + ComponentTemplate ct1 = new ComponentTemplate( + new Template(Settings.builder().put("number_of_replicas", 2).put("index.blocks.write", true).build(), null, null), + null, + null + ); + ComponentTemplate ct2 = new ComponentTemplate( + new Template(Settings.builder().put("index.number_of_replicas", 1).put("index.blocks.read", true).build(), null, null), + null, + null + ); + addComponentTemplate("ct_high", ct1); + addComponentTemplate("ct_low", ct2); + ComposableIndexTemplate it = ComposableIndexTemplate.builder() + .indexPatterns(List.of("i*")) + .template( + new Template(Settings.builder().put("index.blocks.write", false).put("index.number_of_shards", 3).build(), null, null) + ) + .componentTemplates(List.of("ct_low", "ct_high")) + .priority(0L) + .version(1L) + .build(); + addComposableTemplate("my-template", it); + + ProjectMetadata project = getProjectMetadata(); + Settings settings = MetadataIndexTemplateService.resolveSettings(project, "my-template"); + assertThat(settings.get("index.number_of_replicas"), equalTo("2")); + assertThat(settings.get("index.blocks.write"), equalTo("false")); + assertThat(settings.get("index.blocks.read"), equalTo("true")); + assertThat(settings.get("index.number_of_shards"), equalTo("3")); + } + + private static void addComponentTemplate(String name, ComponentTemplate template) throws InterruptedException, ExecutionException { + client().execute(PutComponentTemplateAction.INSTANCE, new PutComponentTemplateAction.Request(name).componentTemplate(template)) + .get(); + } + + private static void addComposableTemplate(String name, ComposableIndexTemplate template) throws InterruptedException, + ExecutionException { client().execute( TransportPutComposableIndexTemplateAction.TYPE, - new TransportPutComposableIndexTemplateAction.Request("my-it").indexTemplate(cit2) + new TransportPutComposableIndexTemplateAction.Request(name).indexTemplate(template) ).get(); } + + private ProjectMetadata getProjectMetadata() { + return client().admin().cluster().prepareState(TEST_REQUEST_TIMEOUT).get().getState().metadata().getProject(ProjectId.DEFAULT); + } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutComponentTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutComponentTemplateAction.java index e7cdf2f7c872f..514dcc507de7c 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutComponentTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutComponentTemplateAction.java @@ -18,14 +18,10 @@ import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.metadata.ComponentTemplate; -import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.MetadataIndexTemplateService; -import org.elasticsearch.cluster.metadata.Template; import org.elasticsearch.cluster.project.ProjectResolver; import org.elasticsearch.cluster.project.ProjectStateRegistry; import org.elasticsearch.cluster.service.ClusterService; -import org.elasticsearch.common.settings.IndexScopedSettings; -import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.FixForMultiProject; import org.elasticsearch.injection.guice.Inject; @@ -39,7 +35,6 @@ public class TransportPutComponentTemplateAction extends AcknowledgedTransportMasterNodeAction { private final MetadataIndexTemplateService indexTemplateService; - private final IndexScopedSettings indexScopedSettings; private final ProjectResolver projectResolver; @Inject @@ -49,7 +44,6 @@ public TransportPutComponentTemplateAction( ThreadPool threadPool, MetadataIndexTemplateService indexTemplateService, ActionFilters actionFilters, - IndexScopedSettings indexScopedSettings, ProjectResolver projectResolver ) { super( @@ -62,7 +56,6 @@ public TransportPutComponentTemplateAction( EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.indexTemplateService = indexTemplateService; - this.indexScopedSettings = indexScopedSettings; this.projectResolver = projectResolver; } @@ -71,46 +64,36 @@ protected ClusterBlockException checkBlock(PutComponentTemplateAction.Request re return state.blocks().globalBlockedException(projectResolver.getProjectId(), ClusterBlockLevel.METADATA_WRITE); } - public static ComponentTemplate normalizeComponentTemplate( - ComponentTemplate componentTemplate, - IndexScopedSettings indexScopedSettings - ) { - Template template = componentTemplate.template(); - // Normalize the index settings if necessary - if (template.settings() != null) { - Settings.Builder builder = Settings.builder().put(template.settings()).normalizePrefix(IndexMetadata.INDEX_SETTING_PREFIX); - Settings settings = builder.build(); - indexScopedSettings.validate(settings, true); - template = Template.builder(template).settings(settings).build(); - componentTemplate = new ComponentTemplate( - template, - componentTemplate.version(), - componentTemplate.metadata(), - componentTemplate.deprecated(), - componentTemplate.createdDateMillis().orElse(null), - componentTemplate.modifiedDateMillis().orElse(null) - ); - } - - return componentTemplate; - } - @Override protected void masterOperation( Task task, final PutComponentTemplateAction.Request request, final ClusterState state, final ActionListener listener - ) { - ComponentTemplate componentTemplate = normalizeComponentTemplate(request.componentTemplate(), indexScopedSettings); - var projectId = projectResolver.getProjectId(); + ) throws Exception { + final var project = projectResolver.getProjectMetadata(state); + final ComponentTemplate componentTemplate = indexTemplateService.normalizeComponentTemplate(request.componentTemplate()); + final ComponentTemplate existingTemplate = project.componentTemplates().get(request.name()); + if (existingTemplate != null) { + if (request.create()) { + listener.onFailure(new IllegalArgumentException("component template [" + request.name() + "] already exists")); + return; + } + // We have an early return here in case the component template already exists and is identical in content. We still need to do + // this check in the cluster state update task in case the cluster state changed since this check. + if (componentTemplate.contentEquals(existingTemplate)) { + listener.onResponse(AcknowledgedResponse.TRUE); + return; + } + } + indexTemplateService.putComponentTemplate( request.cause(), request.create(), request.name(), request.masterNodeTimeout(), componentTemplate, - projectId, + project.id(), listener ); } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/reservedstate/ReservedComposableIndexTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/reservedstate/ReservedComposableIndexTemplateAction.java index a70052d135b42..1654bdca0f847 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/reservedstate/ReservedComposableIndexTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/reservedstate/ReservedComposableIndexTemplateAction.java @@ -18,7 +18,6 @@ import org.elasticsearch.cluster.metadata.MetadataIndexTemplateService; import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.cluster.metadata.ProjectMetadata; -import org.elasticsearch.common.settings.IndexScopedSettings; import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.reservedstate.ReservedClusterStateHandler; import org.elasticsearch.reservedstate.ReservedProjectStateHandler; @@ -56,14 +55,9 @@ public class ReservedComposableIndexTemplateAction public static final String COMPOSABLE_PREFIX = "composable_index_template:"; private final MetadataIndexTemplateService indexTemplateService; - private final IndexScopedSettings indexScopedSettings; - public ReservedComposableIndexTemplateAction( - MetadataIndexTemplateService indexTemplateService, - IndexScopedSettings indexScopedSettings - ) { + public ReservedComposableIndexTemplateAction(MetadataIndexTemplateService indexTemplateService) { this.indexTemplateService = indexTemplateService; - this.indexScopedSettings = indexScopedSettings; } @Override @@ -154,11 +148,7 @@ public TransformState transform(ProjectId projectId, ComponentsAndComposables so // 1. create or update component templates (composable templates depend on them) for (var request : components) { - ComponentTemplate template = TransportPutComponentTemplateAction.normalizeComponentTemplate( - request.componentTemplate(), - indexScopedSettings - ); - + ComponentTemplate template = indexTemplateService.normalizeComponentTemplate(request.componentTemplate()); project = indexTemplateService.addComponentTemplate(project, false, request.name(), template); } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/ComponentTemplate.java b/server/src/main/java/org/elasticsearch/cluster/metadata/ComponentTemplate.java index 92d2fe3460a86..5e128f8ec58b2 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/ComponentTemplate.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/ComponentTemplate.java @@ -188,12 +188,23 @@ public boolean equals(Object obj) { return false; } ComponentTemplate other = (ComponentTemplate) obj; + return contentEquals(other) + && Objects.equals(createdDateMillis, other.createdDateMillis) + && Objects.equals(modifiedDateMillis, other.modifiedDateMillis); + } + + /** + * Check whether the content of this component template is equal to another component template. Can be used to determine if a template + * already exists. + */ + public boolean contentEquals(ComponentTemplate other) { + if (other == null) { + return false; + } return Objects.equals(template, other.template) && Objects.equals(version, other.version) && Objects.equals(metadata, other.metadata) - && Objects.equals(deprecated, other.deprecated) - && Objects.equals(createdDateMillis, other.createdDateMillis) - && Objects.equals(modifiedDateMillis, other.modifiedDateMillis); + && Objects.equals(deprecated, other.deprecated); } @Override diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java index 4aa8e5dfa551e..18fe8e4f750fd 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java @@ -293,25 +293,26 @@ public ProjectMetadata execute(ProjectMetadata currentProject) throws Exception ); } - // Public visible for testing + /** + * Add the given component template to the project. If {@code create} is true, we will fail if there exists a component template with + * the same name. If a component template with the same name exists, but the content is identical, no change will be made. + * This method will perform all necessary validation but assumes that the component template has already been normalized (see + * {@link #normalizeComponentTemplate(ComponentTemplate)}. + */ public ProjectMetadata addComponentTemplate( final ProjectMetadata project, final boolean create, final String name, final ComponentTemplate template - ) throws Exception { - final ComponentTemplate existing = project.componentTemplates().get(name); - if (create && existing != null) { - throw new IllegalArgumentException("component template [" + name + "] already exists"); - } - - CompressedXContent mappings = template.template().mappings(); - CompressedXContent wrappedMappings = wrapMappingsIfNecessary(mappings, xContentRegistry); - - // We may need to normalize index settings, so do that also - Settings finalSettings = template.template().settings(); - if (finalSettings != null) { - finalSettings = Settings.builder().put(finalSettings).normalizePrefix(IndexMetadata.INDEX_SETTING_PREFIX).build(); + ) throws IOException { + final ComponentTemplate existingTemplate = project.componentTemplates().get(name); + if (existingTemplate != null) { + if (create) { + throw new IllegalArgumentException("component template [" + name + "] already exists"); + } + if (template.contentEquals(existingTemplate)) { + return project; + } } // Collect all the composable (index) templates that use this component template, we'll use @@ -324,9 +325,9 @@ public ProjectMetadata addComponentTemplate( .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); // if we're updating a component template, let's check if it's part of any V2 template that will yield the CT update invalid - if (create == false && finalSettings != null) { + if (create == false && template.template().settings() != null) { // if the CT is specifying the `index.hidden` setting it cannot be part of any global template - if (IndexMetadata.INDEX_HIDDEN_SETTING.exists(finalSettings)) { + if (IndexMetadata.INDEX_HIDDEN_SETTING.exists(template.template().settings())) { List globalTemplatesThatUseThisComponent = new ArrayList<>(); for (Map.Entry entry : templatesUsingComponent.entrySet()) { ComposableIndexTemplate templateV2 = entry.getValue(); @@ -350,47 +351,27 @@ public ProjectMetadata addComponentTemplate( } } - final Template finalTemplate = Template.builder(template.template()).settings(finalSettings).mappings(wrappedMappings).build(); - final long now = instantSource.instant().toEpochMilli(); - final ComponentTemplate finalComponentTemplate; - if (existing == null) { - finalComponentTemplate = new ComponentTemplate( - finalTemplate, - template.version(), - template.metadata(), - template.deprecated(), - now, - now - ); - } else { - final ComponentTemplate templateToCompareToExisting = new ComponentTemplate( - finalTemplate, - template.version(), - template.metadata(), - template.deprecated(), - existing.createdDateMillis().orElse(null), - existing.modifiedDateMillis().orElse(null) - ); - if (templateToCompareToExisting.equals(existing)) { - return project; - } - finalComponentTemplate = new ComponentTemplate( - finalTemplate, - template.version(), - template.metadata(), - template.deprecated(), - existing.createdDateMillis().orElse(null), - now - ); - } + final Long now = instantSource.instant().toEpochMilli(); + final Long createdDateMillis = existingTemplate == null ? now : existingTemplate.createdDateMillis().orElse(null); + final ComponentTemplate finalComponentTemplate = new ComponentTemplate( + template.template(), + template.version(), + template.metadata(), + template.deprecated(), + createdDateMillis, + now + ); - validateTemplate(finalSettings, wrappedMappings, indicesService); + // These two validation checks are only scoped to the component template itself (and don't depend on any other entities in the + // cluster state) and could thus be done in the transport action. However, since we're parsing mappings here, we shouldn't be doing + // it directly on the transport thread. Instead, we should fork to a different threadpool (management/generic). + validateTemplate(finalComponentTemplate.template().settings(), finalComponentTemplate.template().mappings(), indicesService); validate(name, finalComponentTemplate.template(), List.of(), null); ProjectMetadata projectWithComponentTemplateAdded = ProjectMetadata.builder(project).put(name, finalComponentTemplate).build(); // Validate all composable index templates that use this component template if (templatesUsingComponent.isEmpty() == false) { - Exception validationFailure = null; + IllegalArgumentException validationFailure = null; for (Map.Entry entry : templatesUsingComponent.entrySet()) { final String composableTemplateName = entry.getKey(); final ComposableIndexTemplate composableTemplate = entry.getValue(); @@ -424,10 +405,42 @@ public ProjectMetadata addComponentTemplate( .addWarningHeaderIfDataRetentionNotEffective(globalRetentionSettings.get(false), false); } - logger.info("{} component template [{}]", existing == null ? "adding" : "updating", name); + logger.info("{} component template [{}]", existingTemplate == null ? "adding" : "updating", name); return projectWithComponentTemplateAdded; } + /** + * Normalize the given component template by trying to normalize settings and wrapping mappings if necessary. Returns the same instance + * if nothing needs to be done. + */ + public ComponentTemplate normalizeComponentTemplate(final ComponentTemplate componentTemplate) throws IOException { + Template template = componentTemplate.template(); + // Normalize the index settings if necessary + Settings prefixedSettings = null; + if (template.settings() != null) { + prefixedSettings = template.settings().maybeNormalizePrefix(IndexMetadata.INDEX_SETTING_PREFIX); + } + // TODO: theoretically, we could avoid parsing the mappings once by combining this wrapping with the mapping validation later on, + // but that refactoring will be non-trivial as we currently don't seem to have methods available to merge already-parsed mappings; + // we only allow merging mappings from CompressedXContent. + CompressedXContent wrappedMappings = MetadataIndexTemplateService.wrapMappingsIfNecessary(template.mappings(), xContentRegistry); + + // No need to build a new component template if we didn't change anything. + // We can check for reference equality since `maybeNormalizePrefix` and `wrapMappingsIfNecessary` return the same instance if + // nothing needs to be done. + if (prefixedSettings == template.settings() && wrappedMappings == template.mappings()) { + return componentTemplate; + } + return new ComponentTemplate( + Template.builder(template).settings(prefixedSettings).mappings(wrappedMappings).build(), + componentTemplate.version(), + componentTemplate.metadata(), + componentTemplate.deprecated(), + componentTemplate.createdDateMillis().orElse(null), + componentTemplate.modifiedDateMillis().orElse(null) + ); + } + /** * Mappings in templates don't have to include _doc, so update the mappings to include this single type if necessary * @@ -2009,7 +2022,7 @@ private static void validateCompositeTemplate( } public static void validateTemplate(Settings validateSettings, CompressedXContent mappings, IndicesService indicesService) - throws Exception { + throws IOException { // Hard to validate settings if they're non-existent, so used empty ones if none were provided Settings settings = validateSettings; if (settings == null) { diff --git a/server/src/main/java/org/elasticsearch/common/settings/Settings.java b/server/src/main/java/org/elasticsearch/common/settings/Settings.java index 8508329165105..321661e3b31ef 100644 --- a/server/src/main/java/org/elasticsearch/common/settings/Settings.java +++ b/server/src/main/java/org/elasticsearch/common/settings/Settings.java @@ -898,6 +898,19 @@ public Settings merge(Settings newSettings) { return builder.build(); } + /** + * Checks if all settings start with the specified prefix and renames any that do not. Returns the current instance if nothing needs + * to be done. See {@link Builder#normalizePrefix(String)} for more info. + */ + public Settings maybeNormalizePrefix(String prefix) { + for (String key : settings.keySet()) { + if (key.startsWith(prefix) == false && key.endsWith("*") == false) { + return builder().put(this).normalizePrefix(prefix).build(); + } + } + return this; + } + /** * A builder allowing to put different settings and then {@link #build()} an immutable * settings implementation. Use {@link Settings#builder()} in order to diff --git a/server/src/main/java/org/elasticsearch/node/NodeConstruction.java b/server/src/main/java/org/elasticsearch/node/NodeConstruction.java index 548ee6f4da22e..e312a1d03b589 100644 --- a/server/src/main/java/org/elasticsearch/node/NodeConstruction.java +++ b/server/src/main/java/org/elasticsearch/node/NodeConstruction.java @@ -1086,7 +1086,7 @@ public Map queryFields() { clusterService, rerouteService, buildReservedClusterStateHandlers(reservedStateHandlerProviders, settingsModule), - buildReservedProjectStateHandlers(reservedStateHandlerProviders, settingsModule, metadataIndexTemplateService), + buildReservedProjectStateHandlers(reservedStateHandlerProviders, metadataIndexTemplateService), pluginsService.loadSingletonServiceProvider(RestExtension.class, RestExtension::allowAll), incrementalBulkService, projectResolver @@ -1687,12 +1687,11 @@ private List> buildReservedClusterStateHandlers( private List> buildReservedProjectStateHandlers( List handlers, - SettingsModule settingsModule, MetadataIndexTemplateService templateService ) { List> reservedStateHandlers = new ArrayList<>(); - reservedStateHandlers.add(new ReservedComposableIndexTemplateAction(templateService, settingsModule.getIndexScopedSettings())); + reservedStateHandlers.add(new ReservedComposableIndexTemplateAction(templateService)); // add all reserved state handlers from plugins handlers.forEach(h -> reservedStateHandlers.addAll(h.projectHandlers())); diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/template/reservedstate/ReservedComposableIndexTemplateActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/template/reservedstate/ReservedComposableIndexTemplateActionTests.java index 828f3147d0306..01f142a4ede95 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/template/reservedstate/ReservedComposableIndexTemplateActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/template/reservedstate/ReservedComposableIndexTemplateActionTests.java @@ -53,6 +53,7 @@ import org.elasticsearch.xcontent.XContentParserConfiguration; import org.elasticsearch.xcontent.XContentType; import org.junit.Before; +import org.mockito.Mockito; import java.io.IOException; import java.util.Collections; @@ -83,6 +84,7 @@ public class ReservedComposableIndexTemplateActionTests extends ESTestCase { private IndicesService indicesService; private DataStreamGlobalRetentionSettings globalRetentionSettings; private ProjectId projectId; + private NamedXContentRegistry xContentRegistry; @Before public void setup() throws IOException { @@ -98,12 +100,13 @@ public void setup() throws IOException { doReturn(indexService).when(indicesService).createIndex(any(), any(), anyBoolean()); globalRetentionSettings = DataStreamGlobalRetentionSettings.create(ClusterSettings.createBuiltInClusterSettings()); + xContentRegistry = Mockito.mock(NamedXContentRegistry.class); templateService = new MetadataIndexTemplateService( mock(ClusterService.class), mock(MetadataCreateIndexService.class), indicesService, indexScopedSettings, - mock(NamedXContentRegistry.class), + xContentRegistry, mock(SystemIndices.class), new IndexSettingProviders(Set.of()), globalRetentionSettings @@ -122,7 +125,7 @@ private TransformState processJSON( public void testComponentValidation() { TransformState prevState = transformState(); - var action = new ReservedComposableIndexTemplateAction(templateService, indexScopedSettings); + var action = new ReservedComposableIndexTemplateAction(templateService); String badComponentJSON = """ { @@ -149,7 +152,7 @@ public void testComponentValidation() { public void testComposableIndexValidation() { TransformState prevState = transformState(); - var action = new ReservedComposableIndexTemplateAction(templateService, indexScopedSettings); + var action = new ReservedComposableIndexTemplateAction(templateService); String badComponentJSON = """ { @@ -239,7 +242,7 @@ public void testComposableIndexValidation() { public void testAddRemoveComponentTemplates() throws Exception { TransformState prevState = transformState(); - var action = new ReservedComposableIndexTemplateAction(templateService, indexScopedSettings); + var action = new ReservedComposableIndexTemplateAction(templateService); String emptyJSON = ""; @@ -312,7 +315,7 @@ public void testAddRemoveComponentTemplates() throws Exception { public void testAddRemoveIndexTemplates() throws Exception { TransformState prevState = transformState(); - var action = new ReservedComposableIndexTemplateAction(templateService, indexScopedSettings); + var action = new ReservedComposableIndexTemplateAction(templateService); String emptyJSON = ""; @@ -502,7 +505,7 @@ public void testAddRemoveIndexTemplates() throws Exception { public void testAddRemoveIndexTemplatesWithOverlap() throws Exception { TransformState prevState = transformState(); - var action = new ReservedComposableIndexTemplateAction(templateService, indexScopedSettings); + var action = new ReservedComposableIndexTemplateAction(templateService); String emptyJSON = ""; @@ -708,7 +711,6 @@ public void testHandlerCorrectness() { threadPool, null, mock(ActionFilters.class), - indexScopedSettings, TestProjectResolvers.alwaysThrow() ); assertEquals(ReservedComposableIndexTemplateAction.NAME, putComponentAction.reservedStateHandlerName().get()); @@ -734,7 +736,7 @@ public void testHandlerCorrectness() { public void testBlockUsingReservedComponentTemplates() throws Exception { TransformState prevState = transformState(); - var action = new ReservedComposableIndexTemplateAction(templateService, indexScopedSettings); + var action = new ReservedComposableIndexTemplateAction(templateService); String settingsJSON = """ { @@ -895,7 +897,7 @@ public void testTemplatesWithReservedPrefix() throws Exception { mock(MetadataCreateIndexService.class), indicesService, indexScopedSettings, - mock(NamedXContentRegistry.class), + xContentRegistry, mock(SystemIndices.class), new IndexSettingProviders(Set.of()), globalRetentionSettings @@ -910,7 +912,7 @@ public void testTemplatesWithReservedPrefix() throws Exception { assertThat(project.templatesV2(), allOf(aMapWithSize(1), hasKey(reservedComposableIndexName(conflictingTemplateName)))); TransformState prevState = transformState(project); - var action = new ReservedComposableIndexTemplateAction(mockedTemplateService, indexScopedSettings); + var action = new ReservedComposableIndexTemplateAction(mockedTemplateService); TransformState updatedState = processJSON(action, prevState, composableTemplate); diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java index 0da788edb296e..a94b92527bc53 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java @@ -30,7 +30,6 @@ import org.elasticsearch.index.IndexSettingProviders; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.mapper.MapperParsingException; -import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.indices.EmptySystemIndices; import org.elasticsearch.indices.IndexTemplateMissingException; import org.elasticsearch.indices.IndicesService; @@ -61,7 +60,6 @@ import java.util.concurrent.atomic.AtomicReference; import static java.util.Collections.singletonList; -import static org.elasticsearch.cluster.metadata.MetadataIndexTemplateService.DEFAULT_TIMESTAMP_FIELD; import static org.elasticsearch.cluster.metadata.MetadataIndexTemplateService.innerRemoveComponentTemplate; import static org.elasticsearch.common.settings.Settings.builder; import static org.elasticsearch.indices.ShardLimitValidatorTests.createTestShardLimitService; @@ -1061,406 +1059,6 @@ public void testFindV2InvalidGlobalTemplate() { } } - public void testResolveConflictingMappings() throws Exception { - MetadataIndexTemplateService service = getMetadataIndexTemplateService(); - ProjectMetadata project = ProjectMetadata.builder(randomProjectIdOrDefault()).build(); - - ComponentTemplate ct1 = new ComponentTemplate(new Template(null, new CompressedXContent(""" - { - "properties": { - "field2": { - "type": "keyword" - } - } - }"""), null), null, null); - ComponentTemplate ct2 = new ComponentTemplate(new Template(null, new CompressedXContent(""" - { - "properties": { - "field2": { - "type": "text" - } - } - }"""), null), null, null); - project = service.addComponentTemplate(project, true, "ct_high", ct1); - project = service.addComponentTemplate(project, true, "ct_low", ct2); - ComposableIndexTemplate it = ComposableIndexTemplate.builder() - .indexPatterns(List.of("i*")) - .template(new Template(null, new CompressedXContent(""" - { - "properties": { - "field": { - "type": "keyword" - } - } - }"""), null)) - .componentTemplates(List.of("ct_low", "ct_high")) - .priority(0L) - .version(1L) - .build(); - project = service.addIndexTemplateV2(project, true, "my-template", it); - - List mappings = MetadataIndexTemplateService.collectMappings(project, "my-template", "my-index"); - - assertNotNull(mappings); - assertThat(mappings.size(), equalTo(3)); - List> parsedMappings = mappings.stream().map(m -> { - try { - return MapperService.parseMapping(NamedXContentRegistry.EMPTY, m); - } catch (Exception e) { - logger.error(e); - fail("failed to parse mappings: " + m.string()); - return null; - } - }).toList(); - - // The order of mappings should be: - // - ct_low - // - ct_high - // - index template - // Because the first elements when merging mappings have the lowest precedence - assertThat(parsedMappings.get(0), equalTo(Map.of("_doc", Map.of("properties", Map.of("field2", Map.of("type", "text")))))); - assertThat(parsedMappings.get(1), equalTo(Map.of("_doc", Map.of("properties", Map.of("field2", Map.of("type", "keyword")))))); - assertThat(parsedMappings.get(2), equalTo(Map.of("_doc", Map.of("properties", Map.of("field", Map.of("type", "keyword")))))); - } - - public void testResolveMappings() throws Exception { - MetadataIndexTemplateService service = getMetadataIndexTemplateService(); - ProjectMetadata project = ProjectMetadata.builder(randomProjectIdOrDefault()).build(); - - ComponentTemplate ct1 = new ComponentTemplate(new Template(null, new CompressedXContent(""" - { - "properties": { - "field1": { - "type": "keyword" - } - } - }"""), null), null, null); - ComponentTemplate ct2 = new ComponentTemplate(new Template(null, new CompressedXContent(""" - { - "properties": { - "field2": { - "type": "text" - } - } - }"""), null), null, null); - project = service.addComponentTemplate(project, true, "ct_high", ct1); - project = service.addComponentTemplate(project, true, "ct_low", ct2); - ComposableIndexTemplate it = ComposableIndexTemplate.builder() - .indexPatterns(List.of("i*")) - .template(new Template(null, new CompressedXContent(""" - { - "properties": { - "field3": { - "type": "integer" - } - } - }"""), null)) - .componentTemplates(List.of("ct_low", "ct_high")) - .priority(0L) - .version(1L) - .build(); - project = service.addIndexTemplateV2(project, true, "my-template", it); - - List mappings = MetadataIndexTemplateService.collectMappings(project, "my-template", "my-index"); - - assertNotNull(mappings); - assertThat(mappings.size(), equalTo(3)); - List> parsedMappings = mappings.stream().map(m -> { - try { - return MapperService.parseMapping(NamedXContentRegistry.EMPTY, m); - } catch (Exception e) { - logger.error(e); - fail("failed to parse mappings: " + m.string()); - return null; - } - }).toList(); - assertThat(parsedMappings.get(0), equalTo(Map.of("_doc", Map.of("properties", Map.of("field2", Map.of("type", "text")))))); - assertThat(parsedMappings.get(1), equalTo(Map.of("_doc", Map.of("properties", Map.of("field1", Map.of("type", "keyword")))))); - assertThat(parsedMappings.get(2), equalTo(Map.of("_doc", Map.of("properties", Map.of("field3", Map.of("type", "integer")))))); - } - - public void testDefinedTimestampMappingIsAddedForDataStreamTemplates() throws Exception { - MetadataIndexTemplateService service = getMetadataIndexTemplateService(); - ProjectMetadata project = ProjectMetadata.builder(randomProjectIdOrDefault()).build(); - - ComponentTemplate ct1 = new ComponentTemplate(new Template(null, new CompressedXContent(""" - { - "properties": { - "field1": { - "type": "keyword" - } - } - }"""), null), null, null); - - project = service.addComponentTemplate(project, true, "ct1", ct1); - - { - ComposableIndexTemplate it = ComposableIndexTemplate.builder() - .indexPatterns(List.of("logs*")) - .template(new Template(null, new CompressedXContent(""" - { - "properties": { - "field2": { - "type": "integer" - } - } - }"""), null)) - .componentTemplates(List.of("ct1")) - .priority(0L) - .version(1L) - .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate()) - .build(); - project = service.addIndexTemplateV2(project, true, "logs-data-stream-template", it); - - List mappings = MetadataIndexTemplateService.collectMappings( - project, - "logs-data-stream-template", - DataStream.getDefaultBackingIndexName("logs", 1L) - ); - - assertNotNull(mappings); - assertThat(mappings.size(), equalTo(4)); - List> parsedMappings = mappings.stream().map(m -> { - try { - return MapperService.parseMapping(NamedXContentRegistry.EMPTY, m); - } catch (Exception e) { - logger.error(e); - fail("failed to parse mappings: " + m.string()); - return null; - } - }).toList(); - - assertThat( - parsedMappings.get(0), - equalTo( - Map.of( - "_doc", - Map.of( - "properties", - Map.of(DEFAULT_TIMESTAMP_FIELD, Map.of("type", "date", "ignore_malformed", "false")), - "_routing", - Map.of("required", false) - ) - ) - ) - ); - assertThat(parsedMappings.get(1), equalTo(Map.of("_doc", Map.of("properties", Map.of("field1", Map.of("type", "keyword")))))); - assertThat(parsedMappings.get(2), equalTo(Map.of("_doc", Map.of("properties", Map.of("field2", Map.of("type", "integer")))))); - } - - { - // indices matched by templates without the data stream field defined don't get the default @timestamp mapping - ComposableIndexTemplate it = ComposableIndexTemplate.builder() - .indexPatterns(List.of("timeseries*")) - .template(new Template(null, new CompressedXContent(""" - { - "properties": { - "field2": { - "type": "integer" - } - } - }"""), null)) - .componentTemplates(List.of("ct1")) - .priority(0L) - .version(1L) - .build(); - project = service.addIndexTemplateV2(project, true, "timeseries-template", it); - - List mappings = MetadataIndexTemplateService.collectMappings(project, "timeseries-template", "timeseries"); - - assertNotNull(mappings); - assertThat(mappings.size(), equalTo(2)); - List> parsedMappings = mappings.stream().map(m -> { - try { - return MapperService.parseMapping(NamedXContentRegistry.EMPTY, m); - } catch (Exception e) { - logger.error(e); - fail("failed to parse mappings: " + m.string()); - return null; - } - }).toList(); - - assertThat(parsedMappings.get(0), equalTo(Map.of("_doc", Map.of("properties", Map.of("field1", Map.of("type", "keyword")))))); - assertThat(parsedMappings.get(1), equalTo(Map.of("_doc", Map.of("properties", Map.of("field2", Map.of("type", "integer")))))); - - // a default @timestamp mapping will not be added if the matching template doesn't have the data stream field configured, even - // if the index name matches that of a data stream backing index - mappings = MetadataIndexTemplateService.collectMappings( - project, - "timeseries-template", - DataStream.getDefaultBackingIndexName("timeseries", 1L) - ); - - assertNotNull(mappings); - assertThat(mappings.size(), equalTo(2)); - parsedMappings = mappings.stream().map(m -> { - try { - return MapperService.parseMapping(NamedXContentRegistry.EMPTY, m); - } catch (Exception e) { - logger.error(e); - fail("failed to parse mappings: " + m.string()); - return null; - } - }).toList(); - - assertThat(parsedMappings.get(0), equalTo(Map.of("_doc", Map.of("properties", Map.of("field1", Map.of("type", "keyword")))))); - assertThat(parsedMappings.get(1), equalTo(Map.of("_doc", Map.of("properties", Map.of("field2", Map.of("type", "integer")))))); - } - } - - public void testUserDefinedMappingTakesPrecedenceOverDefault() throws Exception { - MetadataIndexTemplateService service = getMetadataIndexTemplateService(); - ProjectMetadata project = ProjectMetadata.builder(randomProjectIdOrDefault()).build(); - - { - // user defines a @timestamp mapping as part of a component template - ComponentTemplate ct1 = new ComponentTemplate(new Template(null, new CompressedXContent(""" - { - "properties": { - "@timestamp": { - "type": "date_nanos" - } - } - }"""), null), null, null); - - project = service.addComponentTemplate(project, true, "ct1", ct1); - ComposableIndexTemplate it = ComposableIndexTemplate.builder() - .indexPatterns(List.of("logs*")) - .componentTemplates(List.of("ct1")) - .priority(0L) - .version(1L) - .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate()) - .build(); - project = service.addIndexTemplateV2(project, true, "logs-template", it); - - List mappings = MetadataIndexTemplateService.collectMappings( - project, - "logs-template", - DataStream.getDefaultBackingIndexName("logs", 1L) - ); - - assertNotNull(mappings); - assertThat(mappings.size(), equalTo(3)); - List> parsedMappings = mappings.stream().map(m -> { - try { - return MapperService.parseMapping(NamedXContentRegistry.EMPTY, m); - } catch (Exception e) { - logger.error(e); - fail("failed to parse mappings: " + m.string()); - return null; - } - }).toList(); - assertThat( - parsedMappings.get(0), - equalTo( - Map.of( - "_doc", - Map.of( - "properties", - Map.of(DEFAULT_TIMESTAMP_FIELD, Map.of("type", "date", "ignore_malformed", "false")), - "_routing", - Map.of("required", false) - ) - ) - ) - ); - assertThat( - parsedMappings.get(1), - equalTo(Map.of("_doc", Map.of("properties", Map.of(DEFAULT_TIMESTAMP_FIELD, Map.of("type", "date_nanos"))))) - ); - } - - { - // user defines a @timestamp mapping as part of a composable index template - Template template = new Template(null, new CompressedXContent(""" - { - "properties": { - "@timestamp": { - "type": "date_nanos" - } - } - }"""), null); - ComposableIndexTemplate it = ComposableIndexTemplate.builder() - .indexPatterns(List.of("timeseries*")) - .template(template) - .priority(0L) - .version(1L) - .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate()) - .build(); - project = service.addIndexTemplateV2(project, true, "timeseries-template", it); - - List mappings = MetadataIndexTemplateService.collectMappings( - project, - "timeseries-template", - DataStream.getDefaultBackingIndexName("timeseries-template", 1L) - ); - - assertNotNull(mappings); - assertThat(mappings.size(), equalTo(3)); - List> parsedMappings = mappings.stream().map(m -> { - try { - return MapperService.parseMapping(NamedXContentRegistry.EMPTY, m); - } catch (Exception e) { - logger.error(e); - fail("failed to parse mappings: " + m.string()); - return null; - } - }).toList(); - assertThat( - parsedMappings.get(0), - equalTo( - Map.of( - "_doc", - Map.of( - "properties", - Map.of(DEFAULT_TIMESTAMP_FIELD, Map.of("type", "date", "ignore_malformed", "false")), - "_routing", - Map.of("required", false) - ) - ) - ) - ); - assertThat( - parsedMappings.get(1), - equalTo(Map.of("_doc", Map.of("properties", Map.of(DEFAULT_TIMESTAMP_FIELD, Map.of("type", "date_nanos"))))) - ); - } - } - - public void testResolveSettings() throws Exception { - MetadataIndexTemplateService service = getMetadataIndexTemplateService(); - ProjectMetadata project = ProjectMetadata.builder(randomProjectIdOrDefault()).build(); - - ComponentTemplate ct1 = new ComponentTemplate( - new Template(Settings.builder().put("number_of_replicas", 2).put("index.blocks.write", true).build(), null, null), - null, - null - ); - ComponentTemplate ct2 = new ComponentTemplate( - new Template(Settings.builder().put("index.number_of_replicas", 1).put("index.blocks.read", true).build(), null, null), - null, - null - ); - project = service.addComponentTemplate(project, true, "ct_high", ct1); - project = service.addComponentTemplate(project, true, "ct_low", ct2); - ComposableIndexTemplate it = ComposableIndexTemplate.builder() - .indexPatterns(List.of("i*")) - .template( - new Template(Settings.builder().put("index.blocks.write", false).put("index.number_of_shards", 3).build(), null, null) - ) - .componentTemplates(List.of("ct_low", "ct_high")) - .priority(0L) - .version(1L) - .build(); - project = service.addIndexTemplateV2(project, true, "my-template", it); - - Settings settings = MetadataIndexTemplateService.resolveSettings(project, "my-template"); - assertThat(settings.get("index.number_of_replicas"), equalTo("2")); - assertThat(settings.get("index.blocks.write"), equalTo("false")); - assertThat(settings.get("index.blocks.read"), equalTo("true")); - assertThat(settings.get("index.number_of_shards"), equalTo("3")); - } - public void testResolveAliases() throws Exception { MetadataIndexTemplateService service = getMetadataIndexTemplateService(); ProjectMetadata project = ProjectMetadata.builder(randomProjectIdOrDefault()).build(); @@ -2670,111 +2268,6 @@ public void testAddInvalidTemplateIgnoreService() throws Exception { assertThat(e.getMessage(), containsString("missing component templates [fail] that does not exist")); } - public void testComposableTemplateWithSubobjectsFalse() throws Exception { - MetadataIndexTemplateService service = getMetadataIndexTemplateService(); - ProjectMetadata project = ProjectMetadata.builder(randomProjectIdOrDefault()).build(); - - ComponentTemplate subobjects = new ComponentTemplate(new Template(null, new CompressedXContent(""" - { - "subobjects": false - } - """), null), null, null); - - ComponentTemplate fieldMapping = new ComponentTemplate(new Template(null, new CompressedXContent(""" - { - "properties": { - "parent.subfield": { - "type": "keyword" - } - } - } - """), null), null, null); - - project = service.addComponentTemplate(project, true, "subobjects", subobjects); - project = service.addComponentTemplate(project, true, "field_mapping", fieldMapping); - ComposableIndexTemplate it = ComposableIndexTemplate.builder() - .indexPatterns(List.of("test-*")) - .template(new Template(null, null, null)) - .componentTemplates(List.of("subobjects", "field_mapping")) - .priority(0L) - .version(1L) - .build(); - project = service.addIndexTemplateV2(project, true, "composable-template", it); - - List mappings = MetadataIndexTemplateService.collectMappings(project, "composable-template", "test-index"); - - assertNotNull(mappings); - assertThat(mappings.size(), equalTo(2)); - List> parsedMappings = mappings.stream().map(m -> { - try { - return MapperService.parseMapping(NamedXContentRegistry.EMPTY, m); - } catch (Exception e) { - logger.error(e); - fail("failed to parse mappings: " + m.string()); - return null; - } - }).toList(); - - assertThat(parsedMappings.get(0), equalTo(Map.of("_doc", Map.of("subobjects", false)))); - assertThat( - parsedMappings.get(1), - equalTo(Map.of("_doc", Map.of("properties", Map.of("parent.subfield", Map.of("type", "keyword"))))) - ); - } - - public void testComposableTemplateWithSubobjectsFalseObjectAndSubfield() throws Exception { - MetadataIndexTemplateService service = getMetadataIndexTemplateService(); - ProjectMetadata project = ProjectMetadata.builder(randomProjectIdOrDefault()).build(); - - ComponentTemplate subobjects = new ComponentTemplate(new Template(null, new CompressedXContent(""" - { - "properties": { - "foo": { - "type": "object", - "subobjects": false - }, - "foo.bar": { - "type": "keyword" - } - } - } - """), null), null, null); - - project = service.addComponentTemplate(project, true, "subobjects", subobjects); - ComposableIndexTemplate it = ComposableIndexTemplate.builder() - .indexPatterns(List.of("test-*")) - .template(new Template(null, null, null)) - .componentTemplates(List.of("subobjects", "field_mapping")) - .priority(0L) - .version(1L) - .build(); - project = service.addIndexTemplateV2(project, true, "composable-template", it); - - List mappings = MetadataIndexTemplateService.collectMappings(project, "composable-template", "test-index"); - - assertNotNull(mappings); - assertThat(mappings.size(), equalTo(1)); - List> parsedMappings = mappings.stream().map(m -> { - try { - return MapperService.parseMapping(NamedXContentRegistry.EMPTY, m); - } catch (Exception e) { - logger.error(e); - fail("failed to parse mappings: " + m.string()); - return null; - } - }).toList(); - - assertThat( - parsedMappings.get(0), - equalTo( - Map.of( - "_doc", - Map.of("properties", Map.of("foo.bar", Map.of("type", "keyword"), "foo", Map.of("type", "object", "subobjects", false))) - ) - ) - ); - } - public void testAddIndexTemplateWithDeprecatedComponentTemplate() throws Exception { ProjectMetadata project = ProjectMetadata.builder(randomProjectIdOrDefault()).build(); MetadataIndexTemplateService service = getMetadataIndexTemplateService(); From 550a5bdb65da6f2f3a6a6a39466cf6921f0fed2d Mon Sep 17 00:00:00 2001 From: Niels Bauman <33722607+nielsbauman@users.noreply.github.com> Date: Mon, 29 Sep 2025 14:11:49 -0300 Subject: [PATCH 2/2] Update docs/changelog/135644.yaml --- docs/changelog/135644.yaml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 docs/changelog/135644.yaml diff --git a/docs/changelog/135644.yaml b/docs/changelog/135644.yaml new file mode 100644 index 0000000000000..2ecf60468d2a9 --- /dev/null +++ b/docs/changelog/135644.yaml @@ -0,0 +1,5 @@ +pr: 135644 +summary: Add small optimizations to `PUT _component_template` API +area: Indices APIs +type: enhancement +issues: []