diff --git a/docs/changelog/135644.yaml b/docs/changelog/135644.yaml new file mode 100644 index 0000000000000..2ecf60468d2a9 --- /dev/null +++ b/docs/changelog/135644.yaml @@ -0,0 +1,5 @@ +pr: 135644 +summary: Add small optimizations to `PUT _component_template` API +area: Indices APIs +type: enhancement +issues: [] diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/template/ComposableTemplateIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/template/ComposableTemplateIT.java index e23e342b91d65..8dd8561377d71 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/template/ComposableTemplateIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/template/ComposableTemplateIT.java @@ -13,11 +13,23 @@ import org.elasticsearch.action.admin.indices.template.put.TransportPutComposableIndexTemplateAction; import org.elasticsearch.cluster.metadata.ComponentTemplate; import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; +import org.elasticsearch.cluster.metadata.DataStream; +import org.elasticsearch.cluster.metadata.MetadataIndexTemplateService; +import org.elasticsearch.cluster.metadata.ProjectId; +import org.elasticsearch.cluster.metadata.ProjectMetadata; import org.elasticsearch.cluster.metadata.Template; import org.elasticsearch.common.compress.CompressedXContent; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.xcontent.NamedXContentRegistry; import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ExecutionException; + +import static org.hamcrest.Matchers.equalTo; public class ComposableTemplateIT extends ESIntegTestCase { @@ -32,7 +44,7 @@ public void testComponentTemplatesCanBeUpdatedAfterRestart() throws Exception { } } }"""), null), 3L, Collections.singletonMap("eggplant", "potato")); - client().execute(PutComponentTemplateAction.INSTANCE, new PutComponentTemplateAction.Request("my-ct").componentTemplate(ct)).get(); + addComponentTemplate("my-ct", ct); ComposableIndexTemplate cit = ComposableIndexTemplate.builder() .indexPatterns(Collections.singletonList("coleslaw")) @@ -50,10 +62,7 @@ public void testComponentTemplatesCanBeUpdatedAfterRestart() throws Exception { .version(5L) .metadata(Collections.singletonMap("egg", "bread")) .build(); - client().execute( - TransportPutComposableIndexTemplateAction.TYPE, - new TransportPutComposableIndexTemplateAction.Request("my-it").indexTemplate(cit) - ).get(); + addComposableTemplate("my-it", cit); internalCluster().fullRestart(); ensureGreen(); @@ -67,7 +76,7 @@ public void testComponentTemplatesCanBeUpdatedAfterRestart() throws Exception { } } }"""), null), 3L, Collections.singletonMap("eggplant", "potato")); - client().execute(PutComponentTemplateAction.INSTANCE, new PutComponentTemplateAction.Request("my-ct").componentTemplate(ct2)).get(); + addComponentTemplate("my-ct", ct2); ComposableIndexTemplate cit2 = ComposableIndexTemplate.builder() .indexPatterns(Collections.singletonList("coleslaw")) @@ -85,9 +94,535 @@ public void testComponentTemplatesCanBeUpdatedAfterRestart() throws Exception { .version(5L) .metadata(Collections.singletonMap("egg", "bread")) .build(); + addComposableTemplate("my-it", cit2); + } + + public void testComposableTemplateWithSubobjectsFalseObjectAndSubfield() throws Exception { + ComponentTemplate subobjects = new ComponentTemplate(new Template(null, new CompressedXContent(""" + { + "properties": { + "foo": { + "type": "object", + "subobjects": false + }, + "foo.bar": { + "type": "keyword" + } + } + } + """), null), null, null); + + addComponentTemplate("subobjects", subobjects); + ComposableIndexTemplate it = ComposableIndexTemplate.builder() + .indexPatterns(List.of("test-*")) + .template(new Template(null, null, null)) + .componentTemplates(List.of("subobjects")) + .priority(0L) + .version(1L) + .build(); + addComposableTemplate("composable-template", it); + + ProjectMetadata project = getProjectMetadata(); + List mappings = MetadataIndexTemplateService.collectMappings(project, "composable-template", "test-index"); + + assertNotNull(mappings); + assertThat(mappings.size(), equalTo(1)); + List> parsedMappings = mappings.stream().map(m -> { + try { + return MapperService.parseMapping(NamedXContentRegistry.EMPTY, m); + } catch (Exception e) { + logger.error(e); + fail("failed to parse mappings: " + m.string()); + return null; + } + }).toList(); + + assertThat( + parsedMappings.get(0), + equalTo( + Map.of( + "_doc", + Map.of("properties", Map.of("foo.bar", Map.of("type", "keyword"), "foo", Map.of("type", "object", "subobjects", false))) + ) + ) + ); + } + + public void testComposableTemplateWithSubobjectsFalse() throws Exception { + ComponentTemplate subobjects = new ComponentTemplate(new Template(null, new CompressedXContent(""" + { + "subobjects": false + } + """), null), null, null); + + ComponentTemplate fieldMapping = new ComponentTemplate(new Template(null, new CompressedXContent(""" + { + "properties": { + "parent.subfield": { + "type": "keyword" + } + } + } + """), null), null, null); + + addComponentTemplate("subobjects", subobjects); + addComponentTemplate("field_mapping", fieldMapping); + ComposableIndexTemplate it = ComposableIndexTemplate.builder() + .indexPatterns(List.of("test-*")) + .template(new Template(null, null, null)) + .componentTemplates(List.of("subobjects", "field_mapping")) + .priority(0L) + .version(1L) + .build(); + addComposableTemplate("composable-template", it); + + ProjectMetadata project = getProjectMetadata(); + List mappings = MetadataIndexTemplateService.collectMappings(project, "composable-template", "test-index"); + + assertNotNull(mappings); + assertThat(mappings.size(), equalTo(2)); + List> parsedMappings = mappings.stream().map(m -> { + try { + return MapperService.parseMapping(NamedXContentRegistry.EMPTY, m); + } catch (Exception e) { + logger.error(e); + fail("failed to parse mappings: " + m.string()); + return null; + } + }).toList(); + + assertThat(parsedMappings.get(0), equalTo(Map.of("_doc", Map.of("subobjects", false)))); + assertThat( + parsedMappings.get(1), + equalTo(Map.of("_doc", Map.of("properties", Map.of("parent.subfield", Map.of("type", "keyword"))))) + ); + } + + public void testResolveConflictingMappings() throws Exception { + ComponentTemplate ct1 = new ComponentTemplate(new Template(null, new CompressedXContent(""" + { + "properties": { + "field2": { + "type": "keyword" + } + } + }"""), null), null, null); + ComponentTemplate ct2 = new ComponentTemplate(new Template(null, new CompressedXContent(""" + { + "properties": { + "field2": { + "type": "text" + } + } + }"""), null), null, null); + addComponentTemplate("ct_high", ct1); + addComponentTemplate("ct_low", ct2); + ComposableIndexTemplate it = ComposableIndexTemplate.builder() + .indexPatterns(List.of("i*")) + .template(new Template(null, new CompressedXContent(""" + { + "properties": { + "field": { + "type": "keyword" + } + } + }"""), null)) + .componentTemplates(List.of("ct_low", "ct_high")) + .priority(0L) + .version(1L) + .build(); + addComposableTemplate("my-template", it); + + ProjectMetadata project = getProjectMetadata(); + List mappings = MetadataIndexTemplateService.collectMappings(project, "my-template", "my-index"); + + assertNotNull(mappings); + assertThat(mappings.size(), equalTo(3)); + List> parsedMappings = mappings.stream().map(m -> { + try { + return MapperService.parseMapping(NamedXContentRegistry.EMPTY, m); + } catch (Exception e) { + logger.error(e); + fail("failed to parse mappings: " + m.string()); + return null; + } + }).toList(); + + // The order of mappings should be: + // - ct_low + // - ct_high + // - index template + // Because the first elements when merging mappings have the lowest precedence + assertThat(parsedMappings.get(0), equalTo(Map.of("_doc", Map.of("properties", Map.of("field2", Map.of("type", "text")))))); + assertThat(parsedMappings.get(1), equalTo(Map.of("_doc", Map.of("properties", Map.of("field2", Map.of("type", "keyword")))))); + assertThat(parsedMappings.get(2), equalTo(Map.of("_doc", Map.of("properties", Map.of("field", Map.of("type", "keyword")))))); + } + + public void testResolveMappings() throws Exception { + ComponentTemplate ct1 = new ComponentTemplate(new Template(null, new CompressedXContent(""" + { + "properties": { + "field1": { + "type": "keyword" + } + } + }"""), null), null, null); + ComponentTemplate ct2 = new ComponentTemplate(new Template(null, new CompressedXContent(""" + { + "properties": { + "field2": { + "type": "text" + } + } + }"""), null), null, null); + addComponentTemplate("ct_high", ct1); + addComponentTemplate("ct_low", ct2); + ComposableIndexTemplate it = ComposableIndexTemplate.builder() + .indexPatterns(List.of("i*")) + .template(new Template(null, new CompressedXContent(""" + { + "properties": { + "field3": { + "type": "integer" + } + } + }"""), null)) + .componentTemplates(List.of("ct_low", "ct_high")) + .priority(0L) + .version(1L) + .build(); + addComposableTemplate("my-template", it); + + ProjectMetadata project = getProjectMetadata(); + List mappings = MetadataIndexTemplateService.collectMappings(project, "my-template", "my-index"); + + assertNotNull(mappings); + assertThat(mappings.size(), equalTo(3)); + List> parsedMappings = mappings.stream().map(m -> { + try { + return MapperService.parseMapping(NamedXContentRegistry.EMPTY, m); + } catch (Exception e) { + logger.error(e); + fail("failed to parse mappings: " + m.string()); + return null; + } + }).toList(); + assertThat(parsedMappings.get(0), equalTo(Map.of("_doc", Map.of("properties", Map.of("field2", Map.of("type", "text")))))); + assertThat(parsedMappings.get(1), equalTo(Map.of("_doc", Map.of("properties", Map.of("field1", Map.of("type", "keyword")))))); + assertThat(parsedMappings.get(2), equalTo(Map.of("_doc", Map.of("properties", Map.of("field3", Map.of("type", "integer")))))); + } + + public void testDefinedTimestampMappingIsAddedForDataStreamTemplates() throws Exception { + ComponentTemplate ct1 = new ComponentTemplate(new Template(null, new CompressedXContent(""" + { + "properties": { + "field1": { + "type": "keyword" + } + } + }"""), null), null, null); + + addComponentTemplate("ct1", ct1); + + { + ComposableIndexTemplate it = ComposableIndexTemplate.builder() + .indexPatterns(List.of("logs*")) + .template(new Template(null, new CompressedXContent(""" + { + "properties": { + "field2": { + "type": "integer" + } + } + }"""), null)) + .componentTemplates(List.of("ct1")) + .priority(0L) + .version(1L) + .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate()) + .build(); + addComposableTemplate("logs-data-stream-template", it); + + ProjectMetadata project = getProjectMetadata(); + List mappings = MetadataIndexTemplateService.collectMappings( + project, + "logs-data-stream-template", + DataStream.getDefaultBackingIndexName("logs", 1L) + ); + + assertNotNull(mappings); + assertThat(mappings.size(), equalTo(4)); + List> parsedMappings = mappings.stream().map(m -> { + try { + return MapperService.parseMapping(NamedXContentRegistry.EMPTY, m); + } catch (Exception e) { + logger.error(e); + fail("failed to parse mappings: " + m.string()); + return null; + } + }).toList(); + + assertThat( + parsedMappings.get(0), + equalTo( + Map.of( + "_doc", + Map.of( + "properties", + Map.of( + MetadataIndexTemplateService.DEFAULT_TIMESTAMP_FIELD, + Map.of("type", "date", "ignore_malformed", "false") + ), + "_routing", + Map.of("required", false) + ) + ) + ) + ); + assertThat(parsedMappings.get(1), equalTo(Map.of("_doc", Map.of("properties", Map.of("field1", Map.of("type", "keyword")))))); + assertThat(parsedMappings.get(2), equalTo(Map.of("_doc", Map.of("properties", Map.of("field2", Map.of("type", "integer")))))); + } + + { + // indices matched by templates without the data stream field defined don't get the default @timestamp mapping + ComposableIndexTemplate it = ComposableIndexTemplate.builder() + .indexPatterns(List.of("timeseries*")) + .template(new Template(null, new CompressedXContent(""" + { + "properties": { + "field2": { + "type": "integer" + } + } + }"""), null)) + .componentTemplates(List.of("ct1")) + .priority(0L) + .version(1L) + .build(); + addComposableTemplate("timeseries-template", it); + + ProjectMetadata project = getProjectMetadata(); + List mappings = MetadataIndexTemplateService.collectMappings(project, "timeseries-template", "timeseries"); + + assertNotNull(mappings); + assertThat(mappings.size(), equalTo(2)); + List> parsedMappings = mappings.stream().map(m -> { + try { + return MapperService.parseMapping(NamedXContentRegistry.EMPTY, m); + } catch (Exception e) { + logger.error(e); + fail("failed to parse mappings: " + m.string()); + return null; + } + }).toList(); + + assertThat(parsedMappings.get(0), equalTo(Map.of("_doc", Map.of("properties", Map.of("field1", Map.of("type", "keyword")))))); + assertThat(parsedMappings.get(1), equalTo(Map.of("_doc", Map.of("properties", Map.of("field2", Map.of("type", "integer")))))); + + // a default @timestamp mapping will not be added if the matching template doesn't have the data stream field configured, even + // if the index name matches that of a data stream backing index + mappings = MetadataIndexTemplateService.collectMappings( + project, + "timeseries-template", + DataStream.getDefaultBackingIndexName("timeseries", 1L) + ); + + assertNotNull(mappings); + assertThat(mappings.size(), equalTo(2)); + parsedMappings = mappings.stream().map(m -> { + try { + return MapperService.parseMapping(NamedXContentRegistry.EMPTY, m); + } catch (Exception e) { + logger.error(e); + fail("failed to parse mappings: " + m.string()); + return null; + } + }).toList(); + + assertThat(parsedMappings.get(0), equalTo(Map.of("_doc", Map.of("properties", Map.of("field1", Map.of("type", "keyword")))))); + assertThat(parsedMappings.get(1), equalTo(Map.of("_doc", Map.of("properties", Map.of("field2", Map.of("type", "integer")))))); + } + } + + public void testUserDefinedMappingTakesPrecedenceOverDefault() throws Exception { + { + // user defines a @timestamp mapping as part of a component template + ComponentTemplate ct1 = new ComponentTemplate(new Template(null, new CompressedXContent(""" + { + "properties": { + "@timestamp": { + "type": "date_nanos" + } + } + }"""), null), null, null); + addComponentTemplate("ct1", ct1); + + ComposableIndexTemplate it = ComposableIndexTemplate.builder() + .indexPatterns(List.of("logs*")) + .componentTemplates(List.of("ct1")) + .priority(0L) + .version(1L) + .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate()) + .build(); + addComposableTemplate("logs-template", it); + + ProjectMetadata project = getProjectMetadata(); + List mappings = MetadataIndexTemplateService.collectMappings( + project, + "logs-template", + DataStream.getDefaultBackingIndexName("logs", 1L) + ); + + assertNotNull(mappings); + assertThat(mappings.size(), equalTo(3)); + List> parsedMappings = mappings.stream().map(m -> { + try { + return MapperService.parseMapping(NamedXContentRegistry.EMPTY, m); + } catch (Exception e) { + logger.error(e); + fail("failed to parse mappings: " + m.string()); + return null; + } + }).toList(); + assertThat( + parsedMappings.get(0), + equalTo( + Map.of( + "_doc", + Map.of( + "properties", + Map.of( + MetadataIndexTemplateService.DEFAULT_TIMESTAMP_FIELD, + Map.of("type", "date", "ignore_malformed", "false") + ), + "_routing", + Map.of("required", false) + ) + ) + ) + ); + assertThat( + parsedMappings.get(1), + equalTo( + Map.of( + "_doc", + Map.of("properties", Map.of(MetadataIndexTemplateService.DEFAULT_TIMESTAMP_FIELD, Map.of("type", "date_nanos"))) + ) + ) + ); + } + + { + // user defines a @timestamp mapping as part of a composable index template + Template template = new Template(null, new CompressedXContent(""" + { + "properties": { + "@timestamp": { + "type": "date_nanos" + } + } + }"""), null); + ComposableIndexTemplate it = ComposableIndexTemplate.builder() + .indexPatterns(List.of("timeseries*")) + .template(template) + .priority(0L) + .version(1L) + .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate()) + .build(); + addComposableTemplate("timeseries-template", it); + + ProjectMetadata project = getProjectMetadata(); + List mappings = MetadataIndexTemplateService.collectMappings( + project, + "timeseries-template", + DataStream.getDefaultBackingIndexName("timeseries-template", 1L) + ); + + assertNotNull(mappings); + assertThat(mappings.size(), equalTo(3)); + List> parsedMappings = mappings.stream().map(m -> { + try { + return MapperService.parseMapping(NamedXContentRegistry.EMPTY, m); + } catch (Exception e) { + logger.error(e); + fail("failed to parse mappings: " + m.string()); + return null; + } + }).toList(); + assertThat( + parsedMappings.get(0), + equalTo( + Map.of( + "_doc", + Map.of( + "properties", + Map.of( + MetadataIndexTemplateService.DEFAULT_TIMESTAMP_FIELD, + Map.of("type", "date", "ignore_malformed", "false") + ), + "_routing", + Map.of("required", false) + ) + ) + ) + ); + assertThat( + parsedMappings.get(1), + equalTo( + Map.of( + "_doc", + Map.of("properties", Map.of(MetadataIndexTemplateService.DEFAULT_TIMESTAMP_FIELD, Map.of("type", "date_nanos"))) + ) + ) + ); + } + } + + public void testResolveSettings() throws Exception { + ComponentTemplate ct1 = new ComponentTemplate( + new Template(Settings.builder().put("number_of_replicas", 2).put("index.blocks.write", true).build(), null, null), + null, + null + ); + ComponentTemplate ct2 = new ComponentTemplate( + new Template(Settings.builder().put("index.number_of_replicas", 1).put("index.blocks.read", true).build(), null, null), + null, + null + ); + addComponentTemplate("ct_high", ct1); + addComponentTemplate("ct_low", ct2); + ComposableIndexTemplate it = ComposableIndexTemplate.builder() + .indexPatterns(List.of("i*")) + .template( + new Template(Settings.builder().put("index.blocks.write", false).put("index.number_of_shards", 3).build(), null, null) + ) + .componentTemplates(List.of("ct_low", "ct_high")) + .priority(0L) + .version(1L) + .build(); + addComposableTemplate("my-template", it); + + ProjectMetadata project = getProjectMetadata(); + Settings settings = MetadataIndexTemplateService.resolveSettings(project, "my-template"); + assertThat(settings.get("index.number_of_replicas"), equalTo("2")); + assertThat(settings.get("index.blocks.write"), equalTo("false")); + assertThat(settings.get("index.blocks.read"), equalTo("true")); + assertThat(settings.get("index.number_of_shards"), equalTo("3")); + } + + private static void addComponentTemplate(String name, ComponentTemplate template) throws InterruptedException, ExecutionException { + client().execute(PutComponentTemplateAction.INSTANCE, new PutComponentTemplateAction.Request(name).componentTemplate(template)) + .get(); + } + + private static void addComposableTemplate(String name, ComposableIndexTemplate template) throws InterruptedException, + ExecutionException { client().execute( TransportPutComposableIndexTemplateAction.TYPE, - new TransportPutComposableIndexTemplateAction.Request("my-it").indexTemplate(cit2) + new TransportPutComposableIndexTemplateAction.Request(name).indexTemplate(template) ).get(); } + + private ProjectMetadata getProjectMetadata() { + return client().admin().cluster().prepareState(TEST_REQUEST_TIMEOUT).get().getState().metadata().getProject(ProjectId.DEFAULT); + } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutComponentTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutComponentTemplateAction.java index e7cdf2f7c872f..514dcc507de7c 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutComponentTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutComponentTemplateAction.java @@ -18,14 +18,10 @@ import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.metadata.ComponentTemplate; -import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.MetadataIndexTemplateService; -import org.elasticsearch.cluster.metadata.Template; import org.elasticsearch.cluster.project.ProjectResolver; import org.elasticsearch.cluster.project.ProjectStateRegistry; import org.elasticsearch.cluster.service.ClusterService; -import org.elasticsearch.common.settings.IndexScopedSettings; -import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.FixForMultiProject; import org.elasticsearch.injection.guice.Inject; @@ -39,7 +35,6 @@ public class TransportPutComponentTemplateAction extends AcknowledgedTransportMasterNodeAction { private final MetadataIndexTemplateService indexTemplateService; - private final IndexScopedSettings indexScopedSettings; private final ProjectResolver projectResolver; @Inject @@ -49,7 +44,6 @@ public TransportPutComponentTemplateAction( ThreadPool threadPool, MetadataIndexTemplateService indexTemplateService, ActionFilters actionFilters, - IndexScopedSettings indexScopedSettings, ProjectResolver projectResolver ) { super( @@ -62,7 +56,6 @@ public TransportPutComponentTemplateAction( EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.indexTemplateService = indexTemplateService; - this.indexScopedSettings = indexScopedSettings; this.projectResolver = projectResolver; } @@ -71,46 +64,36 @@ protected ClusterBlockException checkBlock(PutComponentTemplateAction.Request re return state.blocks().globalBlockedException(projectResolver.getProjectId(), ClusterBlockLevel.METADATA_WRITE); } - public static ComponentTemplate normalizeComponentTemplate( - ComponentTemplate componentTemplate, - IndexScopedSettings indexScopedSettings - ) { - Template template = componentTemplate.template(); - // Normalize the index settings if necessary - if (template.settings() != null) { - Settings.Builder builder = Settings.builder().put(template.settings()).normalizePrefix(IndexMetadata.INDEX_SETTING_PREFIX); - Settings settings = builder.build(); - indexScopedSettings.validate(settings, true); - template = Template.builder(template).settings(settings).build(); - componentTemplate = new ComponentTemplate( - template, - componentTemplate.version(), - componentTemplate.metadata(), - componentTemplate.deprecated(), - componentTemplate.createdDateMillis().orElse(null), - componentTemplate.modifiedDateMillis().orElse(null) - ); - } - - return componentTemplate; - } - @Override protected void masterOperation( Task task, final PutComponentTemplateAction.Request request, final ClusterState state, final ActionListener listener - ) { - ComponentTemplate componentTemplate = normalizeComponentTemplate(request.componentTemplate(), indexScopedSettings); - var projectId = projectResolver.getProjectId(); + ) throws Exception { + final var project = projectResolver.getProjectMetadata(state); + final ComponentTemplate componentTemplate = indexTemplateService.normalizeComponentTemplate(request.componentTemplate()); + final ComponentTemplate existingTemplate = project.componentTemplates().get(request.name()); + if (existingTemplate != null) { + if (request.create()) { + listener.onFailure(new IllegalArgumentException("component template [" + request.name() + "] already exists")); + return; + } + // We have an early return here in case the component template already exists and is identical in content. We still need to do + // this check in the cluster state update task in case the cluster state changed since this check. + if (componentTemplate.contentEquals(existingTemplate)) { + listener.onResponse(AcknowledgedResponse.TRUE); + return; + } + } + indexTemplateService.putComponentTemplate( request.cause(), request.create(), request.name(), request.masterNodeTimeout(), componentTemplate, - projectId, + project.id(), listener ); } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/reservedstate/ReservedComposableIndexTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/reservedstate/ReservedComposableIndexTemplateAction.java index a70052d135b42..1654bdca0f847 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/reservedstate/ReservedComposableIndexTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/reservedstate/ReservedComposableIndexTemplateAction.java @@ -18,7 +18,6 @@ import org.elasticsearch.cluster.metadata.MetadataIndexTemplateService; import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.cluster.metadata.ProjectMetadata; -import org.elasticsearch.common.settings.IndexScopedSettings; import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.reservedstate.ReservedClusterStateHandler; import org.elasticsearch.reservedstate.ReservedProjectStateHandler; @@ -56,14 +55,9 @@ public class ReservedComposableIndexTemplateAction public static final String COMPOSABLE_PREFIX = "composable_index_template:"; private final MetadataIndexTemplateService indexTemplateService; - private final IndexScopedSettings indexScopedSettings; - public ReservedComposableIndexTemplateAction( - MetadataIndexTemplateService indexTemplateService, - IndexScopedSettings indexScopedSettings - ) { + public ReservedComposableIndexTemplateAction(MetadataIndexTemplateService indexTemplateService) { this.indexTemplateService = indexTemplateService; - this.indexScopedSettings = indexScopedSettings; } @Override @@ -154,11 +148,7 @@ public TransformState transform(ProjectId projectId, ComponentsAndComposables so // 1. create or update component templates (composable templates depend on them) for (var request : components) { - ComponentTemplate template = TransportPutComponentTemplateAction.normalizeComponentTemplate( - request.componentTemplate(), - indexScopedSettings - ); - + ComponentTemplate template = indexTemplateService.normalizeComponentTemplate(request.componentTemplate()); project = indexTemplateService.addComponentTemplate(project, false, request.name(), template); } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/ComponentTemplate.java b/server/src/main/java/org/elasticsearch/cluster/metadata/ComponentTemplate.java index 92d2fe3460a86..5e128f8ec58b2 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/ComponentTemplate.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/ComponentTemplate.java @@ -188,12 +188,23 @@ public boolean equals(Object obj) { return false; } ComponentTemplate other = (ComponentTemplate) obj; + return contentEquals(other) + && Objects.equals(createdDateMillis, other.createdDateMillis) + && Objects.equals(modifiedDateMillis, other.modifiedDateMillis); + } + + /** + * Check whether the content of this component template is equal to another component template. Can be used to determine if a template + * already exists. + */ + public boolean contentEquals(ComponentTemplate other) { + if (other == null) { + return false; + } return Objects.equals(template, other.template) && Objects.equals(version, other.version) && Objects.equals(metadata, other.metadata) - && Objects.equals(deprecated, other.deprecated) - && Objects.equals(createdDateMillis, other.createdDateMillis) - && Objects.equals(modifiedDateMillis, other.modifiedDateMillis); + && Objects.equals(deprecated, other.deprecated); } @Override diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java index 5c6f5e747886c..82fe246ff3995 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java @@ -294,25 +294,26 @@ public ProjectMetadata execute(ProjectMetadata currentProject) throws Exception ); } - // Public visible for testing + /** + * Add the given component template to the project. If {@code create} is true, we will fail if there exists a component template with + * the same name. If a component template with the same name exists, but the content is identical, no change will be made. + * This method will perform all necessary validation but assumes that the component template has already been normalized (see + * {@link #normalizeComponentTemplate(ComponentTemplate)}. + */ public ProjectMetadata addComponentTemplate( final ProjectMetadata project, final boolean create, final String name, final ComponentTemplate template - ) throws Exception { - final ComponentTemplate existing = project.componentTemplates().get(name); - if (create && existing != null) { - throw new IllegalArgumentException("component template [" + name + "] already exists"); - } - - CompressedXContent mappings = template.template().mappings(); - CompressedXContent wrappedMappings = wrapMappingsIfNecessary(mappings, xContentRegistry); - - // We may need to normalize index settings, so do that also - Settings finalSettings = template.template().settings(); - if (finalSettings != null) { - finalSettings = Settings.builder().put(finalSettings).normalizePrefix(IndexMetadata.INDEX_SETTING_PREFIX).build(); + ) throws IOException { + final ComponentTemplate existingTemplate = project.componentTemplates().get(name); + if (existingTemplate != null) { + if (create) { + throw new IllegalArgumentException("component template [" + name + "] already exists"); + } + if (template.contentEquals(existingTemplate)) { + return project; + } } // Collect all the composable (index) templates that use this component template, we'll use @@ -325,9 +326,9 @@ public ProjectMetadata addComponentTemplate( .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); // if we're updating a component template, let's check if it's part of any V2 template that will yield the CT update invalid - if (create == false && finalSettings != null) { + if (create == false && template.template().settings() != null) { // if the CT is specifying the `index.hidden` setting it cannot be part of any global template - if (IndexMetadata.INDEX_HIDDEN_SETTING.exists(finalSettings)) { + if (IndexMetadata.INDEX_HIDDEN_SETTING.exists(template.template().settings())) { List globalTemplatesThatUseThisComponent = new ArrayList<>(); for (Map.Entry entry : templatesUsingComponent.entrySet()) { ComposableIndexTemplate templateV2 = entry.getValue(); @@ -351,47 +352,27 @@ public ProjectMetadata addComponentTemplate( } } - final Template finalTemplate = Template.builder(template.template()).settings(finalSettings).mappings(wrappedMappings).build(); - final long now = instantSource.instant().toEpochMilli(); - final ComponentTemplate finalComponentTemplate; - if (existing == null) { - finalComponentTemplate = new ComponentTemplate( - finalTemplate, - template.version(), - template.metadata(), - template.deprecated(), - now, - now - ); - } else { - final ComponentTemplate templateToCompareToExisting = new ComponentTemplate( - finalTemplate, - template.version(), - template.metadata(), - template.deprecated(), - existing.createdDateMillis().orElse(null), - existing.modifiedDateMillis().orElse(null) - ); - if (templateToCompareToExisting.equals(existing)) { - return project; - } - finalComponentTemplate = new ComponentTemplate( - finalTemplate, - template.version(), - template.metadata(), - template.deprecated(), - existing.createdDateMillis().orElse(null), - now - ); - } + final Long now = instantSource.instant().toEpochMilli(); + final Long createdDateMillis = existingTemplate == null ? now : existingTemplate.createdDateMillis().orElse(null); + final ComponentTemplate finalComponentTemplate = new ComponentTemplate( + template.template(), + template.version(), + template.metadata(), + template.deprecated(), + createdDateMillis, + now + ); - validateTemplate(finalSettings, wrappedMappings, indicesService); + // These two validation checks are only scoped to the component template itself (and don't depend on any other entities in the + // cluster state) and could thus be done in the transport action. However, since we're parsing mappings here, we shouldn't be doing + // it directly on the transport thread. Instead, we should fork to a different threadpool (management/generic). + validateTemplate(finalComponentTemplate.template().settings(), finalComponentTemplate.template().mappings(), indicesService); validate(name, finalComponentTemplate.template(), List.of(), null); ProjectMetadata projectWithComponentTemplateAdded = ProjectMetadata.builder(project).put(name, finalComponentTemplate).build(); // Validate all composable index templates that use this component template if (templatesUsingComponent.isEmpty() == false) { - Exception validationFailure = null; + IllegalArgumentException validationFailure = null; for (Map.Entry entry : templatesUsingComponent.entrySet()) { final String composableTemplateName = entry.getKey(); final ComposableIndexTemplate composableTemplate = entry.getValue(); @@ -425,10 +406,42 @@ public ProjectMetadata addComponentTemplate( .addWarningHeaderIfDataRetentionNotEffective(globalRetentionSettings.get(false), false); } - logger.info("{} component template [{}]", existing == null ? "adding" : "updating", name); + logger.info("{} component template [{}]", existingTemplate == null ? "adding" : "updating", name); return projectWithComponentTemplateAdded; } + /** + * Normalize the given component template by trying to normalize settings and wrapping mappings if necessary. Returns the same instance + * if nothing needs to be done. + */ + public ComponentTemplate normalizeComponentTemplate(final ComponentTemplate componentTemplate) throws IOException { + Template template = componentTemplate.template(); + // Normalize the index settings if necessary + Settings prefixedSettings = null; + if (template.settings() != null) { + prefixedSettings = template.settings().maybeNormalizePrefix(IndexMetadata.INDEX_SETTING_PREFIX); + } + // TODO: theoretically, we could avoid parsing the mappings once by combining this wrapping with the mapping validation later on, + // but that refactoring will be non-trivial as we currently don't seem to have methods available to merge already-parsed mappings; + // we only allow merging mappings from CompressedXContent. + CompressedXContent wrappedMappings = MetadataIndexTemplateService.wrapMappingsIfNecessary(template.mappings(), xContentRegistry); + + // No need to build a new component template if we didn't change anything. + // We can check for reference equality since `maybeNormalizePrefix` and `wrapMappingsIfNecessary` return the same instance if + // nothing needs to be done. + if (prefixedSettings == template.settings() && wrappedMappings == template.mappings()) { + return componentTemplate; + } + return new ComponentTemplate( + Template.builder(template).settings(prefixedSettings).mappings(wrappedMappings).build(), + componentTemplate.version(), + componentTemplate.metadata(), + componentTemplate.deprecated(), + componentTemplate.createdDateMillis().orElse(null), + componentTemplate.modifiedDateMillis().orElse(null) + ); + } + /** * Mappings in templates don't have to include _doc, so update the mappings to include this single type if necessary * @@ -2048,7 +2061,7 @@ private static void validateCompositeTemplate( } public static void validateTemplate(Settings validateSettings, CompressedXContent mappings, IndicesService indicesService) - throws Exception { + throws IOException { // Hard to validate settings if they're non-existent, so used empty ones if none were provided Settings settings = validateSettings; if (settings == null) { diff --git a/server/src/main/java/org/elasticsearch/common/settings/Settings.java b/server/src/main/java/org/elasticsearch/common/settings/Settings.java index 8508329165105..321661e3b31ef 100644 --- a/server/src/main/java/org/elasticsearch/common/settings/Settings.java +++ b/server/src/main/java/org/elasticsearch/common/settings/Settings.java @@ -898,6 +898,19 @@ public Settings merge(Settings newSettings) { return builder.build(); } + /** + * Checks if all settings start with the specified prefix and renames any that do not. Returns the current instance if nothing needs + * to be done. See {@link Builder#normalizePrefix(String)} for more info. + */ + public Settings maybeNormalizePrefix(String prefix) { + for (String key : settings.keySet()) { + if (key.startsWith(prefix) == false && key.endsWith("*") == false) { + return builder().put(this).normalizePrefix(prefix).build(); + } + } + return this; + } + /** * A builder allowing to put different settings and then {@link #build()} an immutable * settings implementation. Use {@link Settings#builder()} in order to diff --git a/server/src/main/java/org/elasticsearch/node/NodeConstruction.java b/server/src/main/java/org/elasticsearch/node/NodeConstruction.java index f1d5b6cc804bf..201bf648439c5 100644 --- a/server/src/main/java/org/elasticsearch/node/NodeConstruction.java +++ b/server/src/main/java/org/elasticsearch/node/NodeConstruction.java @@ -1096,7 +1096,7 @@ public Map queryFields() { clusterService, rerouteService, buildReservedClusterStateHandlers(reservedStateHandlerProviders, settingsModule), - buildReservedProjectStateHandlers(reservedStateHandlerProviders, settingsModule, metadataIndexTemplateService), + buildReservedProjectStateHandlers(reservedStateHandlerProviders, metadataIndexTemplateService), pluginsService.loadSingletonServiceProvider(RestExtension.class, RestExtension::allowAll), incrementalBulkService, projectResolver @@ -1697,12 +1697,11 @@ private List> buildReservedClusterStateHandlers( private List> buildReservedProjectStateHandlers( List handlers, - SettingsModule settingsModule, MetadataIndexTemplateService templateService ) { List> reservedStateHandlers = new ArrayList<>(); - reservedStateHandlers.add(new ReservedComposableIndexTemplateAction(templateService, settingsModule.getIndexScopedSettings())); + reservedStateHandlers.add(new ReservedComposableIndexTemplateAction(templateService)); // add all reserved state handlers from plugins handlers.forEach(h -> reservedStateHandlers.addAll(h.projectHandlers())); diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/template/reservedstate/ReservedComposableIndexTemplateActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/template/reservedstate/ReservedComposableIndexTemplateActionTests.java index 828f3147d0306..01f142a4ede95 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/template/reservedstate/ReservedComposableIndexTemplateActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/template/reservedstate/ReservedComposableIndexTemplateActionTests.java @@ -53,6 +53,7 @@ import org.elasticsearch.xcontent.XContentParserConfiguration; import org.elasticsearch.xcontent.XContentType; import org.junit.Before; +import org.mockito.Mockito; import java.io.IOException; import java.util.Collections; @@ -83,6 +84,7 @@ public class ReservedComposableIndexTemplateActionTests extends ESTestCase { private IndicesService indicesService; private DataStreamGlobalRetentionSettings globalRetentionSettings; private ProjectId projectId; + private NamedXContentRegistry xContentRegistry; @Before public void setup() throws IOException { @@ -98,12 +100,13 @@ public void setup() throws IOException { doReturn(indexService).when(indicesService).createIndex(any(), any(), anyBoolean()); globalRetentionSettings = DataStreamGlobalRetentionSettings.create(ClusterSettings.createBuiltInClusterSettings()); + xContentRegistry = Mockito.mock(NamedXContentRegistry.class); templateService = new MetadataIndexTemplateService( mock(ClusterService.class), mock(MetadataCreateIndexService.class), indicesService, indexScopedSettings, - mock(NamedXContentRegistry.class), + xContentRegistry, mock(SystemIndices.class), new IndexSettingProviders(Set.of()), globalRetentionSettings @@ -122,7 +125,7 @@ private TransformState processJSON( public void testComponentValidation() { TransformState prevState = transformState(); - var action = new ReservedComposableIndexTemplateAction(templateService, indexScopedSettings); + var action = new ReservedComposableIndexTemplateAction(templateService); String badComponentJSON = """ { @@ -149,7 +152,7 @@ public void testComponentValidation() { public void testComposableIndexValidation() { TransformState prevState = transformState(); - var action = new ReservedComposableIndexTemplateAction(templateService, indexScopedSettings); + var action = new ReservedComposableIndexTemplateAction(templateService); String badComponentJSON = """ { @@ -239,7 +242,7 @@ public void testComposableIndexValidation() { public void testAddRemoveComponentTemplates() throws Exception { TransformState prevState = transformState(); - var action = new ReservedComposableIndexTemplateAction(templateService, indexScopedSettings); + var action = new ReservedComposableIndexTemplateAction(templateService); String emptyJSON = ""; @@ -312,7 +315,7 @@ public void testAddRemoveComponentTemplates() throws Exception { public void testAddRemoveIndexTemplates() throws Exception { TransformState prevState = transformState(); - var action = new ReservedComposableIndexTemplateAction(templateService, indexScopedSettings); + var action = new ReservedComposableIndexTemplateAction(templateService); String emptyJSON = ""; @@ -502,7 +505,7 @@ public void testAddRemoveIndexTemplates() throws Exception { public void testAddRemoveIndexTemplatesWithOverlap() throws Exception { TransformState prevState = transformState(); - var action = new ReservedComposableIndexTemplateAction(templateService, indexScopedSettings); + var action = new ReservedComposableIndexTemplateAction(templateService); String emptyJSON = ""; @@ -708,7 +711,6 @@ public void testHandlerCorrectness() { threadPool, null, mock(ActionFilters.class), - indexScopedSettings, TestProjectResolvers.alwaysThrow() ); assertEquals(ReservedComposableIndexTemplateAction.NAME, putComponentAction.reservedStateHandlerName().get()); @@ -734,7 +736,7 @@ public void testHandlerCorrectness() { public void testBlockUsingReservedComponentTemplates() throws Exception { TransformState prevState = transformState(); - var action = new ReservedComposableIndexTemplateAction(templateService, indexScopedSettings); + var action = new ReservedComposableIndexTemplateAction(templateService); String settingsJSON = """ { @@ -895,7 +897,7 @@ public void testTemplatesWithReservedPrefix() throws Exception { mock(MetadataCreateIndexService.class), indicesService, indexScopedSettings, - mock(NamedXContentRegistry.class), + xContentRegistry, mock(SystemIndices.class), new IndexSettingProviders(Set.of()), globalRetentionSettings @@ -910,7 +912,7 @@ public void testTemplatesWithReservedPrefix() throws Exception { assertThat(project.templatesV2(), allOf(aMapWithSize(1), hasKey(reservedComposableIndexName(conflictingTemplateName)))); TransformState prevState = transformState(project); - var action = new ReservedComposableIndexTemplateAction(mockedTemplateService, indexScopedSettings); + var action = new ReservedComposableIndexTemplateAction(mockedTemplateService); TransformState updatedState = processJSON(action, prevState, composableTemplate); diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java index 0da788edb296e..a94b92527bc53 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java @@ -30,7 +30,6 @@ import org.elasticsearch.index.IndexSettingProviders; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.mapper.MapperParsingException; -import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.indices.EmptySystemIndices; import org.elasticsearch.indices.IndexTemplateMissingException; import org.elasticsearch.indices.IndicesService; @@ -61,7 +60,6 @@ import java.util.concurrent.atomic.AtomicReference; import static java.util.Collections.singletonList; -import static org.elasticsearch.cluster.metadata.MetadataIndexTemplateService.DEFAULT_TIMESTAMP_FIELD; import static org.elasticsearch.cluster.metadata.MetadataIndexTemplateService.innerRemoveComponentTemplate; import static org.elasticsearch.common.settings.Settings.builder; import static org.elasticsearch.indices.ShardLimitValidatorTests.createTestShardLimitService; @@ -1061,406 +1059,6 @@ public void testFindV2InvalidGlobalTemplate() { } } - public void testResolveConflictingMappings() throws Exception { - MetadataIndexTemplateService service = getMetadataIndexTemplateService(); - ProjectMetadata project = ProjectMetadata.builder(randomProjectIdOrDefault()).build(); - - ComponentTemplate ct1 = new ComponentTemplate(new Template(null, new CompressedXContent(""" - { - "properties": { - "field2": { - "type": "keyword" - } - } - }"""), null), null, null); - ComponentTemplate ct2 = new ComponentTemplate(new Template(null, new CompressedXContent(""" - { - "properties": { - "field2": { - "type": "text" - } - } - }"""), null), null, null); - project = service.addComponentTemplate(project, true, "ct_high", ct1); - project = service.addComponentTemplate(project, true, "ct_low", ct2); - ComposableIndexTemplate it = ComposableIndexTemplate.builder() - .indexPatterns(List.of("i*")) - .template(new Template(null, new CompressedXContent(""" - { - "properties": { - "field": { - "type": "keyword" - } - } - }"""), null)) - .componentTemplates(List.of("ct_low", "ct_high")) - .priority(0L) - .version(1L) - .build(); - project = service.addIndexTemplateV2(project, true, "my-template", it); - - List mappings = MetadataIndexTemplateService.collectMappings(project, "my-template", "my-index"); - - assertNotNull(mappings); - assertThat(mappings.size(), equalTo(3)); - List> parsedMappings = mappings.stream().map(m -> { - try { - return MapperService.parseMapping(NamedXContentRegistry.EMPTY, m); - } catch (Exception e) { - logger.error(e); - fail("failed to parse mappings: " + m.string()); - return null; - } - }).toList(); - - // The order of mappings should be: - // - ct_low - // - ct_high - // - index template - // Because the first elements when merging mappings have the lowest precedence - assertThat(parsedMappings.get(0), equalTo(Map.of("_doc", Map.of("properties", Map.of("field2", Map.of("type", "text")))))); - assertThat(parsedMappings.get(1), equalTo(Map.of("_doc", Map.of("properties", Map.of("field2", Map.of("type", "keyword")))))); - assertThat(parsedMappings.get(2), equalTo(Map.of("_doc", Map.of("properties", Map.of("field", Map.of("type", "keyword")))))); - } - - public void testResolveMappings() throws Exception { - MetadataIndexTemplateService service = getMetadataIndexTemplateService(); - ProjectMetadata project = ProjectMetadata.builder(randomProjectIdOrDefault()).build(); - - ComponentTemplate ct1 = new ComponentTemplate(new Template(null, new CompressedXContent(""" - { - "properties": { - "field1": { - "type": "keyword" - } - } - }"""), null), null, null); - ComponentTemplate ct2 = new ComponentTemplate(new Template(null, new CompressedXContent(""" - { - "properties": { - "field2": { - "type": "text" - } - } - }"""), null), null, null); - project = service.addComponentTemplate(project, true, "ct_high", ct1); - project = service.addComponentTemplate(project, true, "ct_low", ct2); - ComposableIndexTemplate it = ComposableIndexTemplate.builder() - .indexPatterns(List.of("i*")) - .template(new Template(null, new CompressedXContent(""" - { - "properties": { - "field3": { - "type": "integer" - } - } - }"""), null)) - .componentTemplates(List.of("ct_low", "ct_high")) - .priority(0L) - .version(1L) - .build(); - project = service.addIndexTemplateV2(project, true, "my-template", it); - - List mappings = MetadataIndexTemplateService.collectMappings(project, "my-template", "my-index"); - - assertNotNull(mappings); - assertThat(mappings.size(), equalTo(3)); - List> parsedMappings = mappings.stream().map(m -> { - try { - return MapperService.parseMapping(NamedXContentRegistry.EMPTY, m); - } catch (Exception e) { - logger.error(e); - fail("failed to parse mappings: " + m.string()); - return null; - } - }).toList(); - assertThat(parsedMappings.get(0), equalTo(Map.of("_doc", Map.of("properties", Map.of("field2", Map.of("type", "text")))))); - assertThat(parsedMappings.get(1), equalTo(Map.of("_doc", Map.of("properties", Map.of("field1", Map.of("type", "keyword")))))); - assertThat(parsedMappings.get(2), equalTo(Map.of("_doc", Map.of("properties", Map.of("field3", Map.of("type", "integer")))))); - } - - public void testDefinedTimestampMappingIsAddedForDataStreamTemplates() throws Exception { - MetadataIndexTemplateService service = getMetadataIndexTemplateService(); - ProjectMetadata project = ProjectMetadata.builder(randomProjectIdOrDefault()).build(); - - ComponentTemplate ct1 = new ComponentTemplate(new Template(null, new CompressedXContent(""" - { - "properties": { - "field1": { - "type": "keyword" - } - } - }"""), null), null, null); - - project = service.addComponentTemplate(project, true, "ct1", ct1); - - { - ComposableIndexTemplate it = ComposableIndexTemplate.builder() - .indexPatterns(List.of("logs*")) - .template(new Template(null, new CompressedXContent(""" - { - "properties": { - "field2": { - "type": "integer" - } - } - }"""), null)) - .componentTemplates(List.of("ct1")) - .priority(0L) - .version(1L) - .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate()) - .build(); - project = service.addIndexTemplateV2(project, true, "logs-data-stream-template", it); - - List mappings = MetadataIndexTemplateService.collectMappings( - project, - "logs-data-stream-template", - DataStream.getDefaultBackingIndexName("logs", 1L) - ); - - assertNotNull(mappings); - assertThat(mappings.size(), equalTo(4)); - List> parsedMappings = mappings.stream().map(m -> { - try { - return MapperService.parseMapping(NamedXContentRegistry.EMPTY, m); - } catch (Exception e) { - logger.error(e); - fail("failed to parse mappings: " + m.string()); - return null; - } - }).toList(); - - assertThat( - parsedMappings.get(0), - equalTo( - Map.of( - "_doc", - Map.of( - "properties", - Map.of(DEFAULT_TIMESTAMP_FIELD, Map.of("type", "date", "ignore_malformed", "false")), - "_routing", - Map.of("required", false) - ) - ) - ) - ); - assertThat(parsedMappings.get(1), equalTo(Map.of("_doc", Map.of("properties", Map.of("field1", Map.of("type", "keyword")))))); - assertThat(parsedMappings.get(2), equalTo(Map.of("_doc", Map.of("properties", Map.of("field2", Map.of("type", "integer")))))); - } - - { - // indices matched by templates without the data stream field defined don't get the default @timestamp mapping - ComposableIndexTemplate it = ComposableIndexTemplate.builder() - .indexPatterns(List.of("timeseries*")) - .template(new Template(null, new CompressedXContent(""" - { - "properties": { - "field2": { - "type": "integer" - } - } - }"""), null)) - .componentTemplates(List.of("ct1")) - .priority(0L) - .version(1L) - .build(); - project = service.addIndexTemplateV2(project, true, "timeseries-template", it); - - List mappings = MetadataIndexTemplateService.collectMappings(project, "timeseries-template", "timeseries"); - - assertNotNull(mappings); - assertThat(mappings.size(), equalTo(2)); - List> parsedMappings = mappings.stream().map(m -> { - try { - return MapperService.parseMapping(NamedXContentRegistry.EMPTY, m); - } catch (Exception e) { - logger.error(e); - fail("failed to parse mappings: " + m.string()); - return null; - } - }).toList(); - - assertThat(parsedMappings.get(0), equalTo(Map.of("_doc", Map.of("properties", Map.of("field1", Map.of("type", "keyword")))))); - assertThat(parsedMappings.get(1), equalTo(Map.of("_doc", Map.of("properties", Map.of("field2", Map.of("type", "integer")))))); - - // a default @timestamp mapping will not be added if the matching template doesn't have the data stream field configured, even - // if the index name matches that of a data stream backing index - mappings = MetadataIndexTemplateService.collectMappings( - project, - "timeseries-template", - DataStream.getDefaultBackingIndexName("timeseries", 1L) - ); - - assertNotNull(mappings); - assertThat(mappings.size(), equalTo(2)); - parsedMappings = mappings.stream().map(m -> { - try { - return MapperService.parseMapping(NamedXContentRegistry.EMPTY, m); - } catch (Exception e) { - logger.error(e); - fail("failed to parse mappings: " + m.string()); - return null; - } - }).toList(); - - assertThat(parsedMappings.get(0), equalTo(Map.of("_doc", Map.of("properties", Map.of("field1", Map.of("type", "keyword")))))); - assertThat(parsedMappings.get(1), equalTo(Map.of("_doc", Map.of("properties", Map.of("field2", Map.of("type", "integer")))))); - } - } - - public void testUserDefinedMappingTakesPrecedenceOverDefault() throws Exception { - MetadataIndexTemplateService service = getMetadataIndexTemplateService(); - ProjectMetadata project = ProjectMetadata.builder(randomProjectIdOrDefault()).build(); - - { - // user defines a @timestamp mapping as part of a component template - ComponentTemplate ct1 = new ComponentTemplate(new Template(null, new CompressedXContent(""" - { - "properties": { - "@timestamp": { - "type": "date_nanos" - } - } - }"""), null), null, null); - - project = service.addComponentTemplate(project, true, "ct1", ct1); - ComposableIndexTemplate it = ComposableIndexTemplate.builder() - .indexPatterns(List.of("logs*")) - .componentTemplates(List.of("ct1")) - .priority(0L) - .version(1L) - .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate()) - .build(); - project = service.addIndexTemplateV2(project, true, "logs-template", it); - - List mappings = MetadataIndexTemplateService.collectMappings( - project, - "logs-template", - DataStream.getDefaultBackingIndexName("logs", 1L) - ); - - assertNotNull(mappings); - assertThat(mappings.size(), equalTo(3)); - List> parsedMappings = mappings.stream().map(m -> { - try { - return MapperService.parseMapping(NamedXContentRegistry.EMPTY, m); - } catch (Exception e) { - logger.error(e); - fail("failed to parse mappings: " + m.string()); - return null; - } - }).toList(); - assertThat( - parsedMappings.get(0), - equalTo( - Map.of( - "_doc", - Map.of( - "properties", - Map.of(DEFAULT_TIMESTAMP_FIELD, Map.of("type", "date", "ignore_malformed", "false")), - "_routing", - Map.of("required", false) - ) - ) - ) - ); - assertThat( - parsedMappings.get(1), - equalTo(Map.of("_doc", Map.of("properties", Map.of(DEFAULT_TIMESTAMP_FIELD, Map.of("type", "date_nanos"))))) - ); - } - - { - // user defines a @timestamp mapping as part of a composable index template - Template template = new Template(null, new CompressedXContent(""" - { - "properties": { - "@timestamp": { - "type": "date_nanos" - } - } - }"""), null); - ComposableIndexTemplate it = ComposableIndexTemplate.builder() - .indexPatterns(List.of("timeseries*")) - .template(template) - .priority(0L) - .version(1L) - .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate()) - .build(); - project = service.addIndexTemplateV2(project, true, "timeseries-template", it); - - List mappings = MetadataIndexTemplateService.collectMappings( - project, - "timeseries-template", - DataStream.getDefaultBackingIndexName("timeseries-template", 1L) - ); - - assertNotNull(mappings); - assertThat(mappings.size(), equalTo(3)); - List> parsedMappings = mappings.stream().map(m -> { - try { - return MapperService.parseMapping(NamedXContentRegistry.EMPTY, m); - } catch (Exception e) { - logger.error(e); - fail("failed to parse mappings: " + m.string()); - return null; - } - }).toList(); - assertThat( - parsedMappings.get(0), - equalTo( - Map.of( - "_doc", - Map.of( - "properties", - Map.of(DEFAULT_TIMESTAMP_FIELD, Map.of("type", "date", "ignore_malformed", "false")), - "_routing", - Map.of("required", false) - ) - ) - ) - ); - assertThat( - parsedMappings.get(1), - equalTo(Map.of("_doc", Map.of("properties", Map.of(DEFAULT_TIMESTAMP_FIELD, Map.of("type", "date_nanos"))))) - ); - } - } - - public void testResolveSettings() throws Exception { - MetadataIndexTemplateService service = getMetadataIndexTemplateService(); - ProjectMetadata project = ProjectMetadata.builder(randomProjectIdOrDefault()).build(); - - ComponentTemplate ct1 = new ComponentTemplate( - new Template(Settings.builder().put("number_of_replicas", 2).put("index.blocks.write", true).build(), null, null), - null, - null - ); - ComponentTemplate ct2 = new ComponentTemplate( - new Template(Settings.builder().put("index.number_of_replicas", 1).put("index.blocks.read", true).build(), null, null), - null, - null - ); - project = service.addComponentTemplate(project, true, "ct_high", ct1); - project = service.addComponentTemplate(project, true, "ct_low", ct2); - ComposableIndexTemplate it = ComposableIndexTemplate.builder() - .indexPatterns(List.of("i*")) - .template( - new Template(Settings.builder().put("index.blocks.write", false).put("index.number_of_shards", 3).build(), null, null) - ) - .componentTemplates(List.of("ct_low", "ct_high")) - .priority(0L) - .version(1L) - .build(); - project = service.addIndexTemplateV2(project, true, "my-template", it); - - Settings settings = MetadataIndexTemplateService.resolveSettings(project, "my-template"); - assertThat(settings.get("index.number_of_replicas"), equalTo("2")); - assertThat(settings.get("index.blocks.write"), equalTo("false")); - assertThat(settings.get("index.blocks.read"), equalTo("true")); - assertThat(settings.get("index.number_of_shards"), equalTo("3")); - } - public void testResolveAliases() throws Exception { MetadataIndexTemplateService service = getMetadataIndexTemplateService(); ProjectMetadata project = ProjectMetadata.builder(randomProjectIdOrDefault()).build(); @@ -2670,111 +2268,6 @@ public void testAddInvalidTemplateIgnoreService() throws Exception { assertThat(e.getMessage(), containsString("missing component templates [fail] that does not exist")); } - public void testComposableTemplateWithSubobjectsFalse() throws Exception { - MetadataIndexTemplateService service = getMetadataIndexTemplateService(); - ProjectMetadata project = ProjectMetadata.builder(randomProjectIdOrDefault()).build(); - - ComponentTemplate subobjects = new ComponentTemplate(new Template(null, new CompressedXContent(""" - { - "subobjects": false - } - """), null), null, null); - - ComponentTemplate fieldMapping = new ComponentTemplate(new Template(null, new CompressedXContent(""" - { - "properties": { - "parent.subfield": { - "type": "keyword" - } - } - } - """), null), null, null); - - project = service.addComponentTemplate(project, true, "subobjects", subobjects); - project = service.addComponentTemplate(project, true, "field_mapping", fieldMapping); - ComposableIndexTemplate it = ComposableIndexTemplate.builder() - .indexPatterns(List.of("test-*")) - .template(new Template(null, null, null)) - .componentTemplates(List.of("subobjects", "field_mapping")) - .priority(0L) - .version(1L) - .build(); - project = service.addIndexTemplateV2(project, true, "composable-template", it); - - List mappings = MetadataIndexTemplateService.collectMappings(project, "composable-template", "test-index"); - - assertNotNull(mappings); - assertThat(mappings.size(), equalTo(2)); - List> parsedMappings = mappings.stream().map(m -> { - try { - return MapperService.parseMapping(NamedXContentRegistry.EMPTY, m); - } catch (Exception e) { - logger.error(e); - fail("failed to parse mappings: " + m.string()); - return null; - } - }).toList(); - - assertThat(parsedMappings.get(0), equalTo(Map.of("_doc", Map.of("subobjects", false)))); - assertThat( - parsedMappings.get(1), - equalTo(Map.of("_doc", Map.of("properties", Map.of("parent.subfield", Map.of("type", "keyword"))))) - ); - } - - public void testComposableTemplateWithSubobjectsFalseObjectAndSubfield() throws Exception { - MetadataIndexTemplateService service = getMetadataIndexTemplateService(); - ProjectMetadata project = ProjectMetadata.builder(randomProjectIdOrDefault()).build(); - - ComponentTemplate subobjects = new ComponentTemplate(new Template(null, new CompressedXContent(""" - { - "properties": { - "foo": { - "type": "object", - "subobjects": false - }, - "foo.bar": { - "type": "keyword" - } - } - } - """), null), null, null); - - project = service.addComponentTemplate(project, true, "subobjects", subobjects); - ComposableIndexTemplate it = ComposableIndexTemplate.builder() - .indexPatterns(List.of("test-*")) - .template(new Template(null, null, null)) - .componentTemplates(List.of("subobjects", "field_mapping")) - .priority(0L) - .version(1L) - .build(); - project = service.addIndexTemplateV2(project, true, "composable-template", it); - - List mappings = MetadataIndexTemplateService.collectMappings(project, "composable-template", "test-index"); - - assertNotNull(mappings); - assertThat(mappings.size(), equalTo(1)); - List> parsedMappings = mappings.stream().map(m -> { - try { - return MapperService.parseMapping(NamedXContentRegistry.EMPTY, m); - } catch (Exception e) { - logger.error(e); - fail("failed to parse mappings: " + m.string()); - return null; - } - }).toList(); - - assertThat( - parsedMappings.get(0), - equalTo( - Map.of( - "_doc", - Map.of("properties", Map.of("foo.bar", Map.of("type", "keyword"), "foo", Map.of("type", "object", "subobjects", false))) - ) - ) - ); - } - public void testAddIndexTemplateWithDeprecatedComponentTemplate() throws Exception { ProjectMetadata project = ProjectMetadata.builder(randomProjectIdOrDefault()).build(); MetadataIndexTemplateService service = getMetadataIndexTemplateService();