diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsAction.java index 784cac38e0ba9..c1d4f83ce553e 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsAction.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsAction.java @@ -258,7 +258,7 @@ static GetDataStreamAction.Response innerOperation( } else { indexTemplate = MetadataIndexTemplateService.findV2Template(state.metadata(), dataStream.getName(), false); if (indexTemplate != null) { - Settings settings = MetadataIndexTemplateService.resolveSettings(state.metadata(), indexTemplate); + Settings settings = dataStream.getEffectiveSettings(state.metadata()); ilmPolicyName = settings.get(IndexMetadata.LIFECYCLE_NAME); if (indexMode == null && state.metadata().templatesV2().get(indexTemplate) != null) { indexMode = resolveMode( @@ -266,7 +266,7 @@ static GetDataStreamAction.Response innerOperation( indexSettingProviders, dataStream, settings, - state.metadata().templatesV2().get(indexTemplate) + dataStream.getEffectiveIndexTemplate(state.metadata()) ); } indexTemplatePreferIlmValue = PREFER_ILM_SETTING.get(settings); diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsActionTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsActionTests.java index 15252528e952b..c6facb8a03f3d 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsActionTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsActionTests.java @@ -11,14 +11,18 @@ import org.elasticsearch.action.datastreams.GetDataStreamAction; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.metadata.DataStreamFailureStoreSettings; import org.elasticsearch.cluster.metadata.DataStreamGlobalRetention; import org.elasticsearch.cluster.metadata.DataStreamGlobalRetentionSettings; import org.elasticsearch.cluster.metadata.DataStreamTestHelper; +import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.cluster.metadata.ProjectMetadata; +import org.elasticsearch.cluster.metadata.Template; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.TimeValue; @@ -27,15 +31,20 @@ import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.index.IndexSettingProviders; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.indices.SystemIndices; import org.elasticsearch.indices.TestIndexNameExpressionResolver; import org.elasticsearch.test.ESTestCase; import java.time.Instant; import java.time.temporal.ChronoUnit; +import java.util.ArrayList; import java.util.List; import java.util.Set; +import java.util.stream.Collectors; +import static org.elasticsearch.cluster.metadata.DataStream.getDefaultBackingIndexName; +import static org.elasticsearch.cluster.metadata.DataStreamTestHelper.createIndexMetadata; import static org.elasticsearch.cluster.metadata.DataStreamTestHelper.getClusterStateWithDataStreams; import static org.elasticsearch.test.LambdaMatchers.transformedItemsMatch; import static org.elasticsearch.test.LambdaMatchers.transformedMatch; @@ -317,9 +326,9 @@ public void testGetTimeSeriesMixedDataStream() { null ); - var name1 = DataStream.getDefaultBackingIndexName("ds-1", 1, instant.toEpochMilli()); - var name2 = DataStream.getDefaultBackingIndexName("ds-1", 2, instant.toEpochMilli()); - var name3 = DataStream.getDefaultBackingIndexName("ds-1", 3, twoHoursAgo.toEpochMilli()); + var name1 = getDefaultBackingIndexName("ds-1", 1, instant.toEpochMilli()); + var name2 = getDefaultBackingIndexName("ds-1", 2, instant.toEpochMilli()); + var name3 = getDefaultBackingIndexName("ds-1", 3, twoHoursAgo.toEpochMilli()); assertThat( response.getDataStreams(), contains( @@ -527,4 +536,120 @@ public void testProvidersAffectMode() { equalTo("standard") ); } + + public void testGetEffectiveSettingsTemplateOnlySettings() { + // Set a lifecycle only in the template, and make sure that is in the response: + ProjectId projectId = randomProjectIdOrDefault(); + GetDataStreamAction.Request req = new GetDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] {}); + final String templatePolicy = "templatePolicy"; + final String templateIndexMode = IndexMode.LOOKUP.getName(); + final String dataStreamPolicy = "dataStreamPolicy"; + final String dataStreamIndexMode = IndexMode.LOGSDB.getName(); + + ClusterState state = getClusterStateWithDataStreamWithSettings( + projectId, + Settings.builder() + .put(IndexMetadata.LIFECYCLE_NAME, templatePolicy) + .put(IndexSettings.MODE.getKey(), templateIndexMode) + .build(), + Settings.EMPTY + ); + + GetDataStreamAction.Response response = TransportGetDataStreamsAction.innerOperation( + state.projectState(projectId), + req, + resolver, + systemIndices, + ClusterSettings.createBuiltInClusterSettings(), + dataStreamGlobalRetentionSettings, + emptyDataStreamFailureStoreSettings, + new IndexSettingProviders(Set.of()), + null + ); + assertNotNull(response.getDataStreams()); + assertThat(response.getDataStreams().size(), equalTo(1)); + assertThat(response.getDataStreams().get(0).getIlmPolicy(), equalTo(templatePolicy)); + assertThat(response.getDataStreams().get(0).getIndexModeName(), equalTo(templateIndexMode)); + } + + public void testGetEffectiveSettings() { + ProjectId projectId = randomProjectIdOrDefault(); + GetDataStreamAction.Request req = new GetDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] {}); + final String templatePolicy = "templatePolicy"; + final String templateIndexMode = IndexMode.LOOKUP.getName(); + final String dataStreamPolicy = "dataStreamPolicy"; + final String dataStreamIndexMode = IndexMode.LOGSDB.getName(); + // Now set a lifecycle in both the template and the data stream, and make sure the response has the data stream one: + ClusterState state = getClusterStateWithDataStreamWithSettings( + projectId, + Settings.builder() + .put(IndexMetadata.LIFECYCLE_NAME, templatePolicy) + .put(IndexSettings.MODE.getKey(), templateIndexMode) + .build(), + Settings.builder() + .put(IndexMetadata.LIFECYCLE_NAME, dataStreamPolicy) + .put(IndexSettings.MODE.getKey(), dataStreamIndexMode) + .build() + ); + GetDataStreamAction.Response response = TransportGetDataStreamsAction.innerOperation( + state.projectState(projectId), + req, + resolver, + systemIndices, + ClusterSettings.createBuiltInClusterSettings(), + dataStreamGlobalRetentionSettings, + emptyDataStreamFailureStoreSettings, + new IndexSettingProviders(Set.of()), + null + ); + assertNotNull(response.getDataStreams()); + assertThat(response.getDataStreams().size(), equalTo(1)); + assertThat(response.getDataStreams().get(0).getIlmPolicy(), equalTo(dataStreamPolicy)); + assertThat(response.getDataStreams().get(0).getIndexModeName(), equalTo(dataStreamIndexMode)); + } + + private static ClusterState getClusterStateWithDataStreamWithSettings( + ProjectId projectId, + Settings templateSettings, + Settings dataStreamSettings + ) { + String dataStreamName = "data-stream-1"; + int numberOfBackingIndices = randomIntBetween(1, 5); + long currentTime = System.currentTimeMillis(); + int replicas = 0; + boolean replicated = false; + ProjectMetadata.Builder builder = ProjectMetadata.builder(projectId); + builder.put( + "template_1", + ComposableIndexTemplate.builder() + .indexPatterns(List.of("*")) + .template(Template.builder().settings(templateSettings)) + .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate()) + .build() + ); + + List backingIndices = new ArrayList<>(); + for (int backingIndexNumber = 1; backingIndexNumber <= numberOfBackingIndices; backingIndexNumber++) { + backingIndices.add( + createIndexMetadata( + getDefaultBackingIndexName(dataStreamName, backingIndexNumber, currentTime), + true, + templateSettings, + replicas + ) + ); + } + List allIndices = new ArrayList<>(backingIndices); + + DataStream ds = DataStream.builder( + dataStreamName, + backingIndices.stream().map(IndexMetadata::getIndex).collect(Collectors.toList()) + ).setGeneration(numberOfBackingIndices).setSettings(dataStreamSettings).setReplicated(replicated).build(); + builder.put(ds); + + for (IndexMetadata index : allIndices) { + builder.put(index, false); + } + return ClusterState.builder(new ClusterName("_name")).putProjectMetadata(builder.build()).build(); + } } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceIT.java new file mode 100644 index 0000000000000..20826adcc1356 --- /dev/null +++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceIT.java @@ -0,0 +1,78 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.cluster.metadata; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.admin.indices.create.CreateIndexClusterStateUpdateRequest; +import org.elasticsearch.action.admin.indices.get.GetIndexRequest; +import org.elasticsearch.action.admin.indices.get.GetIndexResponse; +import org.elasticsearch.action.support.master.ShardsAcknowledgedResponse; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.test.ESIntegTestCase; + +import java.util.Locale; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +import static org.hamcrest.Matchers.equalTo; + +public class MetadataCreateIndexServiceIT extends ESIntegTestCase { + + public void testRequestTemplateIsRespected() throws InterruptedException { + /* + * This test passes a template in the CreateIndexClusterStateUpdateRequest, and makes sure that the settings from that template + * are used when creating the index. + */ + MetadataCreateIndexService metadataCreateIndexService = internalCluster().getCurrentMasterNodeInstance( + MetadataCreateIndexService.class + ); + final String indexName = randomAlphaOfLength(20).toLowerCase(Locale.ROOT); + final int numberOfReplicas = randomIntBetween(1, 7); + CreateIndexClusterStateUpdateRequest request = new CreateIndexClusterStateUpdateRequest( + "testRequestTemplateIsRespected", + ProjectId.DEFAULT, + indexName, + randomAlphaOfLength(20) + ); + request.setMatchingTemplate( + ComposableIndexTemplate.builder() + .template(Template.builder().settings(Settings.builder().put("index.number_of_replicas", numberOfReplicas))) + .build() + ); + final CountDownLatch listenerCalledLatch = new CountDownLatch(1); + ActionListener listener = new ActionListener<>() { + @Override + public void onResponse(ShardsAcknowledgedResponse shardsAcknowledgedResponse) { + listenerCalledLatch.countDown(); + } + + @Override + public void onFailure(Exception e) { + logger.error(e); + listenerCalledLatch.countDown(); + } + }; + + metadataCreateIndexService.createIndex( + TimeValue.THIRTY_SECONDS, + TimeValue.THIRTY_SECONDS, + TimeValue.THIRTY_SECONDS, + request, + listener + ); + listenerCalledLatch.await(10, TimeUnit.SECONDS); + GetIndexResponse response = admin().indices() + .getIndex(new GetIndexRequest(TimeValue.THIRTY_SECONDS).indices(indexName)) + .actionGet(); + Settings settings = response.getSettings().get(indexName); + assertThat(settings.get("index.number_of_replicas"), equalTo(Integer.toString(numberOfReplicas))); + } +} diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverService.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverService.java index 161ea20252461..9082dad757f8c 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverService.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverService.java @@ -66,7 +66,6 @@ import static org.elasticsearch.cluster.metadata.IndexAbstraction.Type.ALIAS; import static org.elasticsearch.cluster.metadata.IndexAbstraction.Type.DATA_STREAM; -import static org.elasticsearch.cluster.metadata.MetadataCreateDataStreamService.lookupTemplateForDataStream; import static org.elasticsearch.cluster.metadata.MetadataIndexTemplateService.findV1Templates; import static org.elasticsearch.cluster.metadata.MetadataIndexTemplateService.findV2Template; import static org.elasticsearch.cluster.routing.allocation.allocator.AllocationActionListener.rerouteCompletionIsNotRequired; @@ -325,7 +324,7 @@ private RolloverResult rolloverDataStream( final SystemDataStreamDescriptor systemDataStreamDescriptor; if (dataStream.isSystem() == false) { systemDataStreamDescriptor = null; - templateV2 = lookupTemplateForDataStream(dataStreamName, metadata); + templateV2 = dataStream.getEffectiveIndexTemplate(projectState.metadata()); } else { systemDataStreamDescriptor = systemIndices.findMatchingDataStreamDescriptor(dataStreamName); if (systemDataStreamDescriptor == null) { diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java index 25018bb32bf3b..dc2af8809b7bc 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java @@ -267,7 +267,7 @@ public static Template resolveTemplate( List mappings = MetadataCreateIndexService.collectV2Mappings( null, // empty request mapping as the user can't specify any explicit mappings via the simulate api simulatedProject, - matchingTemplate, + template, xContentRegistry, simulatedIndexName ); diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java index 261962de96b28..72dcbd13e7e00 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java @@ -381,7 +381,12 @@ public ComposableIndexTemplate getEffectiveIndexTemplate(ProjectMetadata project public Settings getEffectiveSettings(ProjectMetadata projectMetadata) { ComposableIndexTemplate template = getMatchingIndexTemplate(projectMetadata); - Settings templateSettings = template.template() == null ? Settings.EMPTY : template.template().settings(); + final Settings templateSettings; + if (template.template() == null || template.template().settings() == null) { + templateSettings = Settings.EMPTY; + } else { + templateSettings = template.template().settings(); + } return templateSettings.merge(settings); } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java index 6355fdc8387f9..9b50872d8fa80 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java @@ -441,6 +441,18 @@ public ClusterState applyCreateIndexRequest( ? IndexMetadata.INDEX_HIDDEN_SETTING.get(request.settings()) : null; + ComposableIndexTemplate templateFromRequest = request.matchingTemplate(); + if (templateFromRequest != null) { + return applyCreateIndexRequestWithV2Template( + currentState, + request, + silent, + templateFromRequest, + metadataTransformer, + rerouteListener + ); + } + // Check to see if a v2 template matched final String v2Template = MetadataIndexTemplateService.findV2Template( projectMetadata, @@ -705,7 +717,6 @@ private ClusterState applyCreateIndexRequestWithV2Template( final Metadata metadata = currentState.getMetadata(); final ProjectMetadata projectMetadata = metadata.getProject(request.projectId()); - final RoutingTable routingTable = currentState.routingTable(request.projectId()); ComposableIndexTemplate template = projectMetadata.templatesV2().get(templateName); final boolean isDataStream = template.getDataStreamTemplate() != null; @@ -719,11 +730,28 @@ private ClusterState applyCreateIndexRequestWithV2Template( + "use create data stream api instead" ); } + return applyCreateIndexRequestWithV2Template(currentState, request, silent, template, projectMetadataTransformer, rerouteListener); + } + + private ClusterState applyCreateIndexRequestWithV2Template( + final ClusterState currentState, + final CreateIndexClusterStateUpdateRequest request, + final boolean silent, + final ComposableIndexTemplate template, + final BiConsumer projectMetadataTransformer, + final ActionListener rerouteListener + ) throws Exception { + + final Metadata metadata = currentState.getMetadata(); + final ProjectMetadata projectMetadata = metadata.getProject(request.projectId()); + final RoutingTable routingTable = currentState.routingTable(request.projectId()); + + final boolean isDataStream = template.getDataStreamTemplate() != null; final List mappings = collectV2Mappings( request.mappings(), projectMetadata, - templateName, + template, xContentRegistry, request.index() ); @@ -734,7 +762,7 @@ private ClusterState applyCreateIndexRequestWithV2Template( currentState.blocks(), routingTable, request, - resolveSettings(projectMetadata, templateName), + resolveSettings(template, projectMetadata.componentTemplates()), mappings, null, settings, @@ -756,7 +784,7 @@ private ClusterState applyCreateIndexRequestWithV2Template( request.index(), // data stream aliases are created separately in MetadataCreateDataStreamService::createDataStream isDataStream ? Set.of() : request.aliases(), - isDataStream ? List.of() : MetadataIndexTemplateService.resolveAliases(projectMetadata, templateName), + isDataStream ? List.of() : MetadataIndexTemplateService.resolveAliases(projectMetadata, template), projectMetadata, xContentRegistry, // the context is used ony for validation so it's fine to pass fake values for the shard id and the current timestamp @@ -764,7 +792,7 @@ private ClusterState applyCreateIndexRequestWithV2Template( IndexService.dateMathExpressionResolverAt(request.getNameResolvedAt()), systemIndices::isSystemName ), - Collections.singletonList(templateName), + Collections.singletonList("provided in request"), projectMetadataTransformer, rerouteListener ); @@ -906,17 +934,6 @@ private static List collectSystemV2Mappings( return collectV2Mappings(null, templateMappings, xContentRegistry); } - public static List collectV2Mappings( - @Nullable final String requestMappings, - final ProjectMetadata projectMetadata, - final String templateName, - final NamedXContentRegistry xContentRegistry, - final String indexName - ) throws Exception { - List templateMappings = MetadataIndexTemplateService.collectMappings(projectMetadata, templateName, indexName); - return collectV2Mappings(requestMappings, templateMappings, xContentRegistry); - } - private static List collectV2Mappings( @Nullable final String requestMappings, final List templateMappings, @@ -933,6 +950,21 @@ private static List collectV2Mappings( return result; } + public static List collectV2Mappings( + @Nullable final String requestMappings, + final ProjectMetadata projectMetadata, + final ComposableIndexTemplate template, + final NamedXContentRegistry xContentRegistry, + final String indexName + ) throws Exception { + List templateMappings = MetadataIndexTemplateService.collectMappings( + template, + projectMetadata.componentTemplates(), + indexName + ); + return collectV2Mappings(requestMappings, templateMappings, xContentRegistry); + } + private ClusterState applyCreateIndexRequestWithExistingMetadata( final ClusterState currentState, final CreateIndexClusterStateUpdateRequest request, diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java index 26fd4635b7185..a4a46ad2e8ad4 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java @@ -63,6 +63,8 @@ import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.lessThanOrEqualTo; import static org.hamcrest.Matchers.nullValue; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -648,7 +650,15 @@ public void testRolloverClusterStateForDataStream() throws Exception { false ); long after = testThreadPool.absoluteTimeInMillis(); - + Settings rolledOverIndexSettings = rolloverResult.clusterState() + .projectState(projectId) + .metadata() + .index(rolloverResult.rolloverIndexName()) + .getSettings(); + Set rolledOverIndexSettingNames = rolledOverIndexSettings.keySet(); + for (String settingName : dataStream.getEffectiveSettings(clusterState.projectState(projectId).metadata()).keySet()) { + assertTrue(rolledOverIndexSettingNames.contains(settingName)); + } String newIndexName = DataStream.getDefaultBackingIndexName(dataStream.getName(), dataStream.getGeneration() + 1); assertEquals(sourceIndexName, rolloverResult.sourceIndexName()); assertEquals(newIndexName, rolloverResult.rolloverIndexName()); @@ -719,7 +729,15 @@ public void testRolloverClusterStateForDataStreamFailureStore() throws Exception true ); long after = testThreadPool.absoluteTimeInMillis(); - + Settings rolledOverIndexSettings = rolloverResult.clusterState() + .projectState(projectId) + .metadata() + .index(rolloverResult.rolloverIndexName()) + .getSettings(); + Set rolledOverIndexSettingNames = rolledOverIndexSettings.keySet(); + for (String settingName : dataStream.getSettings().keySet()) { + assertFalse(rolledOverIndexSettingNames.contains(settingName)); + } var epochMillis = System.currentTimeMillis(); String sourceIndexName = DataStream.getDefaultFailureStoreName(dataStream.getName(), dataStream.getGeneration(), epochMillis); String newIndexName = DataStream.getDefaultFailureStoreName(dataStream.getName(), dataStream.getGeneration() + 1, epochMillis);