Using DataStream::getEffectiveSettings for rollover

modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsAction.java

@@ -258,15 +258,15 @@ static GetDataStreamAction.Response innerOperation(
             } else {
                 indexTemplate = MetadataIndexTemplateService.findV2Template(state.metadata(), dataStream.getName(), false);
                 if (indexTemplate != null) {
-                    Settings settings = MetadataIndexTemplateService.resolveSettings(state.metadata(), indexTemplate);
+                    Settings settings = dataStream.getEffectiveSettings(state.metadata());
                     ilmPolicyName = settings.get(IndexMetadata.LIFECYCLE_NAME);
                     if (indexMode == null && state.metadata().templatesV2().get(indexTemplate) != null) {
                         indexMode = resolveMode(
                             state,
                             indexSettingProviders,
                             dataStream,
                             settings,
-                            state.metadata().templatesV2().get(indexTemplate)
+                            dataStream.getEffectiveIndexTemplate(state.metadata())
                         );
                     }
                     indexTemplatePreferIlmValue = PREFER_ILM_SETTING.get(settings);

modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsActionTests.java

@@ -11,14 +11,18 @@
 import org.elasticsearch.action.datastreams.GetDataStreamAction;
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.ComposableIndexTemplate;
 import org.elasticsearch.cluster.metadata.DataStream;
 import org.elasticsearch.cluster.metadata.DataStreamFailureStoreSettings;
 import org.elasticsearch.cluster.metadata.DataStreamGlobalRetention;
 import org.elasticsearch.cluster.metadata.DataStreamGlobalRetentionSettings;
 import org.elasticsearch.cluster.metadata.DataStreamTestHelper;
+import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.Metadata;
+import org.elasticsearch.cluster.metadata.ProjectId;
 import org.elasticsearch.cluster.metadata.ProjectMetadata;
+import org.elasticsearch.cluster.metadata.Template;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.core.TimeValue;
@@ -27,15 +31,20 @@
 import org.elasticsearch.index.IndexMode;
 import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.index.IndexSettingProviders;
+import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.indices.SystemIndices;
 import org.elasticsearch.indices.TestIndexNameExpressionResolver;
 import org.elasticsearch.test.ESTestCase;
 
 import java.time.Instant;
 import java.time.temporal.ChronoUnit;
+import java.util.ArrayList;
 import java.util.List;
 import java.util.Set;
+import java.util.stream.Collectors;
 
+import static org.elasticsearch.cluster.metadata.DataStream.getDefaultBackingIndexName;
+import static org.elasticsearch.cluster.metadata.DataStreamTestHelper.createIndexMetadata;
 import static org.elasticsearch.cluster.metadata.DataStreamTestHelper.getClusterStateWithDataStreams;
 import static org.elasticsearch.test.LambdaMatchers.transformedItemsMatch;
 import static org.elasticsearch.test.LambdaMatchers.transformedMatch;
@@ -527,4 +536,114 @@ public void testProvidersAffectMode() {
             equalTo("standard")
         );
     }
+
+    public void testGetEffectiveSettings() {
+        ProjectId projectId = randomProjectIdOrDefault();
+        GetDataStreamAction.Request req = new GetDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] {});
+        final String templatePolicy = "templatePolicy";
+        final String templateIndexMode = IndexMode.LOOKUP.getName();
+        final String dataStreamPolicy = "dataStreamPolicy";
+        final String dataStreamIndexMode = IndexMode.LOGSDB.getName();
+        {
+            // First, set a lifecycle only in the template, and make sure that is in the response:
+            ClusterState state = getClusterStateWithDataStreamWithSettings(
+                projectId,
+                Settings.builder()
+                    .put(IndexMetadata.LIFECYCLE_NAME, templatePolicy)
+                    .put(IndexSettings.MODE.getKey(), templateIndexMode)
+                    .build(),
+                Settings.EMPTY
+            );
+
+            GetDataStreamAction.Response response = TransportGetDataStreamsAction.innerOperation(
+                state.projectState(projectId),
+                req,
+                resolver,
+                systemIndices,
+                ClusterSettings.createBuiltInClusterSettings(),
+                dataStreamGlobalRetentionSettings,
+                emptyDataStreamFailureStoreSettings,
+                new IndexSettingProviders(Set.of()),
+                null
+            );
+            assertNotNull(response.getDataStreams());
+            assertThat(response.getDataStreams().size(), equalTo(1));
+            assertThat(response.getDataStreams().get(0).getIlmPolicy(), equalTo(templatePolicy));
+            assertThat(response.getDataStreams().get(0).getIndexModeName(), equalTo(templateIndexMode));
+        }
+        {
+            // Now set a lifecycle in both the template and the data stream, and make sure the response has the data stream one:
+            ClusterState state = getClusterStateWithDataStreamWithSettings(
+                projectId,
+                Settings.builder()
+                    .put(IndexMetadata.LIFECYCLE_NAME, templatePolicy)
+                    .put(IndexSettings.MODE.getKey(), templateIndexMode)
+                    .build(),
+                Settings.builder()
+                    .put(IndexMetadata.LIFECYCLE_NAME, dataStreamPolicy)
+                    .put(IndexSettings.MODE.getKey(), dataStreamIndexMode)
+                    .build()
+            );
+            GetDataStreamAction.Response response = TransportGetDataStreamsAction.innerOperation(
+                state.projectState(projectId),
+                req,
+                resolver,
+                systemIndices,
+                ClusterSettings.createBuiltInClusterSettings(),
+                dataStreamGlobalRetentionSettings,
+                emptyDataStreamFailureStoreSettings,
+                new IndexSettingProviders(Set.of()),
+                null
+            );
+            assertNotNull(response.getDataStreams());
+            assertThat(response.getDataStreams().size(), equalTo(1));
+            assertThat(response.getDataStreams().get(0).getIlmPolicy(), equalTo(dataStreamPolicy));
+            assertThat(response.getDataStreams().get(0).getIndexModeName(), equalTo(dataStreamIndexMode));
+        }
+    }
+
+    private static ClusterState getClusterStateWithDataStreamWithSettings(
+        ProjectId projectId,
+        Settings templateSettings,
+        Settings dataStreamSettings
+    ) {
+        String dataStreamName = "data-stream-1";
+        int numberOfBackingIndices = randomIntBetween(1, 5);
+        long currentTime = System.currentTimeMillis();
+        int replicas = 0;
+        boolean replicated = false;
+        ProjectMetadata.Builder builder = ProjectMetadata.builder(projectId);
+        builder.put(
+            "template_1",
+            ComposableIndexTemplate.builder()
+                .indexPatterns(List.of("*"))
+                .template(Template.builder().settings(templateSettings))
+                .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate())
+                .build()
+        );
+
+        List<IndexMetadata> backingIndices = new ArrayList<>();
+        for (int backingIndexNumber = 1; backingIndexNumber <= numberOfBackingIndices; backingIndexNumber++) {
+            backingIndices.add(
+                createIndexMetadata(
+                    getDefaultBackingIndexName(dataStreamName, backingIndexNumber, currentTime),
+                    true,
+                    templateSettings,
+                    replicas
+                )
+            );
+        }
+        List<IndexMetadata> allIndices = new ArrayList<>(backingIndices);
+
+        DataStream ds = DataStream.builder(
+            dataStreamName,
+            backingIndices.stream().map(IndexMetadata::getIndex).collect(Collectors.toList())
+        ).setGeneration(numberOfBackingIndices).setSettings(dataStreamSettings).setReplicated(replicated).build();
+        builder.put(ds);
+
+        for (IndexMetadata index : allIndices) {
+            builder.put(index, false);
+        }
+        return ClusterState.builder(new ClusterName("_name")).putProjectMetadata(builder.build()).build();
+    }
 }

server/src/internalClusterTest/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceIT.java

@@ -0,0 +1,78 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.cluster.metadata;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.admin.indices.create.CreateIndexClusterStateUpdateRequest;
+import org.elasticsearch.action.admin.indices.get.GetIndexRequest;
+import org.elasticsearch.action.admin.indices.get.GetIndexResponse;
+import org.elasticsearch.action.support.master.ShardsAcknowledgedResponse;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.test.ESIntegTestCase;
+
+import java.util.Locale;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class MetadataCreateIndexServiceIT extends ESIntegTestCase {
+
+    public void testRequestTemplateIsRespected() throws InterruptedException {
+        /*
+         * This test passes a template in the CreateIndexClusterStateUpdateRequest, and makes sure that the settings from that template
+         * are used when creating the index.
+         */
+        MetadataCreateIndexService metadataCreateIndexService = internalCluster().getCurrentMasterNodeInstance(
+            MetadataCreateIndexService.class
+        );
+        final String indexName = randomAlphaOfLength(20).toLowerCase(Locale.ROOT);
+        final int numberOfReplicas = randomIntBetween(1, 7);
+        CreateIndexClusterStateUpdateRequest request = new CreateIndexClusterStateUpdateRequest(
+            "testRequestTemplateIsRespected",
+            ProjectId.DEFAULT,
+            indexName,
+            randomAlphaOfLength(20)
+        );
+        request.setMatchingTemplate(
+            ComposableIndexTemplate.builder()
+                .template(Template.builder().settings(Settings.builder().put("index.number_of_replicas", numberOfReplicas)))
+                .build()
+        );
+        final CountDownLatch listenerCalledLatch = new CountDownLatch(1);
+        ActionListener<ShardsAcknowledgedResponse> listener = new ActionListener<>() {
+            @Override
+            public void onResponse(ShardsAcknowledgedResponse shardsAcknowledgedResponse) {
+                listenerCalledLatch.countDown();
+            }
+
+            @Override
+            public void onFailure(Exception e) {
+                logger.error(e);
+                listenerCalledLatch.countDown();
+            }
+        };
+
+        metadataCreateIndexService.createIndex(
+            TimeValue.THIRTY_SECONDS,
+            TimeValue.THIRTY_SECONDS,
+            TimeValue.THIRTY_SECONDS,
+            request,
+            listener
+        );
+        listenerCalledLatch.await(10, TimeUnit.SECONDS);
+        GetIndexResponse response = admin().indices()
+            .getIndex(new GetIndexRequest(TimeValue.THIRTY_SECONDS).indices(indexName))
+            .actionGet();
+        Settings settings = response.getSettings().get(indexName);
+        assertThat(settings.get("index.number_of_replicas"), equalTo(Integer.toString(numberOfReplicas)));
+    }
+}

server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverService.java

@@ -66,7 +66,6 @@
 
 import static org.elasticsearch.cluster.metadata.IndexAbstraction.Type.ALIAS;
 import static org.elasticsearch.cluster.metadata.IndexAbstraction.Type.DATA_STREAM;
-import static org.elasticsearch.cluster.metadata.MetadataCreateDataStreamService.lookupTemplateForDataStream;
 import static org.elasticsearch.cluster.metadata.MetadataIndexTemplateService.findV1Templates;
 import static org.elasticsearch.cluster.metadata.MetadataIndexTemplateService.findV2Template;
 import static org.elasticsearch.cluster.routing.allocation.allocator.AllocationActionListener.rerouteCompletionIsNotRequired;
@@ -325,7 +324,7 @@ private RolloverResult rolloverDataStream(
         final SystemDataStreamDescriptor systemDataStreamDescriptor;
         if (dataStream.isSystem() == false) {
             systemDataStreamDescriptor = null;
-            templateV2 = lookupTemplateForDataStream(dataStreamName, metadata);
+            templateV2 = dataStream.getEffectiveIndexTemplate(projectState.metadata());
         } else {
             systemDataStreamDescriptor = systemIndices.findMatchingDataStreamDescriptor(dataStreamName);
             if (systemDataStreamDescriptor == null) {

server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java

@@ -267,7 +267,7 @@ public static Template resolveTemplate(
         List<CompressedXContent> mappings = MetadataCreateIndexService.collectV2Mappings(
             null, // empty request mapping as the user can't specify any explicit mappings via the simulate api
             simulatedProject,
-            matchingTemplate,
+            template,
             xContentRegistry,
             simulatedIndexName
         );

server/src/main/java/org/elasticsearch/action/datastreams/GetDataStreamAction.java

@@ -213,6 +213,7 @@ public static class DataStreamInfo implements SimpleDiffable<DataStreamInfo>, To
 
             public static final ParseField STATUS_FIELD = new ParseField("status");
             public static final ParseField INDEX_TEMPLATE_FIELD = new ParseField("template");
+            public static final ParseField SETTINGS_FIELD = new ParseField("settings");
             public static final ParseField PREFER_ILM = new ParseField("prefer_ilm");
             public static final ParseField MANAGED_BY = new ParseField("managed_by");
             public static final ParseField NEXT_GENERATION_INDEX_MANAGED_BY = new ParseField("next_generation_managed_by");
@@ -416,7 +417,9 @@ public XContentBuilder toXContent(
                     builder.endArray();
                     builder.endObject();
                 }
-
+                builder.startObject(SETTINGS_FIELD.getPreferredName());
+                dataStream.getSettings().toXContent(builder, params);
+                builder.endObject();
                 builder.startObject(DataStream.FAILURE_STORE_FIELD.getPreferredName());
                 builder.field(FAILURE_STORE_ENABLED.getPreferredName(), failureStoreEffectivelyEnabled);
                 builder.field(DataStream.ROLLOVER_ON_WRITE_FIELD.getPreferredName(), dataStream.getFailureComponent().isRolloverOnWrite());