[8.19] Backport data stream settings

docs/reference/data-streams/change-mappings-and-settings.asciidoc

@@ -596,7 +596,8 @@ stream's oldest backing index.
       "allow_custom_routing": false,
       "replicated": false,
       "index_mode": "standard",
-      "rollover_on_write": false
+      "rollover_on_write": false,
+      "settings": {}
     }
   ]
 }

docs/reference/data-streams/downsampling-manual.asciidoc

@@ -382,7 +382,8 @@ This returns:
             "end": "2023-07-26T13:26:42.000Z"
           }
         ]
-      }
+      },
+      "settings": {}
     }
   ]
 }

docs/reference/data-streams/lifecycle/tutorial-migrate-data-stream-from-ilm-to-dsl.asciidoc

@@ -11,6 +11,16 @@ being managed by data stream lifecycle. As we'll see, {ilm-init} and data stream
 can co-manage a data stream; however, an index can only be managed by one system at
 a time.
 
+[source,console]
+--------------------------------------------------
+DELETE _data_stream/dsl-data-stream
+DELETE _index_template/dsl-data-stream-template
+DELETE _ilm/policy/pre-dsl-ilm-policy
+--------------------------------------------------
+// TEARDOWN
+
+//////////////////////////
+
 [discrete]
 [[migrate-dsl-ilm-tldr]]
 ==== TL;DR
@@ -141,7 +151,8 @@ and that the next generation index will also be managed by {ilm-init}:
       "allow_custom_routing": false,
       "replicated": false,
       "index_mode": "standard",
-      "rollover_on_write": false
+      "rollover_on_write": false,
+      "settings": {}
     }
   ]
 }
@@ -283,7 +294,8 @@ GET _data_stream/dsl-data-stream
       "allow_custom_routing": false,
       "replicated": false,
       "index_mode": "standard",
-      "rollover_on_write": false
+      "rollover_on_write": false,
+      "settings": {}
     }
   ]
 }
@@ -367,7 +379,8 @@ GET _data_stream/dsl-data-stream
       "allow_custom_routing": false,
       "replicated": false,
       "index_mode": "standard",
-      "rollover_on_write": false
+      "rollover_on_write": false,
+      "settings": {}
     }
   ]
 }
@@ -469,7 +482,8 @@ GET _data_stream/dsl-data-stream
       "allow_custom_routing": false,
       "replicated": false,
       "index_mode": "standard",
-      "rollover_on_write": false
+      "rollover_on_write": false,
+      "settings": {}
     }
   ]
 }
@@ -488,15 +502,3 @@ GET _data_stream/dsl-data-stream
 Had we removed the {ilm-init} policy from the index template when we <<update-index-template-for-dsl, updated>>
 it, the write index of the data stream will now be `Unmanaged` because the index
 wouldn't have the {ilm-init} policy configured to fallback onto.
-
-//////////////////////////
-[source,console]
---------------------------------------------------
-DELETE _data_stream/dsl-data-stream
-DELETE _index_template/dsl-data-stream-template
-DELETE _ilm/policy/pre-dsl-ilm-policy
---------------------------------------------------
-// TEST[continued]
-
-//////////////////////////
-

docs/reference/indices/get-data-stream.asciidoc

@@ -330,7 +330,8 @@ The API returns the following response:
       "system": false,
       "allow_custom_routing": false,
       "replicated": false,
-      "rollover_on_write": false
+      "rollover_on_write": false,
+      "settings": {}
     },
     {
       "name": "my-data-stream-two",
@@ -361,7 +362,8 @@ The API returns the following response:
       "system": false,
       "allow_custom_routing": false,
       "replicated": false,
-      "rollover_on_write": false
+      "rollover_on_write": false,
+      "settings": {}
     }
   ]
 }

modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamFeatures.java

@@ -33,6 +33,8 @@ public class DataStreamFeatures implements FeatureSpecification {
         "data_stream.downsample.default_aggregate_metric_fix"
     );
 
+    public static final NodeFeature LOGS_STREAM_FEATURE = new NodeFeature("logs_stream");
+
     @Override
     public Map<NodeFeature, Version> getHistoricalFeatures() {
         return Map.of(DATA_STREAM_LIFECYCLE, Version.V_8_11_0);
@@ -51,6 +53,6 @@ public Set<NodeFeature> getFeatures() {
 
     @Override
     public Set<NodeFeature> getTestFeatures() {
-        return Set.of(DATA_STREAM_FAILURE_STORE_TSDB_FIX, DOWNSAMPLE_AGGREGATE_DEFAULT_METRIC_FIX);
+        return Set.of(DATA_STREAM_FAILURE_STORE_TSDB_FIX, DOWNSAMPLE_AGGREGATE_DEFAULT_METRIC_FIX, LOGS_STREAM_FEATURE);
     }
 }

modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamsPlugin.java

@@ -16,10 +16,12 @@
 import org.elasticsearch.action.datastreams.DataStreamsStatsAction;
 import org.elasticsearch.action.datastreams.DeleteDataStreamAction;
 import org.elasticsearch.action.datastreams.GetDataStreamAction;
+import org.elasticsearch.action.datastreams.GetDataStreamSettingsAction;
 import org.elasticsearch.action.datastreams.MigrateToDataStreamAction;
 import org.elasticsearch.action.datastreams.ModifyDataStreamsAction;
 import org.elasticsearch.action.datastreams.PromoteDataStreamAction;
 import org.elasticsearch.action.datastreams.PutDataStreamOptionsAction;
+import org.elasticsearch.action.datastreams.UpdateDataStreamSettingsAction;
 import org.elasticsearch.action.datastreams.lifecycle.ExplainDataStreamLifecycleAction;
 import org.elasticsearch.action.datastreams.lifecycle.GetDataStreamLifecycleAction;
 import org.elasticsearch.action.datastreams.lifecycle.PutDataStreamLifecycleAction;
@@ -37,10 +39,12 @@
 import org.elasticsearch.datastreams.action.TransportCreateDataStreamAction;
 import org.elasticsearch.datastreams.action.TransportDataStreamsStatsAction;
 import org.elasticsearch.datastreams.action.TransportDeleteDataStreamAction;
+import org.elasticsearch.datastreams.action.TransportGetDataStreamSettingsAction;
 import org.elasticsearch.datastreams.action.TransportGetDataStreamsAction;
 import org.elasticsearch.datastreams.action.TransportMigrateToDataStreamAction;
 import org.elasticsearch.datastreams.action.TransportModifyDataStreamsAction;
 import org.elasticsearch.datastreams.action.TransportPromoteDataStreamAction;
+import org.elasticsearch.datastreams.action.TransportUpdateDataStreamSettingsAction;
 import org.elasticsearch.datastreams.lifecycle.DataStreamLifecycleErrorStore;
 import org.elasticsearch.datastreams.lifecycle.DataStreamLifecycleService;
 import org.elasticsearch.datastreams.lifecycle.action.DeleteDataStreamLifecycleAction;
@@ -68,10 +72,12 @@
 import org.elasticsearch.datastreams.rest.RestCreateDataStreamAction;
 import org.elasticsearch.datastreams.rest.RestDataStreamsStatsAction;
 import org.elasticsearch.datastreams.rest.RestDeleteDataStreamAction;
+import org.elasticsearch.datastreams.rest.RestGetDataStreamSettingsAction;
 import org.elasticsearch.datastreams.rest.RestGetDataStreamsAction;
 import org.elasticsearch.datastreams.rest.RestMigrateToDataStreamAction;
 import org.elasticsearch.datastreams.rest.RestModifyDataStreamsAction;
 import org.elasticsearch.datastreams.rest.RestPromoteDataStreamAction;
+import org.elasticsearch.datastreams.rest.RestUpdateDataStreamSettingsAction;
 import org.elasticsearch.features.NodeFeature;
 import org.elasticsearch.health.HealthIndicatorService;
 import org.elasticsearch.index.IndexSettingProvider;
@@ -241,6 +247,8 @@ public Collection<?> createComponents(PluginServices services) {
         actions.add(new ActionHandler<>(GetDataStreamOptionsAction.INSTANCE, TransportGetDataStreamOptionsAction.class));
         actions.add(new ActionHandler<>(PutDataStreamOptionsAction.INSTANCE, TransportPutDataStreamOptionsAction.class));
         actions.add(new ActionHandler<>(DeleteDataStreamOptionsAction.INSTANCE, TransportDeleteDataStreamOptionsAction.class));
+        actions.add(new ActionHandler<>(GetDataStreamSettingsAction.INSTANCE, TransportGetDataStreamSettingsAction.class));
+        actions.add(new ActionHandler<>(UpdateDataStreamSettingsAction.INSTANCE, TransportUpdateDataStreamSettingsAction.class));
         return actions;
     }
 
@@ -276,6 +284,8 @@ public List<RestHandler> getRestHandlers(
         handlers.add(new RestGetDataStreamOptionsAction());
         handlers.add(new RestPutDataStreamOptionsAction());
         handlers.add(new RestDeleteDataStreamOptionsAction());
+        handlers.add(new RestGetDataStreamSettingsAction());
+        handlers.add(new RestUpdateDataStreamSettingsAction());
         return handlers;
     }
 

modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/TransportGetDataStreamSettingsAction.java

@@ -0,0 +1,90 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.datastreams.action;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.datastreams.GetDataStreamSettingsAction;
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.action.support.master.TransportMasterNodeReadAction;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.block.ClusterBlockException;
+import org.elasticsearch.cluster.block.ClusterBlockLevel;
+import org.elasticsearch.cluster.metadata.DataStream;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.settings.SettingsFilter;
+import org.elasticsearch.injection.guice.Inject;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.TransportService;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+public class TransportGetDataStreamSettingsAction extends TransportMasterNodeReadAction<
+    GetDataStreamSettingsAction.Request,
+    GetDataStreamSettingsAction.Response> {
+    private final IndexNameExpressionResolver indexNameExpressionResolver;
+    private final SettingsFilter settingsFilter;
+
+    @Inject
+    public TransportGetDataStreamSettingsAction(
+        TransportService transportService,
+        ClusterService clusterService,
+        ThreadPool threadPool,
+        SettingsFilter settingsFilter,
+        ActionFilters actionFilters,
+        IndexNameExpressionResolver indexNameExpressionResolver
+    ) {
+        super(
+            GetDataStreamSettingsAction.NAME,
+            transportService,
+            clusterService,
+            threadPool,
+            actionFilters,
+            GetDataStreamSettingsAction.Request::localOnly,
+            GetDataStreamSettingsAction.Response::localOnly,
+            threadPool.executor(ThreadPool.Names.MANAGEMENT)
+        );
+        this.indexNameExpressionResolver = indexNameExpressionResolver;
+        this.settingsFilter = settingsFilter;
+    }
+
+    @Override
+    protected ClusterBlockException checkBlock(GetDataStreamSettingsAction.Request request, ClusterState state) {
+        return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE);
+    }
+
+    @Override
+    protected void masterOperation(
+        Task task,
+        GetDataStreamSettingsAction.Request request,
+        ClusterState state,
+        ActionListener<GetDataStreamSettingsAction.Response> listener
+    ) throws Exception {
+        List<String> dataStreamNames = indexNameExpressionResolver.dataStreamNames(
+            clusterService.state(),
+            IndicesOptions.DEFAULT,
+            request.indices()
+        );
+        Map<String, DataStream> dataStreamMap = state.metadata().dataStreams();
+        List<GetDataStreamSettingsAction.DataStreamSettingsResponse> responseList = new ArrayList<>(dataStreamNames.size());
+        for (String dataStreamName : dataStreamNames) {
+            DataStream dataStream = dataStreamMap.get(dataStreamName);
+            Settings settings = settingsFilter.filter(dataStream.getSettings());
+            Settings effectiveSettings = settingsFilter.filter(dataStream.getEffectiveSettings(state.metadata()));
+            responseList.add(new GetDataStreamSettingsAction.DataStreamSettingsResponse(dataStreamName, settings, effectiveSettings));
+        }
+        listener.onResponse(new GetDataStreamSettingsAction.Response(responseList));
+    }
+}

modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsAction.java

@@ -238,15 +238,15 @@ static GetDataStreamAction.Response innerOperation(
             } else {
                 indexTemplate = MetadataIndexTemplateService.findV2Template(state.metadata(), dataStream.getName(), false);
                 if (indexTemplate != null) {
-                    Settings settings = MetadataIndexTemplateService.resolveSettings(state.metadata(), indexTemplate);
+                    Settings settings = dataStream.getEffectiveSettings(state.metadata());
                     ilmPolicyName = settings.get(IndexMetadata.LIFECYCLE_NAME);
                     if (indexMode == null && state.metadata().templatesV2().get(indexTemplate) != null) {
                         indexMode = resolveMode(
                             state,
                             indexSettingProviders,
                             dataStream,
                             settings,
-                            state.metadata().templatesV2().get(indexTemplate)
+                            dataStream.getEffectiveIndexTemplate(state.metadata())
                         );
                     }
                     indexTemplatePreferIlmValue = PREFER_ILM_SETTING.get(settings);