[8.19] Add index mode to get data stream API (#122486)

docs/changelog/122486.yaml

@@ -0,0 +1,5 @@
+pr: 122486
+summary: Add index mode to get data stream API
+area: Data streams
+type: enhancement
+issues: []

docs/reference/data-streams/change-mappings-and-settings.asciidoc

@@ -575,13 +575,15 @@ stream's oldest backing index.
           "index_name": ".ds-my-data-stream-2099.03.07-000001", <1>
           "index_uuid": "Gpdiyq8sRuK9WuthvAdFbw",
           "prefer_ilm": true,
-          "managed_by": "Unmanaged"
+          "managed_by": "Unmanaged",
+          "index_mode" : "standard"
         },
         {
           "index_name": ".ds-my-data-stream-2099.03.08-000002",
           "index_uuid": "_eEfRrFHS9OyhqWntkgHAQ",
           "prefer_ilm": true,
-          "managed_by": "Unmanaged"
+          "managed_by": "Unmanaged",
+          "index_mode" : "standard"
         }
       ],
       "generation": 2,
@@ -593,6 +595,7 @@ stream's oldest backing index.
       "system": false,
       "allow_custom_routing": false,
       "replicated": false,
+      "index_mode": "standard",
       "rollover_on_write": false
     }
   ]

docs/reference/data-streams/downsampling-manual.asciidoc

@@ -360,6 +360,7 @@ This returns:
           "index_name": ".ds-my-data-stream-2023.07.26-000001", <1>
           "index_uuid": "ltOJGmqgTVm4T-Buoe7Acg",
           "prefer_ilm": true,
+          "index_mode": "time_series",
           "managed_by": "Unmanaged"
         }
       ],
@@ -373,6 +374,7 @@ This returns:
       "allow_custom_routing": false,
       "replicated": false,
       "rollover_on_write": false,
+      "index_mode": "time_series",
       "time_series": {
         "temporal_ranges": [
           {

docs/reference/data-streams/lifecycle/tutorial-migrate-data-stream-from-ilm-to-dsl.asciidoc

@@ -118,14 +118,16 @@ and that the next generation index will also be managed by {ilm-init}:
           "index_uuid": "xCEhwsp8Tey0-FLNFYVwSg",
           "prefer_ilm": true,                                       <2>
           "ilm_policy": "pre-dsl-ilm-policy",                       <3>
-          "managed_by": "Index Lifecycle Management"                <4>
+          "managed_by": "Index Lifecycle Management",               <4>
+          "index_mode": "standard"
         },
         {
           "index_name": ".ds-dsl-data-stream-2023.10.19-000002",
           "index_uuid": "PA_JquKGSiKcAKBA8DJ5gw",
           "prefer_ilm": true,
           "ilm_policy": "pre-dsl-ilm-policy",
-          "managed_by": "Index Lifecycle Management"
+          "managed_by": "Index Lifecycle Management",
+          "index_mode": "standard"
         }
       ],
       "generation": 2,
@@ -138,6 +140,7 @@ and that the next generation index will also be managed by {ilm-init}:
       "system": false,
       "allow_custom_routing": false,
       "replicated": false,
+      "index_mode": "standard",
       "rollover_on_write": false
     }
   ]
@@ -251,14 +254,16 @@ GET _data_stream/dsl-data-stream
           "index_uuid": "xCEhwsp8Tey0-FLNFYVwSg",
           "prefer_ilm": true,
           "ilm_policy": "pre-dsl-ilm-policy",
-          "managed_by": "Index Lifecycle Management"                <1>
+          "managed_by": "Index Lifecycle Management",               <1>
+          "index_mode": "standard"
         },
         {
           "index_name": ".ds-dsl-data-stream-2023.10.19-000002",
           "index_uuid": "PA_JquKGSiKcAKBA8DJ5gw",
           "prefer_ilm": true,
           "ilm_policy": "pre-dsl-ilm-policy",
-          "managed_by": "Index Lifecycle Management"                <2>
+          "managed_by": "Index Lifecycle Management",               <2>
+          "index_mode": "standard"
         }
       ],
       "generation": 2,
@@ -277,6 +282,7 @@ GET _data_stream/dsl-data-stream
       "system": false,
       "allow_custom_routing": false,
       "replicated": false,
+      "index_mode": "standard",
       "rollover_on_write": false
     }
   ]
@@ -324,20 +330,23 @@ GET _data_stream/dsl-data-stream
           "index_uuid": "xCEhwsp8Tey0-FLNFYVwSg",
           "prefer_ilm": true,
           "ilm_policy": "pre-dsl-ilm-policy",
+          "index_mode": "standard",
           "managed_by": "Index Lifecycle Management"                <1>
         },
         {
           "index_name": ".ds-dsl-data-stream-2023.10.19-000002",
           "index_uuid": "PA_JquKGSiKcAKBA8DJ5gw",
           "prefer_ilm": true,
           "ilm_policy": "pre-dsl-ilm-policy",
+          "index_mode": "standard",
           "managed_by": "Index Lifecycle Management"                <2>
         },
         {
           "index_name": ".ds-dsl-data-stream-2023.10.19-000003",
           "index_uuid": "PA_JquKGSiKcAKBA8abcd1",
           "prefer_ilm": false,                                      <3>
           "ilm_policy": "pre-dsl-ilm-policy",
+          "index_mode": "standard",
           "managed_by": "Data stream lifecycle"                     <4>
         }
       ],
@@ -357,6 +366,7 @@ GET _data_stream/dsl-data-stream
       "system": false,
       "allow_custom_routing": false,
       "replicated": false,
+      "index_mode": "standard",
       "rollover_on_write": false
     }
   ]
@@ -424,20 +434,23 @@ GET _data_stream/dsl-data-stream
           "index_uuid": "xCEhwsp8Tey0-FLNFYVwSg",
           "prefer_ilm": true,
           "ilm_policy": "pre-dsl-ilm-policy",
+          "index_mode": "standard",
           "managed_by": "Index Lifecycle Management"
         },
         {
           "index_name": ".ds-dsl-data-stream-2023.10.19-000002",
           "index_uuid": "PA_JquKGSiKcAKBA8DJ5gw",
           "prefer_ilm": true,
           "ilm_policy": "pre-dsl-ilm-policy",
+          "index_mode": "standard",
           "managed_by": "Index Lifecycle Management"
         },
         {
           "index_name": ".ds-dsl-data-stream-2023.10.19-000003",
           "index_uuid": "PA_JquKGSiKcAKBA8abcd1",
           "prefer_ilm": false,
           "ilm_policy": "pre-dsl-ilm-policy",
+          "index_mode": "standard",
           "managed_by": "Index Lifecycle Management"                <1>
         }
       ],
@@ -455,6 +468,7 @@ GET _data_stream/dsl-data-stream
       "system": false,
       "allow_custom_routing": false,
       "replicated": false,
+      "index_mode": "standard",
       "rollover_on_write": false
     }
   ]

docs/reference/indices/get-data-stream.asciidoc

@@ -304,21 +304,24 @@ The API returns the following response:
           "index_uuid": "xCEhwsp8Tey0-FLNFYVwSg",
           "prefer_ilm": true,
           "ilm_policy": "my-lifecycle-policy",
-          "managed_by": "Index Lifecycle Management"
+          "managed_by": "Index Lifecycle Management",
+          "index_mode": "standard"
         },
         {
           "index_name": ".ds-my-data-stream-2099.03.08-000002",
           "index_uuid": "PA_JquKGSiKcAKBA8DJ5gw",
           "prefer_ilm": true,
           "ilm_policy": "my-lifecycle-policy",
-          "managed_by": "Index Lifecycle Management"
+          "managed_by": "Index Lifecycle Management",
+          "index_mode": "standard"
         }
       ],
       "generation": 2,
       "_meta": {
         "my-meta-field": "foo"
       },
       "status": "GREEN",
+      "index_mode": "standard",
       "next_generation_managed_by": "Index Lifecycle Management",
       "prefer_ilm": true,
       "template": "my-index-template",
@@ -340,14 +343,16 @@ The API returns the following response:
           "index_uuid": "3liBu2SYS5axasRt6fUIpA",
           "prefer_ilm": true,
           "ilm_policy": "my-lifecycle-policy",
-          "managed_by": "Index Lifecycle Management"
+          "managed_by": "Index Lifecycle Management",
+          "index_mode": "standard"
         }
       ],
       "generation": 1,
       "_meta": {
         "my-meta-field": "foo"
       },
       "status": "YELLOW",
+      "index_mode": "standard",
       "next_generation_managed_by": "Index Lifecycle Management",
       "prefer_ilm": true,
       "template": "my-index-template",

modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsAction.java

@@ -24,6 +24,7 @@
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.health.ClusterStateHealth;
+import org.elasticsearch.cluster.metadata.ComposableIndexTemplate;
 import org.elasticsearch.cluster.metadata.DataStream;
 import org.elasticsearch.cluster.metadata.DataStreamFailureStoreSettings;
 import org.elasticsearch.cluster.metadata.DataStreamGlobalRetentionSettings;
@@ -39,6 +40,9 @@
 import org.elasticsearch.core.Tuple;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexMode;
+import org.elasticsearch.index.IndexSettingProvider;
+import org.elasticsearch.index.IndexSettingProviders;
+import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.indices.SystemDataStreamDescriptor;
 import org.elasticsearch.indices.SystemIndices;
 import org.elasticsearch.injection.guice.Inject;
@@ -52,6 +56,7 @@
 import java.util.Comparator;
 import java.util.HashMap;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
 import java.util.stream.Collectors;
 
@@ -67,6 +72,7 @@ public class TransportGetDataStreamsAction extends TransportMasterNodeReadAction
     private final ClusterSettings clusterSettings;
     private final DataStreamGlobalRetentionSettings globalRetentionSettings;
     private final DataStreamFailureStoreSettings dataStreamFailureStoreSettings;
+    private final IndexSettingProviders indexSettingProviders;
     private final Client client;
 
     @Inject
@@ -79,6 +85,7 @@ public TransportGetDataStreamsAction(
         SystemIndices systemIndices,
         DataStreamGlobalRetentionSettings globalRetentionSettings,
         DataStreamFailureStoreSettings dataStreamFailureStoreSettings,
+        IndexSettingProviders indexSettingProviders,
         Client client
     ) {
         super(
@@ -96,6 +103,7 @@ public TransportGetDataStreamsAction(
         this.globalRetentionSettings = globalRetentionSettings;
         clusterSettings = clusterService.getClusterSettings();
         this.dataStreamFailureStoreSettings = dataStreamFailureStoreSettings;
+        this.indexSettingProviders = indexSettingProviders;
         this.client = new OriginSettingClient(client, "stack");
     }
 
@@ -128,6 +136,7 @@ public void onResponse(DataStreamsStatsAction.Response response) {
                             clusterSettings,
                             globalRetentionSettings,
                             dataStreamFailureStoreSettings,
+                            indexSettingProviders,
                             maxTimestamps
                         )
                     );
@@ -148,12 +157,43 @@ public void onFailure(Exception e) {
                     clusterSettings,
                     globalRetentionSettings,
                     dataStreamFailureStoreSettings,
+                    indexSettingProviders,
                     null
                 )
             );
         }
     }
 
+    /**
+     * Resolves the index mode ("index.mode" setting) for the given data stream, from the template or additional setting providers
+     */
+    @Nullable
+    static IndexMode resolveMode(
+        ClusterState state,
+        IndexSettingProviders indexSettingProviders,
+        DataStream dataStream,
+        Settings settings,
+        ComposableIndexTemplate indexTemplate
+    ) {
+        IndexMode indexMode = state.metadata().retrieveIndexModeFromTemplate(indexTemplate);
+        for (IndexSettingProvider provider : indexSettingProviders.getIndexSettingProviders()) {
+            Settings addlSettinsg = provider.getAdditionalIndexSettings(
+                MetadataIndexTemplateService.VALIDATE_INDEX_NAME,
+                dataStream.getName(),
+                indexMode,
+                state.metadata(),
+                Instant.now(),
+                settings,
+                List.of()
+            );
+            var rawMode = addlSettinsg.get(IndexSettings.MODE.getKey());
+            if (rawMode != null) {
+                indexMode = Enum.valueOf(IndexMode.class, rawMode.toUpperCase(Locale.ROOT));
+            }
+        }
+        return indexMode;
+    }
+
     static GetDataStreamAction.Response innerOperation(
         ClusterState state,
         GetDataStreamAction.Request request,
@@ -162,6 +202,7 @@ static GetDataStreamAction.Response innerOperation(
         ClusterSettings clusterSettings,
         DataStreamGlobalRetentionSettings globalRetentionSettings,
         DataStreamFailureStoreSettings dataStreamFailureStoreSettings,
+        IndexSettingProviders indexSettingProviders,
         @Nullable Map<String, Long> maxTimestamps
     ) {
         List<DataStream> dataStreams = getDataStreams(state, indexNameExpressionResolver, request);
@@ -173,6 +214,7 @@ static GetDataStreamAction.Response innerOperation(
             final String indexTemplate;
             boolean indexTemplatePreferIlmValue = true;
             String ilmPolicyName = null;
+            IndexMode indexMode = dataStream.getIndexMode();
             if (dataStream.isSystem()) {
                 SystemDataStreamDescriptor dataStreamDescriptor = systemIndices.findMatchingDataStreamDescriptor(dataStream.getName());
                 indexTemplate = dataStreamDescriptor != null ? dataStreamDescriptor.getDataStreamName() : null;
@@ -182,13 +224,31 @@ static GetDataStreamAction.Response innerOperation(
                         dataStreamDescriptor.getComponentTemplates()
                     );
                     ilmPolicyName = settings.get(IndexMetadata.LIFECYCLE_NAME);
+                    if (indexMode == null) {
+                        indexMode = resolveMode(
+                            state,
+                            indexSettingProviders,
+                            dataStream,
+                            settings,
+                            dataStreamDescriptor.getComposableIndexTemplate()
+                        );
+                    }
                     indexTemplatePreferIlmValue = PREFER_ILM_SETTING.get(settings);
                 }
             } else {
                 indexTemplate = MetadataIndexTemplateService.findV2Template(state.metadata(), dataStream.getName(), false);
                 if (indexTemplate != null) {
                     Settings settings = MetadataIndexTemplateService.resolveSettings(state.metadata(), indexTemplate);
                     ilmPolicyName = settings.get(IndexMetadata.LIFECYCLE_NAME);
+                    if (indexMode == null && state.metadata().templatesV2().get(indexTemplate) != null) {
+                        indexMode = resolveMode(
+                            state,
+                            indexSettingProviders,
+                            dataStream,
+                            settings,
+                            state.metadata().templatesV2().get(indexTemplate)
+                        );
+                    }
                     indexTemplatePreferIlmValue = PREFER_ILM_SETTING.get(settings);
                 } else {
                     LOGGER.warn(
@@ -280,7 +340,9 @@ public int compareTo(IndexInfo o) {
                     timeSeries,
                     backingIndicesSettingsValues,
                     indexTemplatePreferIlmValue,
-                    maxTimestamps == null ? null : maxTimestamps.get(dataStream.getName())
+                    maxTimestamps == null ? null : maxTimestamps.get(dataStream.getName()),
+                    // Default to standard mode if not specified; should we set this to "unset" or "unspecified" instead?
+                    indexMode == null ? IndexMode.STANDARD.getName() : indexMode.getName()
                 )
             );
         }
@@ -310,7 +372,11 @@ private static void collectIndexSettingsValues(
             } else {
                 managedBy = ManagedBy.UNMANAGED;
             }
-            backingIndicesSettingsValues.put(index, new IndexProperties(preferIlm, indexMetadata.getLifecyclePolicyName(), managedBy));
+            String indexMode = IndexSettings.MODE.get(indexMetadata.getSettings()).getName();
+            backingIndicesSettingsValues.put(
+                index,
+                new IndexProperties(preferIlm, indexMetadata.getLifecyclePolicyName(), managedBy, indexMode)
+            );
         }
     }