Add index mode to get data stream API (elastic#122486)

This commit adds the `index_mode` for both the data stream and each
backing index to the output of `GET /_data_stream`. An example looks
like:

```
{
  "data_streams" : [
    {
      "name" : "foo-things",
      "indices" : [
        {
          "index_name" : ".ds-foo-things-2025.02.13-000001",
          ...
          "index_mode" : "standard"
        }
      ],
      ...
      "index_mode" : "standard"
    },
    {
      "name" : "logs-foo-bar",
      "indices" : [
        {
          "index_name" : ".ds-logs-foo-bar-2025.02.13-000001",
          ...
          "index_mode" : "logsdb"
        },
        {
          "index_name" : ".ds-logs-foo-bar-2025.02.13-000002",
          ...
          "index_mode" : "logsdb"
        }
      ],
      ...
      "index_mode" : "logsdb",
    }
  ]
}
```

(cherry picked from commit 47706b5)

# Conflicts:
#	docs/reference/indices/get-data-stream.asciidoc
#	server/src/main/java/org/elasticsearch/TransportVersions.java
#	server/src/main/java/org/elasticsearch/action/datastreams/GetDataStreamAction.java
#	server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java
#	x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProvider.java
#	x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProviderTests.java

Author: dakrone

docs/changelog/122486.yaml

@@ -0,0 +1,5 @@
+pr: 122486
+summary: Add index mode to get data stream API
+area: Data streams
+type: enhancement
+issues: []

docs/reference/indices/get-data-stream.asciidoc

@@ -304,21 +304,24 @@ The API returns the following response:
           "index_uuid": "xCEhwsp8Tey0-FLNFYVwSg",
           "prefer_ilm": true,
           "ilm_policy": "my-lifecycle-policy",
-          "managed_by": "Index Lifecycle Management"
+          "managed_by": "Index Lifecycle Management",
+          "index_mode": "standard"
         },
         {
           "index_name": ".ds-my-data-stream-2099.03.08-000002",
           "index_uuid": "PA_JquKGSiKcAKBA8DJ5gw",
           "prefer_ilm": true,
           "ilm_policy": "my-lifecycle-policy",
-          "managed_by": "Index Lifecycle Management"
+          "managed_by": "Index Lifecycle Management",
+          "index_mode": "standard"
         }
       ],
       "generation": 2,
       "_meta": {
         "my-meta-field": "foo"
       },
       "status": "GREEN",
+      "index_mode": "standard",
       "next_generation_managed_by": "Index Lifecycle Management",
       "prefer_ilm": true,
       "template": "my-index-template",
@@ -340,14 +343,16 @@ The API returns the following response:
           "index_uuid": "3liBu2SYS5axasRt6fUIpA",
           "prefer_ilm": true,
           "ilm_policy": "my-lifecycle-policy",
-          "managed_by": "Index Lifecycle Management"
+          "managed_by": "Index Lifecycle Management",
+          "index_mode": "standard"
         }
       ],
       "generation": 1,
       "_meta": {
         "my-meta-field": "foo"
       },
       "status": "YELLOW",
+      "index_mode": "standard",
       "next_generation_managed_by": "Index Lifecycle Management",
       "prefer_ilm": true,
       "template": "my-index-template",

modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsAction.java

@@ -24,6 +24,7 @@
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.health.ClusterStateHealth;
+import org.elasticsearch.cluster.metadata.ComposableIndexTemplate;
 import org.elasticsearch.cluster.metadata.DataStream;
 import org.elasticsearch.cluster.metadata.DataStreamFailureStoreSettings;
 import org.elasticsearch.cluster.metadata.DataStreamGlobalRetentionSettings;
@@ -39,6 +40,9 @@
 import org.elasticsearch.core.Tuple;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexMode;
+import org.elasticsearch.index.IndexSettingProvider;
+import org.elasticsearch.index.IndexSettingProviders;
+import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.indices.SystemDataStreamDescriptor;
 import org.elasticsearch.indices.SystemIndices;
 import org.elasticsearch.injection.guice.Inject;
@@ -52,6 +56,7 @@
 import java.util.Comparator;
 import java.util.HashMap;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
 import java.util.stream.Collectors;
 
@@ -67,6 +72,7 @@ public class TransportGetDataStreamsAction extends TransportMasterNodeReadAction
     private final ClusterSettings clusterSettings;
     private final DataStreamGlobalRetentionSettings globalRetentionSettings;
     private final DataStreamFailureStoreSettings dataStreamFailureStoreSettings;
+    private final IndexSettingProviders indexSettingProviders;
     private final Client client;
 
     @Inject
@@ -79,6 +85,7 @@ public TransportGetDataStreamsAction(
         SystemIndices systemIndices,
         DataStreamGlobalRetentionSettings globalRetentionSettings,
         DataStreamFailureStoreSettings dataStreamFailureStoreSettings,
+        IndexSettingProviders indexSettingProviders,
         Client client
     ) {
         super(
@@ -96,6 +103,7 @@ public TransportGetDataStreamsAction(
         this.globalRetentionSettings = globalRetentionSettings;
         clusterSettings = clusterService.getClusterSettings();
         this.dataStreamFailureStoreSettings = dataStreamFailureStoreSettings;
+        this.indexSettingProviders = indexSettingProviders;
         this.client = new OriginSettingClient(client, "stack");
     }
 
@@ -128,6 +136,7 @@ public void onResponse(DataStreamsStatsAction.Response response) {
                             clusterSettings,
                             globalRetentionSettings,
                             dataStreamFailureStoreSettings,
+                            indexSettingProviders,
                             maxTimestamps
                         )
                     );
@@ -148,12 +157,43 @@ public void onFailure(Exception e) {
                     clusterSettings,
                     globalRetentionSettings,
                     dataStreamFailureStoreSettings,
+                    indexSettingProviders,
                     null
                 )
             );
         }
     }
 
+    /**
+     * Resolves the index mode ("index.mode" setting) for the given data stream, from the template or additional setting providers
+     */
+    @Nullable
+    static IndexMode resolveMode(
+        ClusterState state,
+        IndexSettingProviders indexSettingProviders,
+        DataStream dataStream,
+        Settings settings,
+        ComposableIndexTemplate indexTemplate
+    ) {
+        IndexMode indexMode = state.metadata().retrieveIndexModeFromTemplate(indexTemplate);
+        for (IndexSettingProvider provider : indexSettingProviders.getIndexSettingProviders()) {
+            Settings addlSettinsg = provider.getAdditionalIndexSettings(
+                MetadataIndexTemplateService.VALIDATE_INDEX_NAME,
+                dataStream.getName(),
+                indexMode,
+                state.metadata(),
+                Instant.now(),
+                settings,
+                List.of()
+            );
+            var rawMode = addlSettinsg.get(IndexSettings.MODE.getKey());
+            if (rawMode != null) {
+                indexMode = Enum.valueOf(IndexMode.class, rawMode.toUpperCase(Locale.ROOT));
+            }
+        }
+        return indexMode;
+    }
+
     static GetDataStreamAction.Response innerOperation(
         ClusterState state,
         GetDataStreamAction.Request request,
@@ -162,6 +202,7 @@ static GetDataStreamAction.Response innerOperation(
         ClusterSettings clusterSettings,
         DataStreamGlobalRetentionSettings globalRetentionSettings,
         DataStreamFailureStoreSettings dataStreamFailureStoreSettings,
+        IndexSettingProviders indexSettingProviders,
         @Nullable Map<String, Long> maxTimestamps
     ) {
         List<DataStream> dataStreams = getDataStreams(state, indexNameExpressionResolver, request);
@@ -173,6 +214,7 @@ static GetDataStreamAction.Response innerOperation(
             final String indexTemplate;
             boolean indexTemplatePreferIlmValue = true;
             String ilmPolicyName = null;
+            IndexMode indexMode = dataStream.getIndexMode();
             if (dataStream.isSystem()) {
                 SystemDataStreamDescriptor dataStreamDescriptor = systemIndices.findMatchingDataStreamDescriptor(dataStream.getName());
                 indexTemplate = dataStreamDescriptor != null ? dataStreamDescriptor.getDataStreamName() : null;
@@ -182,13 +224,31 @@ static GetDataStreamAction.Response innerOperation(
                         dataStreamDescriptor.getComponentTemplates()
                     );
                     ilmPolicyName = settings.get(IndexMetadata.LIFECYCLE_NAME);
+                    if (indexMode == null) {
+                        indexMode = resolveMode(
+                            state,
+                            indexSettingProviders,
+                            dataStream,
+                            settings,
+                            dataStreamDescriptor.getComposableIndexTemplate()
+                        );
+                    }
                     indexTemplatePreferIlmValue = PREFER_ILM_SETTING.get(settings);
                 }
             } else {
                 indexTemplate = MetadataIndexTemplateService.findV2Template(state.metadata(), dataStream.getName(), false);
                 if (indexTemplate != null) {
                     Settings settings = MetadataIndexTemplateService.resolveSettings(state.metadata(), indexTemplate);
                     ilmPolicyName = settings.get(IndexMetadata.LIFECYCLE_NAME);
+                    if (indexMode == null && state.metadata().templatesV2().get(indexTemplate) != null) {
+                        indexMode = resolveMode(
+                            state,
+                            indexSettingProviders,
+                            dataStream,
+                            settings,
+                            state.metadata().templatesV2().get(indexTemplate)
+                        );
+                    }
                     indexTemplatePreferIlmValue = PREFER_ILM_SETTING.get(settings);
                 } else {
                     LOGGER.warn(
@@ -280,7 +340,9 @@ public int compareTo(IndexInfo o) {
                     timeSeries,
                     backingIndicesSettingsValues,
                     indexTemplatePreferIlmValue,
-                    maxTimestamps == null ? null : maxTimestamps.get(dataStream.getName())
+                    maxTimestamps == null ? null : maxTimestamps.get(dataStream.getName()),
+                    // Default to standard mode if not specified; should we set this to "unset" or "unspecified" instead?
+                    indexMode == null ? IndexMode.STANDARD.getName() : indexMode.getName()
                 )
             );
         }
@@ -310,7 +372,11 @@ private static void collectIndexSettingsValues(
             } else {
                 managedBy = ManagedBy.UNMANAGED;
             }
-            backingIndicesSettingsValues.put(index, new IndexProperties(preferIlm, indexMetadata.getLifecyclePolicyName(), managedBy));
+            String indexMode = IndexSettings.MODE.get(indexMetadata.getSettings()).getName();
+            backingIndicesSettingsValues.put(
+                index,
+                new IndexProperties(preferIlm, indexMetadata.getLifecyclePolicyName(), managedBy, indexMode)
+            );
         }
     }
 

modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/GetDataStreamsResponseTests.java

@@ -92,13 +92,13 @@ public void testResponseIlmAndDataStreamLifecycleRepresentation() throws Excepti
             String ilmPolicyName = "rollover-30days";
             Map<Index, Response.IndexProperties> indexSettingsValues = Map.of(
                 firstGenerationIndex,
-                new Response.IndexProperties(true, ilmPolicyName, ManagedBy.ILM),
+                new Response.IndexProperties(true, ilmPolicyName, ManagedBy.ILM, null),
                 secondGenerationIndex,
-                new Response.IndexProperties(false, ilmPolicyName, ManagedBy.LIFECYCLE),
+                new Response.IndexProperties(false, ilmPolicyName, ManagedBy.LIFECYCLE, null),
                 writeIndex,
-                new Response.IndexProperties(false, null, ManagedBy.LIFECYCLE),
+                new Response.IndexProperties(false, null, ManagedBy.LIFECYCLE, null),
                 failureStoreIndex,
-                new Response.IndexProperties(false, null, ManagedBy.LIFECYCLE)
+                new Response.IndexProperties(false, null, ManagedBy.LIFECYCLE, null)
             );
 
             Response.DataStreamInfo dataStreamInfo = new Response.DataStreamInfo(
@@ -110,6 +110,7 @@ public void testResponseIlmAndDataStreamLifecycleRepresentation() throws Excepti
                 null,
                 indexSettingsValues,
                 false,
+                null,
                 null
             );
             Response response = new Response(List.of(dataStreamInfo));
@@ -191,13 +192,13 @@ public void testResponseIlmAndDataStreamLifecycleRepresentation() throws Excepti
             String ilmPolicyName = "rollover-30days";
             Map<Index, Response.IndexProperties> indexSettingsValues = Map.of(
                 firstGenerationIndex,
-                new Response.IndexProperties(true, ilmPolicyName, ManagedBy.ILM),
+                new Response.IndexProperties(true, ilmPolicyName, ManagedBy.ILM, null),
                 secondGenerationIndex,
-                new Response.IndexProperties(true, ilmPolicyName, ManagedBy.ILM),
+                new Response.IndexProperties(true, ilmPolicyName, ManagedBy.ILM, null),
                 writeIndex,
-                new Response.IndexProperties(false, null, ManagedBy.UNMANAGED),
+                new Response.IndexProperties(false, null, ManagedBy.UNMANAGED, null),
                 failureStoreIndex,
-                new Response.IndexProperties(false, null, ManagedBy.UNMANAGED)
+                new Response.IndexProperties(false, null, ManagedBy.UNMANAGED, null)
             );
 
             Response.DataStreamInfo dataStreamInfo = new Response.DataStreamInfo(
@@ -209,6 +210,7 @@ public void testResponseIlmAndDataStreamLifecycleRepresentation() throws Excepti
                 null,
                 indexSettingsValues,
                 false,
+                null,
                 null
             );
             Response response = new Response(List.of(dataStreamInfo));
@@ -359,7 +361,8 @@ private Response.DataStreamInfo mutateInstance(Response.DataStreamInfo instance)
                         new Response.IndexProperties(
                             randomBoolean(),
                             randomAlphaOfLengthBetween(50, 100),
-                            randomBoolean() ? ManagedBy.ILM : ManagedBy.LIFECYCLE
+                            randomBoolean() ? ManagedBy.ILM : ManagedBy.LIFECYCLE,
+                            null
                         )
                     )
             );
@@ -378,7 +381,8 @@ private Response.DataStreamInfo mutateInstance(Response.DataStreamInfo instance)
             timeSeries,
             indexSettings,
             templatePreferIlm,
-            maximumTimestamp
+            maximumTimestamp,
+            null
         );
     }
 
@@ -399,7 +403,8 @@ private Map<Index, Response.IndexProperties> generateRandomIndexSettingsValues()
                 new Response.IndexProperties(
                     randomBoolean(),
                     randomAlphaOfLengthBetween(50, 100),
-                    randomBoolean() ? ManagedBy.ILM : ManagedBy.LIFECYCLE
+                    randomBoolean() ? ManagedBy.ILM : ManagedBy.LIFECYCLE,
+                    randomBoolean() ? randomFrom(IndexMode.values()).getName() : null
                 )
             );
         }
@@ -417,7 +422,8 @@ private Response.DataStreamInfo generateRandomDataStreamInfo() {
             timeSeries != null ? new Response.TimeSeries(timeSeries) : null,
             generateRandomIndexSettingsValues(),
             randomBoolean(),
-            usually() ? randomNonNegativeLong() : null
+            usually() ? randomNonNegativeLong() : null,
+            usually() ? randomFrom(IndexMode.values()).getName() : null
         );
     }
 }