diff --git a/docs/changelog/122486.yaml b/docs/changelog/122486.yaml new file mode 100644 index 0000000000000..027d2a5e63ba3 --- /dev/null +++ b/docs/changelog/122486.yaml @@ -0,0 +1,5 @@ +pr: 122486 +summary: Add index mode to get data stream API +area: Data streams +type: enhancement +issues: [] diff --git a/docs/reference/data-streams/change-mappings-and-settings.asciidoc b/docs/reference/data-streams/change-mappings-and-settings.asciidoc index 1290f289e5bbd..e07e079519377 100644 --- a/docs/reference/data-streams/change-mappings-and-settings.asciidoc +++ b/docs/reference/data-streams/change-mappings-and-settings.asciidoc @@ -575,13 +575,15 @@ stream's oldest backing index. "index_name": ".ds-my-data-stream-2099.03.07-000001", <1> "index_uuid": "Gpdiyq8sRuK9WuthvAdFbw", "prefer_ilm": true, - "managed_by": "Unmanaged" + "managed_by": "Unmanaged", + "index_mode" : "standard" }, { "index_name": ".ds-my-data-stream-2099.03.08-000002", "index_uuid": "_eEfRrFHS9OyhqWntkgHAQ", "prefer_ilm": true, - "managed_by": "Unmanaged" + "managed_by": "Unmanaged", + "index_mode" : "standard" } ], "generation": 2, @@ -593,6 +595,7 @@ stream's oldest backing index. "system": false, "allow_custom_routing": false, "replicated": false, + "index_mode": "standard", "rollover_on_write": false } ] diff --git a/docs/reference/data-streams/downsampling-manual.asciidoc b/docs/reference/data-streams/downsampling-manual.asciidoc index 44ae77d072034..cb170a7596701 100644 --- a/docs/reference/data-streams/downsampling-manual.asciidoc +++ b/docs/reference/data-streams/downsampling-manual.asciidoc @@ -360,6 +360,7 @@ This returns: "index_name": ".ds-my-data-stream-2023.07.26-000001", <1> "index_uuid": "ltOJGmqgTVm4T-Buoe7Acg", "prefer_ilm": true, + "index_mode": "time_series", "managed_by": "Unmanaged" } ], @@ -373,6 +374,7 @@ This returns: "allow_custom_routing": false, "replicated": false, "rollover_on_write": false, + "index_mode": "time_series", "time_series": { "temporal_ranges": [ { diff --git a/docs/reference/data-streams/lifecycle/tutorial-migrate-data-stream-from-ilm-to-dsl.asciidoc b/docs/reference/data-streams/lifecycle/tutorial-migrate-data-stream-from-ilm-to-dsl.asciidoc index 13ee6935f7eb5..c56bf218708bc 100644 --- a/docs/reference/data-streams/lifecycle/tutorial-migrate-data-stream-from-ilm-to-dsl.asciidoc +++ b/docs/reference/data-streams/lifecycle/tutorial-migrate-data-stream-from-ilm-to-dsl.asciidoc @@ -118,14 +118,16 @@ and that the next generation index will also be managed by {ilm-init}: "index_uuid": "xCEhwsp8Tey0-FLNFYVwSg", "prefer_ilm": true, <2> "ilm_policy": "pre-dsl-ilm-policy", <3> - "managed_by": "Index Lifecycle Management" <4> + "managed_by": "Index Lifecycle Management", <4> + "index_mode": "standard" }, { "index_name": ".ds-dsl-data-stream-2023.10.19-000002", "index_uuid": "PA_JquKGSiKcAKBA8DJ5gw", "prefer_ilm": true, "ilm_policy": "pre-dsl-ilm-policy", - "managed_by": "Index Lifecycle Management" + "managed_by": "Index Lifecycle Management", + "index_mode": "standard" } ], "generation": 2, @@ -138,6 +140,7 @@ and that the next generation index will also be managed by {ilm-init}: "system": false, "allow_custom_routing": false, "replicated": false, + "index_mode": "standard", "rollover_on_write": false } ] @@ -251,14 +254,16 @@ GET _data_stream/dsl-data-stream "index_uuid": "xCEhwsp8Tey0-FLNFYVwSg", "prefer_ilm": true, "ilm_policy": "pre-dsl-ilm-policy", - "managed_by": "Index Lifecycle Management" <1> + "managed_by": "Index Lifecycle Management", <1> + "index_mode": "standard" }, { "index_name": ".ds-dsl-data-stream-2023.10.19-000002", "index_uuid": "PA_JquKGSiKcAKBA8DJ5gw", "prefer_ilm": true, "ilm_policy": "pre-dsl-ilm-policy", - "managed_by": "Index Lifecycle Management" <2> + "managed_by": "Index Lifecycle Management", <2> + "index_mode": "standard" } ], "generation": 2, @@ -277,6 +282,7 @@ GET _data_stream/dsl-data-stream "system": false, "allow_custom_routing": false, "replicated": false, + "index_mode": "standard", "rollover_on_write": false } ] @@ -324,6 +330,7 @@ GET _data_stream/dsl-data-stream "index_uuid": "xCEhwsp8Tey0-FLNFYVwSg", "prefer_ilm": true, "ilm_policy": "pre-dsl-ilm-policy", + "index_mode": "standard", "managed_by": "Index Lifecycle Management" <1> }, { @@ -331,6 +338,7 @@ GET _data_stream/dsl-data-stream "index_uuid": "PA_JquKGSiKcAKBA8DJ5gw", "prefer_ilm": true, "ilm_policy": "pre-dsl-ilm-policy", + "index_mode": "standard", "managed_by": "Index Lifecycle Management" <2> }, { @@ -338,6 +346,7 @@ GET _data_stream/dsl-data-stream "index_uuid": "PA_JquKGSiKcAKBA8abcd1", "prefer_ilm": false, <3> "ilm_policy": "pre-dsl-ilm-policy", + "index_mode": "standard", "managed_by": "Data stream lifecycle" <4> } ], @@ -357,6 +366,7 @@ GET _data_stream/dsl-data-stream "system": false, "allow_custom_routing": false, "replicated": false, + "index_mode": "standard", "rollover_on_write": false } ] @@ -424,6 +434,7 @@ GET _data_stream/dsl-data-stream "index_uuid": "xCEhwsp8Tey0-FLNFYVwSg", "prefer_ilm": true, "ilm_policy": "pre-dsl-ilm-policy", + "index_mode": "standard", "managed_by": "Index Lifecycle Management" }, { @@ -431,6 +442,7 @@ GET _data_stream/dsl-data-stream "index_uuid": "PA_JquKGSiKcAKBA8DJ5gw", "prefer_ilm": true, "ilm_policy": "pre-dsl-ilm-policy", + "index_mode": "standard", "managed_by": "Index Lifecycle Management" }, { @@ -438,6 +450,7 @@ GET _data_stream/dsl-data-stream "index_uuid": "PA_JquKGSiKcAKBA8abcd1", "prefer_ilm": false, "ilm_policy": "pre-dsl-ilm-policy", + "index_mode": "standard", "managed_by": "Index Lifecycle Management" <1> } ], @@ -455,6 +468,7 @@ GET _data_stream/dsl-data-stream "system": false, "allow_custom_routing": false, "replicated": false, + "index_mode": "standard", "rollover_on_write": false } ] diff --git a/docs/reference/indices/get-data-stream.asciidoc b/docs/reference/indices/get-data-stream.asciidoc index ccab53f020e5f..a46d247d9a9a0 100644 --- a/docs/reference/indices/get-data-stream.asciidoc +++ b/docs/reference/indices/get-data-stream.asciidoc @@ -304,14 +304,16 @@ The API returns the following response: "index_uuid": "xCEhwsp8Tey0-FLNFYVwSg", "prefer_ilm": true, "ilm_policy": "my-lifecycle-policy", - "managed_by": "Index Lifecycle Management" + "managed_by": "Index Lifecycle Management", + "index_mode": "standard" }, { "index_name": ".ds-my-data-stream-2099.03.08-000002", "index_uuid": "PA_JquKGSiKcAKBA8DJ5gw", "prefer_ilm": true, "ilm_policy": "my-lifecycle-policy", - "managed_by": "Index Lifecycle Management" + "managed_by": "Index Lifecycle Management", + "index_mode": "standard" } ], "generation": 2, @@ -319,6 +321,7 @@ The API returns the following response: "my-meta-field": "foo" }, "status": "GREEN", + "index_mode": "standard", "next_generation_managed_by": "Index Lifecycle Management", "prefer_ilm": true, "template": "my-index-template", @@ -340,7 +343,8 @@ The API returns the following response: "index_uuid": "3liBu2SYS5axasRt6fUIpA", "prefer_ilm": true, "ilm_policy": "my-lifecycle-policy", - "managed_by": "Index Lifecycle Management" + "managed_by": "Index Lifecycle Management", + "index_mode": "standard" } ], "generation": 1, @@ -348,6 +352,7 @@ The API returns the following response: "my-meta-field": "foo" }, "status": "YELLOW", + "index_mode": "standard", "next_generation_managed_by": "Index Lifecycle Management", "prefer_ilm": true, "template": "my-index-template", diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsAction.java index 2681bf1615665..0759feef71053 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsAction.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsAction.java @@ -24,6 +24,7 @@ import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.health.ClusterStateHealth; +import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.metadata.DataStreamFailureStoreSettings; import org.elasticsearch.cluster.metadata.DataStreamGlobalRetentionSettings; @@ -39,6 +40,9 @@ import org.elasticsearch.core.Tuple; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexMode; +import org.elasticsearch.index.IndexSettingProvider; +import org.elasticsearch.index.IndexSettingProviders; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.indices.SystemDataStreamDescriptor; import org.elasticsearch.indices.SystemIndices; import org.elasticsearch.injection.guice.Inject; @@ -52,6 +56,7 @@ import java.util.Comparator; import java.util.HashMap; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.stream.Collectors; @@ -67,6 +72,7 @@ public class TransportGetDataStreamsAction extends TransportMasterNodeReadAction private final ClusterSettings clusterSettings; private final DataStreamGlobalRetentionSettings globalRetentionSettings; private final DataStreamFailureStoreSettings dataStreamFailureStoreSettings; + private final IndexSettingProviders indexSettingProviders; private final Client client; @Inject @@ -79,6 +85,7 @@ public TransportGetDataStreamsAction( SystemIndices systemIndices, DataStreamGlobalRetentionSettings globalRetentionSettings, DataStreamFailureStoreSettings dataStreamFailureStoreSettings, + IndexSettingProviders indexSettingProviders, Client client ) { super( @@ -96,6 +103,7 @@ public TransportGetDataStreamsAction( this.globalRetentionSettings = globalRetentionSettings; clusterSettings = clusterService.getClusterSettings(); this.dataStreamFailureStoreSettings = dataStreamFailureStoreSettings; + this.indexSettingProviders = indexSettingProviders; this.client = new OriginSettingClient(client, "stack"); } @@ -128,6 +136,7 @@ public void onResponse(DataStreamsStatsAction.Response response) { clusterSettings, globalRetentionSettings, dataStreamFailureStoreSettings, + indexSettingProviders, maxTimestamps ) ); @@ -148,12 +157,43 @@ public void onFailure(Exception e) { clusterSettings, globalRetentionSettings, dataStreamFailureStoreSettings, + indexSettingProviders, null ) ); } } + /** + * Resolves the index mode ("index.mode" setting) for the given data stream, from the template or additional setting providers + */ + @Nullable + static IndexMode resolveMode( + ClusterState state, + IndexSettingProviders indexSettingProviders, + DataStream dataStream, + Settings settings, + ComposableIndexTemplate indexTemplate + ) { + IndexMode indexMode = state.metadata().retrieveIndexModeFromTemplate(indexTemplate); + for (IndexSettingProvider provider : indexSettingProviders.getIndexSettingProviders()) { + Settings addlSettinsg = provider.getAdditionalIndexSettings( + MetadataIndexTemplateService.VALIDATE_INDEX_NAME, + dataStream.getName(), + indexMode, + state.metadata(), + Instant.now(), + settings, + List.of() + ); + var rawMode = addlSettinsg.get(IndexSettings.MODE.getKey()); + if (rawMode != null) { + indexMode = Enum.valueOf(IndexMode.class, rawMode.toUpperCase(Locale.ROOT)); + } + } + return indexMode; + } + static GetDataStreamAction.Response innerOperation( ClusterState state, GetDataStreamAction.Request request, @@ -162,6 +202,7 @@ static GetDataStreamAction.Response innerOperation( ClusterSettings clusterSettings, DataStreamGlobalRetentionSettings globalRetentionSettings, DataStreamFailureStoreSettings dataStreamFailureStoreSettings, + IndexSettingProviders indexSettingProviders, @Nullable Map maxTimestamps ) { List dataStreams = getDataStreams(state, indexNameExpressionResolver, request); @@ -173,6 +214,7 @@ static GetDataStreamAction.Response innerOperation( final String indexTemplate; boolean indexTemplatePreferIlmValue = true; String ilmPolicyName = null; + IndexMode indexMode = dataStream.getIndexMode(); if (dataStream.isSystem()) { SystemDataStreamDescriptor dataStreamDescriptor = systemIndices.findMatchingDataStreamDescriptor(dataStream.getName()); indexTemplate = dataStreamDescriptor != null ? dataStreamDescriptor.getDataStreamName() : null; @@ -182,6 +224,15 @@ static GetDataStreamAction.Response innerOperation( dataStreamDescriptor.getComponentTemplates() ); ilmPolicyName = settings.get(IndexMetadata.LIFECYCLE_NAME); + if (indexMode == null) { + indexMode = resolveMode( + state, + indexSettingProviders, + dataStream, + settings, + dataStreamDescriptor.getComposableIndexTemplate() + ); + } indexTemplatePreferIlmValue = PREFER_ILM_SETTING.get(settings); } } else { @@ -189,6 +240,15 @@ static GetDataStreamAction.Response innerOperation( if (indexTemplate != null) { Settings settings = MetadataIndexTemplateService.resolveSettings(state.metadata(), indexTemplate); ilmPolicyName = settings.get(IndexMetadata.LIFECYCLE_NAME); + if (indexMode == null && state.metadata().templatesV2().get(indexTemplate) != null) { + indexMode = resolveMode( + state, + indexSettingProviders, + dataStream, + settings, + state.metadata().templatesV2().get(indexTemplate) + ); + } indexTemplatePreferIlmValue = PREFER_ILM_SETTING.get(settings); } else { LOGGER.warn( @@ -280,7 +340,9 @@ public int compareTo(IndexInfo o) { timeSeries, backingIndicesSettingsValues, indexTemplatePreferIlmValue, - maxTimestamps == null ? null : maxTimestamps.get(dataStream.getName()) + maxTimestamps == null ? null : maxTimestamps.get(dataStream.getName()), + // Default to standard mode if not specified; should we set this to "unset" or "unspecified" instead? + indexMode == null ? IndexMode.STANDARD.getName() : indexMode.getName() ) ); } @@ -310,7 +372,11 @@ private static void collectIndexSettingsValues( } else { managedBy = ManagedBy.UNMANAGED; } - backingIndicesSettingsValues.put(index, new IndexProperties(preferIlm, indexMetadata.getLifecyclePolicyName(), managedBy)); + String indexMode = IndexSettings.MODE.get(indexMetadata.getSettings()).getName(); + backingIndicesSettingsValues.put( + index, + new IndexProperties(preferIlm, indexMetadata.getLifecyclePolicyName(), managedBy, indexMode) + ); } } diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/GetDataStreamsResponseTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/GetDataStreamsResponseTests.java index e483158c484c1..4f1a7e0b266e0 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/GetDataStreamsResponseTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/GetDataStreamsResponseTests.java @@ -92,13 +92,13 @@ public void testResponseIlmAndDataStreamLifecycleRepresentation() throws Excepti String ilmPolicyName = "rollover-30days"; Map indexSettingsValues = Map.of( firstGenerationIndex, - new Response.IndexProperties(true, ilmPolicyName, ManagedBy.ILM), + new Response.IndexProperties(true, ilmPolicyName, ManagedBy.ILM, null), secondGenerationIndex, - new Response.IndexProperties(false, ilmPolicyName, ManagedBy.LIFECYCLE), + new Response.IndexProperties(false, ilmPolicyName, ManagedBy.LIFECYCLE, null), writeIndex, - new Response.IndexProperties(false, null, ManagedBy.LIFECYCLE), + new Response.IndexProperties(false, null, ManagedBy.LIFECYCLE, null), failureStoreIndex, - new Response.IndexProperties(false, null, ManagedBy.LIFECYCLE) + new Response.IndexProperties(false, null, ManagedBy.LIFECYCLE, null) ); Response.DataStreamInfo dataStreamInfo = new Response.DataStreamInfo( @@ -110,6 +110,7 @@ public void testResponseIlmAndDataStreamLifecycleRepresentation() throws Excepti null, indexSettingsValues, false, + null, null ); Response response = new Response(List.of(dataStreamInfo)); @@ -191,13 +192,13 @@ public void testResponseIlmAndDataStreamLifecycleRepresentation() throws Excepti String ilmPolicyName = "rollover-30days"; Map indexSettingsValues = Map.of( firstGenerationIndex, - new Response.IndexProperties(true, ilmPolicyName, ManagedBy.ILM), + new Response.IndexProperties(true, ilmPolicyName, ManagedBy.ILM, null), secondGenerationIndex, - new Response.IndexProperties(true, ilmPolicyName, ManagedBy.ILM), + new Response.IndexProperties(true, ilmPolicyName, ManagedBy.ILM, null), writeIndex, - new Response.IndexProperties(false, null, ManagedBy.UNMANAGED), + new Response.IndexProperties(false, null, ManagedBy.UNMANAGED, null), failureStoreIndex, - new Response.IndexProperties(false, null, ManagedBy.UNMANAGED) + new Response.IndexProperties(false, null, ManagedBy.UNMANAGED, null) ); Response.DataStreamInfo dataStreamInfo = new Response.DataStreamInfo( @@ -209,6 +210,7 @@ public void testResponseIlmAndDataStreamLifecycleRepresentation() throws Excepti null, indexSettingsValues, false, + null, null ); Response response = new Response(List.of(dataStreamInfo)); @@ -279,13 +281,13 @@ public void testResponseIlmAndDataStreamLifecycleRepresentation() throws Excepti String ilmPolicyName = "rollover-30days"; Map indexSettingsValues = Map.of( firstGenerationIndex, - new Response.IndexProperties(true, ilmPolicyName, ManagedBy.ILM), + new Response.IndexProperties(true, ilmPolicyName, ManagedBy.ILM, null), secondGenerationIndex, - new Response.IndexProperties(false, ilmPolicyName, ManagedBy.LIFECYCLE), + new Response.IndexProperties(false, ilmPolicyName, ManagedBy.LIFECYCLE, null), writeIndex, - new Response.IndexProperties(true, null, ManagedBy.LIFECYCLE), + new Response.IndexProperties(true, null, ManagedBy.LIFECYCLE, null), failureStoreIndex, - new Response.IndexProperties(randomBoolean(), ilmPolicyName, ManagedBy.LIFECYCLE) + new Response.IndexProperties(randomBoolean(), ilmPolicyName, ManagedBy.LIFECYCLE, null) ); Response.DataStreamInfo dataStreamInfo = new Response.DataStreamInfo( @@ -297,6 +299,7 @@ public void testResponseIlmAndDataStreamLifecycleRepresentation() throws Excepti null, indexSettingsValues, false, + null, null ); Response response = new Response(List.of(dataStreamInfo)); @@ -359,7 +362,8 @@ private Response.DataStreamInfo mutateInstance(Response.DataStreamInfo instance) new Response.IndexProperties( randomBoolean(), randomAlphaOfLengthBetween(50, 100), - randomBoolean() ? ManagedBy.ILM : ManagedBy.LIFECYCLE + randomBoolean() ? ManagedBy.ILM : ManagedBy.LIFECYCLE, + null ) ) ); @@ -378,7 +382,8 @@ private Response.DataStreamInfo mutateInstance(Response.DataStreamInfo instance) timeSeries, indexSettings, templatePreferIlm, - maximumTimestamp + maximumTimestamp, + null ); } @@ -399,7 +404,8 @@ private Map generateRandomIndexSettingsValues() new Response.IndexProperties( randomBoolean(), randomAlphaOfLengthBetween(50, 100), - randomBoolean() ? ManagedBy.ILM : ManagedBy.LIFECYCLE + randomBoolean() ? ManagedBy.ILM : ManagedBy.LIFECYCLE, + randomBoolean() ? randomFrom(IndexMode.values()).getName() : null ) ); } @@ -417,7 +423,8 @@ private Response.DataStreamInfo generateRandomDataStreamInfo() { timeSeries != null ? new Response.TimeSeries(timeSeries) : null, generateRandomIndexSettingsValues(), randomBoolean(), - usually() ? randomNonNegativeLong() : null + usually() ? randomNonNegativeLong() : null, + usually() ? randomFrom(IndexMode.values()).getName() : null ); } } diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsActionTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsActionTests.java index 0189ff6d59745..bcb3236d19c87 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsActionTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsActionTests.java @@ -23,7 +23,9 @@ import org.elasticsearch.core.TimeValue; import org.elasticsearch.core.Tuple; import org.elasticsearch.index.Index; +import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexNotFoundException; +import org.elasticsearch.index.IndexSettingProviders; import org.elasticsearch.indices.SystemIndices; import org.elasticsearch.indices.TestIndexNameExpressionResolver; import org.elasticsearch.test.ESTestCase; @@ -31,6 +33,7 @@ import java.time.Instant; import java.time.temporal.ChronoUnit; import java.util.List; +import java.util.Set; import static org.elasticsearch.cluster.metadata.DataStreamTestHelper.getClusterStateWithDataStreams; import static org.elasticsearch.test.LambdaMatchers.transformedItemsMatch; @@ -173,6 +176,7 @@ public void testGetTimeSeriesDataStream() { ClusterSettings.createBuiltInClusterSettings(), dataStreamGlobalRetentionSettings, emptyDataStreamFailureStoreSettings, + new IndexSettingProviders(Set.of()), null ); assertThat( @@ -205,6 +209,7 @@ public void testGetTimeSeriesDataStream() { ClusterSettings.createBuiltInClusterSettings(), dataStreamGlobalRetentionSettings, emptyDataStreamFailureStoreSettings, + new IndexSettingProviders(Set.of()), null ); assertThat( @@ -257,6 +262,7 @@ public void testGetTimeSeriesDataStreamWithOutOfOrderIndices() { ClusterSettings.createBuiltInClusterSettings(), dataStreamGlobalRetentionSettings, emptyDataStreamFailureStoreSettings, + new IndexSettingProviders(Set.of()), null ); assertThat( @@ -302,6 +308,7 @@ public void testGetTimeSeriesMixedDataStream() { ClusterSettings.createBuiltInClusterSettings(), dataStreamGlobalRetentionSettings, emptyDataStreamFailureStoreSettings, + new IndexSettingProviders(Set.of()), null ); @@ -349,6 +356,7 @@ public void testPassingGlobalRetention() { ClusterSettings.createBuiltInClusterSettings(), dataStreamGlobalRetentionSettings, emptyDataStreamFailureStoreSettings, + new IndexSettingProviders(Set.of()), null ); assertThat(response.getDataGlobalRetention(), nullValue()); @@ -375,6 +383,7 @@ public void testPassingGlobalRetention() { ClusterSettings.createBuiltInClusterSettings(), withGlobalRetentionSettings, emptyDataStreamFailureStoreSettings, + new IndexSettingProviders(Set.of()), null ); assertThat(response.getDataGlobalRetention(), equalTo(dataGlobalRetention)); @@ -405,6 +414,7 @@ public void testDataStreamIsFailureStoreEffectivelyEnabled_disabled() { ClusterSettings.createBuiltInClusterSettings(), dataStreamGlobalRetentionSettings, emptyDataStreamFailureStoreSettings, + new IndexSettingProviders(Set.of()), null ); assertThat(response.getDataStreams(), hasSize(1)); @@ -434,6 +444,7 @@ public void testDataStreamIsFailureStoreEffectivelyEnabled_enabledExplicitly() { ClusterSettings.createBuiltInClusterSettings(), dataStreamGlobalRetentionSettings, emptyDataStreamFailureStoreSettings, + new IndexSettingProviders(Set.of()), null ); assertThat(response.getDataStreams(), hasSize(1)); @@ -469,9 +480,58 @@ public void testDataStreamIsFailureStoreEffectivelyEnabled_enabledByClusterSetti .build() ) ), + new IndexSettingProviders(Set.of()), null ); assertThat(response.getDataStreams(), hasSize(1)); assertThat(response.getDataStreams().get(0).isFailureStoreEffectivelyEnabled(), is(true)); } + + public void testProvidersAffectMode() { + ClusterState state = DataStreamTestHelper.getClusterStateWithDataStreams( + List.of(Tuple.tuple("data-stream-1", 2)), + List.of(), + System.currentTimeMillis(), + Settings.EMPTY, + 0, + false, + false + ); + + var req = new GetDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] {}); + var response = TransportGetDataStreamsAction.innerOperation( + state, + req, + resolver, + systemIndices, + ClusterSettings.createBuiltInClusterSettings(), + dataStreamGlobalRetentionSettings, + emptyDataStreamFailureStoreSettings, + new IndexSettingProviders( + Set.of( + ( + indexName, + dataStreamName, + templateIndexMode, + metadata, + resolvedAt, + indexTemplateAndCreateRequestSettings, + combinedTemplateMappings) -> Settings.builder().put("index.mode", IndexMode.LOOKUP).build() + ) + ), + null + ); + assertThat(response.getDataStreams().get(0).getIndexModeName(), equalTo("lookup")); + assertThat( + response.getDataStreams() + .get(0) + .getIndexSettingsValues() + .values() + .stream() + .findFirst() + .map(GetDataStreamAction.Response.IndexProperties::indexMode) + .orElse("bad"), + equalTo("standard") + ); + } } diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java index 404d6517406d8..ce1a3eb494ca9 100644 --- a/server/src/main/java/org/elasticsearch/TransportVersions.java +++ b/server/src/main/java/org/elasticsearch/TransportVersions.java @@ -221,6 +221,7 @@ static TransportVersion def(int id) { public static final TransportVersion ESQL_DRIVER_TASK_DESCRIPTION_8_19 = def(8_841_0_30); public static final TransportVersion ML_INFERENCE_HUGGING_FACE_CHAT_COMPLETION_ADDED_8_19 = def(8_841_0_31); public static final TransportVersion FIELD_CAPS_ADD_CLUSTER_ALIAS = def(8_841_0_32); + public static final TransportVersion INCLUDE_INDEX_MODE_IN_GET_DATA_STREAM_BACKPORT_8_19 = def(8_841_0_33); /* * STOP! READ THIS FIRST! No, really, diff --git a/server/src/main/java/org/elasticsearch/action/datastreams/GetDataStreamAction.java b/server/src/main/java/org/elasticsearch/action/datastreams/GetDataStreamAction.java index 68a2a1716e48a..172903727ffb5 100644 --- a/server/src/main/java/org/elasticsearch/action/datastreams/GetDataStreamAction.java +++ b/server/src/main/java/org/elasticsearch/action/datastreams/GetDataStreamAction.java @@ -232,6 +232,7 @@ public static class DataStreamInfo implements SimpleDiffable, To ); public static final ParseField FAILURE_STORE_ENABLED = new ParseField("enabled"); public static final ParseField MAXIMUM_TIMESTAMP = new ParseField("maximum_timestamp"); + public static final ParseField INDEX_MODE = new ParseField("index_mode"); private final DataStream dataStream; private final ClusterHealthStatus dataStreamStatus; @@ -246,6 +247,8 @@ public static class DataStreamInfo implements SimpleDiffable, To private final boolean templatePreferIlmValue; @Nullable private final Long maximumTimestamp; + @Nullable + private final String indexMode; public DataStreamInfo( DataStream dataStream, @@ -256,7 +259,8 @@ public DataStreamInfo( @Nullable TimeSeries timeSeries, Map indexSettingsValues, boolean templatePreferIlmValue, - @Nullable Long maximumTimestamp + @Nullable Long maximumTimestamp, + @Nullable String indexMode ) { this.dataStream = dataStream; this.failureStoreEffectivelyEnabled = failureStoreEffectivelyEnabled; @@ -267,6 +271,7 @@ public DataStreamInfo( this.indexSettingsValues = indexSettingsValues; this.templatePreferIlmValue = templatePreferIlmValue; this.maximumTimestamp = maximumTimestamp; + this.indexMode = indexMode; } @SuppressWarnings("unchecked") @@ -287,6 +292,9 @@ public DataStreamInfo( : Map.of(); this.templatePreferIlmValue = in.getTransportVersion().onOrAfter(V_8_11_X) ? in.readBoolean() : true; this.maximumTimestamp = in.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0) ? in.readOptionalVLong() : null; + this.indexMode = in.getTransportVersion().onOrAfter(TransportVersions.INCLUDE_INDEX_MODE_IN_GET_DATA_STREAM_BACKPORT_8_19) + ? in.readOptionalString() + : null; } public DataStream getDataStream() { @@ -329,6 +337,11 @@ public Long getMaximumTimestamp() { return maximumTimestamp; } + @Nullable + public String getIndexModeName() { + return indexMode; + } + @Override public void writeTo(StreamOutput out) throws IOException { dataStream.writeTo(out); @@ -348,6 +361,9 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { out.writeOptionalVLong(maximumTimestamp); } + if (out.getTransportVersion().onOrAfter(TransportVersions.INCLUDE_INDEX_MODE_IN_GET_DATA_STREAM_BACKPORT_8_19)) { + out.writeOptionalString(indexMode); + } } @Override @@ -400,6 +416,9 @@ public XContentBuilder toXContent( if (this.maximumTimestamp != null) { builder.field(MAXIMUM_TIMESTAMP.getPreferredName(), this.maximumTimestamp); } + if (this.indexMode != null) { + builder.field(INDEX_MODE.getPreferredName(), indexMode); + } addAutoShardingEvent(builder, params, dataStream.getAutoShardingEvent()); if (timeSeries != null) { builder.startObject(TIME_SERIES.getPreferredName()); @@ -455,6 +474,7 @@ private XContentBuilder indicesToXContent(XContentBuilder builder, List i builder.field(ILM_POLICY_FIELD.getPreferredName(), indexProperties.ilmPolicyName()); } } + builder.field(INDEX_MODE.getPreferredName(), indexProperties.indexMode); } builder.endObject(); @@ -515,7 +535,8 @@ public boolean equals(Object o) { && Objects.equals(ilmPolicyName, that.ilmPolicyName) && Objects.equals(timeSeries, that.timeSeries) && Objects.equals(indexSettingsValues, that.indexSettingsValues) - && Objects.equals(maximumTimestamp, that.maximumTimestamp); + && Objects.equals(maximumTimestamp, that.maximumTimestamp) + && Objects.equals(indexMode, that.indexMode); } @Override @@ -529,7 +550,8 @@ public int hashCode() { timeSeries, indexSettingsValues, templatePreferIlmValue, - maximumTimestamp + maximumTimestamp, + indexMode ); } } @@ -566,9 +588,18 @@ public int hashCode() { * Encapsulates the configured properties we want to display for each backing index. * They'll usually be settings values, but could also be additional properties derived from settings. */ - public record IndexProperties(boolean preferIlm, @Nullable String ilmPolicyName, ManagedBy managedBy) implements Writeable { + public record IndexProperties(boolean preferIlm, @Nullable String ilmPolicyName, ManagedBy managedBy, @Nullable String indexMode) + implements + Writeable { public IndexProperties(StreamInput in) throws IOException { - this(in.readBoolean(), in.readOptionalString(), in.readEnum(ManagedBy.class)); + this( + in.readBoolean(), + in.readOptionalString(), + in.readEnum(ManagedBy.class), + in.getTransportVersion().onOrAfter(TransportVersions.INCLUDE_INDEX_MODE_IN_GET_DATA_STREAM_BACKPORT_8_19) + ? in.readOptionalString() + : "unknown" + ); } @Override @@ -576,6 +607,9 @@ public void writeTo(StreamOutput out) throws IOException { out.writeBoolean(preferIlm); out.writeOptionalString(ilmPolicyName); out.writeEnum(managedBy); + if (out.getTransportVersion().onOrAfter(TransportVersions.INCLUDE_INDEX_MODE_IN_GET_DATA_STREAM_BACKPORT_8_19)) { + out.writeOptionalString(indexMode); + } } } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java index 926e20e1752be..1ab6498b012d0 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java @@ -91,6 +91,9 @@ public class MetadataIndexTemplateService { public static final String DEFAULT_TIMESTAMP_FIELD = "@timestamp"; public static final CompressedXContent DEFAULT_TIMESTAMP_MAPPING_WITHOUT_ROUTING; + // Names used for validating templates when we do not know the index or data stream name + public static final String VALIDATE_INDEX_NAME = "validate-index-name"; + public static final String VALIDATE_DATA_STREAM_NAME = "validate-data-stream-name"; private static final CompressedXContent DEFAULT_TIMESTAMP_MAPPING_WITH_ROUTING; @@ -708,8 +711,8 @@ void validateIndexTemplateV2(String name, ComposableIndexTemplate indexTemplate, var finalSettings = Settings.builder(); for (var provider : indexSettingProviders) { var newAdditionalSettings = provider.getAdditionalIndexSettings( - "validate-index-name", - indexTemplate.getDataStreamTemplate() != null ? "validate-data-stream-name" : null, + VALIDATE_INDEX_NAME, + indexTemplate.getDataStreamTemplate() != null ? VALIDATE_DATA_STREAM_NAME : null, metadata.retrieveIndexModeFromTemplate(indexTemplate), currentState.getMetadata(), now, diff --git a/server/src/test/java/org/elasticsearch/action/datastreams/GetDataStreamActionTests.java b/server/src/test/java/org/elasticsearch/action/datastreams/GetDataStreamActionTests.java index 64d6c6ef002e9..0614356943d3a 100644 --- a/server/src/test/java/org/elasticsearch/action/datastreams/GetDataStreamActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/datastreams/GetDataStreamActionTests.java @@ -92,6 +92,7 @@ private static GetDataStreamAction.Response.DataStreamInfo newDataStreamInfo(boo null, Map.of(), randomBoolean(), + null, null ); } diff --git a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProvider.java b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProvider.java index d990a43148427..a98f0a1e33af9 100644 --- a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProvider.java +++ b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProvider.java @@ -13,6 +13,7 @@ import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.cluster.metadata.MetadataIndexTemplateService; import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.regex.Regex; @@ -100,7 +101,9 @@ public Settings getAdditionalIndexSettings( ) { Settings.Builder settingsBuilder = null; boolean isLogsDB = templateIndexMode == IndexMode.LOGSDB; - boolean isTemplateValidation = "validate-index-name".equals(indexName); + // This index name is used when validating component and index templates, we should skip this check in that case. + // (See MetadataIndexTemplateService#validateIndexTemplateV2(...) method) + boolean isTemplateValidation = MetadataIndexTemplateService.VALIDATE_INDEX_NAME.equals(indexName); // Inject logsdb index mode, based on the logs pattern. if (isLogsdbEnabled @@ -118,8 +121,6 @@ && matchesLogsPattern(dataStreamName)) { if (mappingHints.hasSyntheticSourceUsage && supportFallbackToStoredSource.get() && minNodeVersion.get().get().onOrAfter(Version.V_8_17_0)) { - // This index name is used when validating component and index templates, we should skip this check in that case. - // (See MetadataIndexTemplateService#validateIndexTemplateV2(...) method) boolean legacyLicensedUsageOfSyntheticSourceAllowed = isLegacyLicensedUsageOfSyntheticSourceAllowed( templateIndexMode, indexName, @@ -216,7 +217,7 @@ MappingHints getMappingHints( Settings indexTemplateAndCreateRequestSettings, List combinedTemplateMappings ) { - if ("validate-index-name".equals(indexName)) { + if (MetadataIndexTemplateService.VALIDATE_INDEX_NAME.equals(indexName)) { // This index name is used when validating component and index templates, we should skip this check in that case. // (See MetadataIndexTemplateService#validateIndexTemplateV2(...) method) return MappingHints.EMPTY; diff --git a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProviderTests.java b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProviderTests.java index b586acdf002ab..0efca7175a56e 100644 --- a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProviderTests.java +++ b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProviderTests.java @@ -14,6 +14,7 @@ import org.elasticsearch.cluster.metadata.DataStreamTestHelper; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.cluster.metadata.MetadataIndexTemplateService; import org.elasticsearch.cluster.metadata.Template; import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.settings.Settings; @@ -473,7 +474,7 @@ public void testNewIndexHasSyntheticSourceUsage() throws IOException { } public void testValidateIndexName() throws IOException { - String indexName = "validate-index-name"; + String indexName = MetadataIndexTemplateService.VALIDATE_INDEX_NAME; String mapping = """ { "_doc": {