diff --git a/docs/changelog/122486.yaml b/docs/changelog/122486.yaml new file mode 100644 index 0000000000000..027d2a5e63ba3 --- /dev/null +++ b/docs/changelog/122486.yaml @@ -0,0 +1,5 @@ +pr: 122486 +summary: Add index mode to get data stream API +area: Data streams +type: enhancement +issues: [] diff --git a/docs/reference/indices/get-data-stream.asciidoc b/docs/reference/indices/get-data-stream.asciidoc new file mode 100644 index 0000000000000..a46d247d9a9a0 --- /dev/null +++ b/docs/reference/indices/get-data-stream.asciidoc @@ -0,0 +1,376 @@ +[role="xpack"] +[[indices-get-data-stream]] +=== Get data stream API +++++ +Get data stream +++++ + +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-data-stream[Data stream APIs]. +-- + +Retrieves information about one or more <>. +See <>. + +//// +[source,console] +---- +PUT /_ilm/policy/my-lifecycle-policy +{ + "policy": { + "phases": { + "hot": { + "actions": { + "rollover": { + "max_primary_shard_size": "25GB" + } + } + }, + "delete": { + "min_age": "30d", + "actions": { + "delete": {} + } + } + } + } +} + +PUT /_index_template/my-index-template +{ + "index_patterns": [ "my-data-stream*" ], + "data_stream": {}, + "template": { + "settings": { + "index.lifecycle.name": "my-lifecycle-policy" + } + }, + "_meta": { + "my-meta-field": "foo" + } +} + +PUT /_data_stream/my-data-stream + +POST /my-data-stream/_rollover + +PUT /_data_stream/my-data-stream-two + +DELETE /_data_stream/my-data-stream*/_lifecycle +---- +// TESTSETUP +//// + +//// +[source,console] +---- +DELETE /_data_stream/* +DELETE /_index_template/* +DELETE /_ilm/policy/my-lifecycle-policy +---- +// TEARDOWN +//// + +[source,console] +---- +GET /_data_stream/my-data-stream +---- + +[[get-data-stream-api-request]] +==== {api-request-title} + +`GET /_data_stream/` + +[[get-data-stream-api-prereqs]] +==== {api-prereq-title} + +* If the {es} {security-features} are enabled, you must have the +`view_index_metadata` or `manage` <> +for the data stream. + +[[get-data-stream-api-path-params]] +==== {api-path-parms-title} + +``:: +(Optional, string) +Comma-separated list of data stream names used to limit the request. Wildcard +(`*`) expressions are supported. If omitted, all data streams will be +returned. + +[role="child_attributes"] +[[get-data-stream-api-query-parms]] +==== {api-query-parms-title} + +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=ds-expand-wildcards] ++ +Defaults to `open`. + +`include_defaults`:: +(Optional, Boolean) Functionality in preview:[]. If `true`, return all default settings in the response. +Defaults to `false`. + +`verbose`:: +(Optional, Boolean). If `true`, Returns the `maximum_timestamp` corresponding to the `@timestamp` field for documents in the data stream. +Defaults to `false`. + +[role="child_attributes"] +[[get-data-stream-api-response-body]] +==== {api-response-body-title} + +`data_streams`:: +(array of objects) +Contains information about retrieved data streams. ++ +.Properties of objects in `data_streams` +[%collapsible%open] +==== +`name`:: +(string) +Name of the data stream. + +`timestamp_field`:: +(object) +Contains information about the data stream's `@timestamp` field. ++ +.Properties of `timestamp_field` +[%collapsible%open] +===== +`name`:: +(string) +Name of the data stream's timestamp field, which must be `@timestamp`. The +`@timestamp` field must be included in every document indexed to the data +stream. +===== + +`indices`:: +(array of objects) +Array of objects containing information about the data stream's backing +indices. ++ +The last item in this array contains information about the stream's current +<>. ++ +.Properties of `indices` objects +[%collapsible%open] +===== +`index_name`:: +(string) +Name of the backing index. For naming conventions, see +<>. + +`index_uuid`:: +(string) +Universally unique identifier (UUID) for the index. + +`prefer_ilm`:: +(boolean) +Functionality in preview:[]. Indicates if this index is configured to prefer {ilm} +when both {ilm-cap} and <> are configured to +manage this index. + +`managed_by`:: +(string) +Functionality in preview:[]. Indicates the system that managed this index. +===== + +`generation`:: +(integer) +Current <> for the data stream. This number +acts as a cumulative count of the stream's rollovers, starting at `1`. + +`_meta`:: +(object) +Custom metadata for the stream, copied from the `_meta` object of the +stream's matching <>. If empty, +the response omits this property. + +`status`:: +(string) +<> of the data stream. ++ +This health status is based on the state of the primary and replica shards of +the stream's backing indices. ++ +.Values for `status` +[%collapsible%open] +===== +`GREEN`::: +All shards are assigned. + +`YELLOW`::: +All primary shards are assigned, but one or more replica shards are +unassigned. + +`RED`::: +One or more primary shards are unassigned, so some data is unavailable. +===== + +`template`:: +(string) +Name of the index template used to create the data stream's backing indices. ++ +The template's index pattern must match the name of this data stream. See +<>. + +`ilm_policy`:: +(string) +Name of the current {ilm-init} lifecycle policy in the stream's matching index +template. This lifecycle policy is set in the `index.lifecycle.name` setting. ++ +If the template does not include a lifecycle policy, this property is not +included in the response. ++ +NOTE: A data stream's backing indices may be assigned different lifecycle +policies. To retrieve the lifecycle policy for individual backing indices, +use the <>. + +`next_generation_managed_by`:: +(string) +Functionality in preview:[]. Indicates the system that will managed the next generation index +(i.e. the next data stream write index). + +`prefer_ilm`:: +(boolean) +Functionality in preview:[]. Indicates if the index template used to create the data +stream's backing indices is configured to prefer {ilm-cap} when both {ilm-cap} and +<> are configured to manage this index. + +`hidden`:: +(Boolean) If `true`, the data stream is <>. + +`system`:: +(Boolean) +If `true`, the data stream is created and managed by an Elastic stack component +and cannot be modified through normal user interaction. + +`allow_custom_routing`:: +(Boolean) +If `true`, the data stream this data stream allows custom routing on write request. + +`replicated`:: +(Boolean) +If `true`, the data stream is created and managed by {ccr} and the local +cluster can not write into this data stream or change its mappings. + +`lifecycle`:: +(object) +Functionality in preview:[]. Contains the configuration for the data stream lifecycle management of this data stream. ++ +.Properties of `lifecycle` +[%collapsible%open] +===== +`data_retention`:: +(string) +If defined, every document added to this data stream will be stored at least for this time frame. Any time after this +duration the document could be deleted. When empty, every document in this data stream will be stored indefinitely. + +`rollover`:: +(object) +The conditions which will trigger the rollover of a backing index as configured by the cluster setting +`cluster.lifecycle.default.rollover`. This property is an implementation detail and it will only be retrieved when the query +param `include_defaults` is set to `true`. The contents of this field are subject to change. +===== + +`rollover_on_write`:: +(Boolean) +If `true`, the next write to this data stream will trigger a rollover first and the document will be +indexed in the new backing index. If the rollover fails the indexing request will fail too. +==== + +[[get-data-stream-api-example]] +==== {api-examples-title} + +[source,console] +---- +GET _data_stream/my-data-stream* +---- + +The API returns the following response: + +[source,console-result] +---- +{ + "data_streams": [ + { + "name": "my-data-stream", + "timestamp_field": { + "name": "@timestamp" + }, + "indices": [ + { + "index_name": ".ds-my-data-stream-2099.03.07-000001", + "index_uuid": "xCEhwsp8Tey0-FLNFYVwSg", + "prefer_ilm": true, + "ilm_policy": "my-lifecycle-policy", + "managed_by": "Index Lifecycle Management", + "index_mode": "standard" + }, + { + "index_name": ".ds-my-data-stream-2099.03.08-000002", + "index_uuid": "PA_JquKGSiKcAKBA8DJ5gw", + "prefer_ilm": true, + "ilm_policy": "my-lifecycle-policy", + "managed_by": "Index Lifecycle Management", + "index_mode": "standard" + } + ], + "generation": 2, + "_meta": { + "my-meta-field": "foo" + }, + "status": "GREEN", + "index_mode": "standard", + "next_generation_managed_by": "Index Lifecycle Management", + "prefer_ilm": true, + "template": "my-index-template", + "ilm_policy": "my-lifecycle-policy", + "hidden": false, + "system": false, + "allow_custom_routing": false, + "replicated": false, + "rollover_on_write": false + }, + { + "name": "my-data-stream-two", + "timestamp_field": { + "name": "@timestamp" + }, + "indices": [ + { + "index_name": ".ds-my-data-stream-two-2099.03.08-000001", + "index_uuid": "3liBu2SYS5axasRt6fUIpA", + "prefer_ilm": true, + "ilm_policy": "my-lifecycle-policy", + "managed_by": "Index Lifecycle Management", + "index_mode": "standard" + } + ], + "generation": 1, + "_meta": { + "my-meta-field": "foo" + }, + "status": "YELLOW", + "index_mode": "standard", + "next_generation_managed_by": "Index Lifecycle Management", + "prefer_ilm": true, + "template": "my-index-template", + "ilm_policy": "my-lifecycle-policy", + "hidden": false, + "system": false, + "allow_custom_routing": false, + "replicated": false, + "rollover_on_write": false + } + ] +} +---- +// TESTRESPONSE[s/"index_name": ".ds-my-data-stream-2099.03.07-000001"/"index_name": $body.data_streams.0.indices.0.index_name/] +// TESTRESPONSE[s/"index_uuid": "xCEhwsp8Tey0-FLNFYVwSg"/"index_uuid": $body.data_streams.0.indices.0.index_uuid/] +// TESTRESPONSE[s/"index_name": ".ds-my-data-stream-2099.03.08-000002"/"index_name": $body.data_streams.0.indices.1.index_name/] +// TESTRESPONSE[s/"index_uuid": "PA_JquKGSiKcAKBA8DJ5gw"/"index_uuid": $body.data_streams.0.indices.1.index_uuid/] +// TESTRESPONSE[s/"index_name": ".ds-my-data-stream-two-2099.03.08-000001"/"index_name": $body.data_streams.1.indices.0.index_name/] +// TESTRESPONSE[s/"index_uuid": "3liBu2SYS5axasRt6fUIpA"/"index_uuid": $body.data_streams.1.indices.0.index_uuid/] +// TESTRESPONSE[s/"status": "GREEN"/"status": "YELLOW"/] +// TESTRESPONSE[s/"replicated": false/"replicated": false,"failure_store":{"enabled": false, "indices": [], "rollover_on_write": true}/] diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsAction.java index 5728e5c0a8dbf..f37158ba3fc6e 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsAction.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsAction.java @@ -24,6 +24,7 @@ import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.health.ClusterStateHealth; +import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.metadata.DataStreamFailureStoreSettings; import org.elasticsearch.cluster.metadata.DataStreamGlobalRetentionSettings; @@ -40,6 +41,9 @@ import org.elasticsearch.core.Tuple; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexMode; +import org.elasticsearch.index.IndexSettingProvider; +import org.elasticsearch.index.IndexSettingProviders; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.indices.SystemDataStreamDescriptor; import org.elasticsearch.indices.SystemIndices; import org.elasticsearch.injection.guice.Inject; @@ -53,6 +57,7 @@ import java.util.Comparator; import java.util.HashMap; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.stream.Collectors; @@ -68,6 +73,7 @@ public class TransportGetDataStreamsAction extends TransportMasterNodeReadProjec private final ClusterSettings clusterSettings; private final DataStreamGlobalRetentionSettings globalRetentionSettings; private final DataStreamFailureStoreSettings dataStreamFailureStoreSettings; + private final IndexSettingProviders indexSettingProviders; private final Client client; @Inject @@ -81,6 +87,7 @@ public TransportGetDataStreamsAction( SystemIndices systemIndices, DataStreamGlobalRetentionSettings globalRetentionSettings, DataStreamFailureStoreSettings dataStreamFailureStoreSettings, + IndexSettingProviders indexSettingProviders, Client client ) { super( @@ -99,6 +106,7 @@ public TransportGetDataStreamsAction( this.globalRetentionSettings = globalRetentionSettings; clusterSettings = clusterService.getClusterSettings(); this.dataStreamFailureStoreSettings = dataStreamFailureStoreSettings; + this.indexSettingProviders = indexSettingProviders; this.client = new OriginSettingClient(client, "stack"); } @@ -131,6 +139,7 @@ public void onResponse(DataStreamsStatsAction.Response response) { clusterSettings, globalRetentionSettings, dataStreamFailureStoreSettings, + indexSettingProviders, maxTimestamps ) ); @@ -151,12 +160,43 @@ public void onFailure(Exception e) { clusterSettings, globalRetentionSettings, dataStreamFailureStoreSettings, + indexSettingProviders, null ) ); } } + /** + * Resolves the index mode ("index.mode" setting) for the given data stream, from the template or additional setting providers + */ + @Nullable + static IndexMode resolveMode( + ProjectState state, + IndexSettingProviders indexSettingProviders, + DataStream dataStream, + Settings settings, + ComposableIndexTemplate indexTemplate + ) { + IndexMode indexMode = state.metadata().retrieveIndexModeFromTemplate(indexTemplate); + for (IndexSettingProvider provider : indexSettingProviders.getIndexSettingProviders()) { + Settings addlSettinsg = provider.getAdditionalIndexSettings( + MetadataIndexTemplateService.VALIDATE_INDEX_NAME, + dataStream.getName(), + indexMode, + state.metadata(), + Instant.now(), + settings, + List.of() + ); + var rawMode = addlSettinsg.get(IndexSettings.MODE.getKey()); + if (rawMode != null) { + indexMode = Enum.valueOf(IndexMode.class, rawMode.toUpperCase(Locale.ROOT)); + } + } + return indexMode; + } + static GetDataStreamAction.Response innerOperation( ProjectState state, GetDataStreamAction.Request request, @@ -165,6 +205,7 @@ static GetDataStreamAction.Response innerOperation( ClusterSettings clusterSettings, DataStreamGlobalRetentionSettings globalRetentionSettings, DataStreamFailureStoreSettings dataStreamFailureStoreSettings, + IndexSettingProviders indexSettingProviders, @Nullable Map maxTimestamps ) { List dataStreams = getDataStreams(state.metadata(), indexNameExpressionResolver, request); @@ -177,6 +218,7 @@ static GetDataStreamAction.Response innerOperation( final String indexTemplate; boolean indexTemplatePreferIlmValue = true; String ilmPolicyName = null; + IndexMode indexMode = dataStream.getIndexMode(); if (dataStream.isSystem()) { SystemDataStreamDescriptor dataStreamDescriptor = systemIndices.findMatchingDataStreamDescriptor(dataStream.getName()); indexTemplate = dataStreamDescriptor != null ? dataStreamDescriptor.getDataStreamName() : null; @@ -186,6 +228,15 @@ static GetDataStreamAction.Response innerOperation( dataStreamDescriptor.getComponentTemplates() ); ilmPolicyName = settings.get(IndexMetadata.LIFECYCLE_NAME); + if (indexMode == null) { + indexMode = resolveMode( + state, + indexSettingProviders, + dataStream, + settings, + dataStreamDescriptor.getComposableIndexTemplate() + ); + } indexTemplatePreferIlmValue = PREFER_ILM_SETTING.get(settings); } } else { @@ -193,6 +244,15 @@ static GetDataStreamAction.Response innerOperation( if (indexTemplate != null) { Settings settings = MetadataIndexTemplateService.resolveSettings(state.metadata(), indexTemplate); ilmPolicyName = settings.get(IndexMetadata.LIFECYCLE_NAME); + if (indexMode == null && state.metadata().templatesV2().get(indexTemplate) != null) { + indexMode = resolveMode( + state, + indexSettingProviders, + dataStream, + settings, + state.metadata().templatesV2().get(indexTemplate) + ); + } indexTemplatePreferIlmValue = PREFER_ILM_SETTING.get(settings); } else { LOGGER.warn( @@ -285,7 +345,9 @@ public int compareTo(IndexInfo o) { timeSeries, backingIndicesSettingsValues, indexTemplatePreferIlmValue, - maxTimestamps == null ? null : maxTimestamps.get(dataStream.getName()) + maxTimestamps == null ? null : maxTimestamps.get(dataStream.getName()), + // Default to standard mode if not specified; should we set this to "unset" or "unspecified" instead? + indexMode == null ? IndexMode.STANDARD.getName() : indexMode.getName() ) ); } @@ -314,7 +376,11 @@ private static void collectIndexSettingsValues( } else { managedBy = ManagedBy.UNMANAGED; } - backingIndicesSettingsValues.put(index, new IndexProperties(preferIlm, indexMetadata.getLifecyclePolicyName(), managedBy)); + String indexMode = IndexSettings.MODE.get(indexMetadata.getSettings()).getName(); + backingIndicesSettingsValues.put( + index, + new IndexProperties(preferIlm, indexMetadata.getLifecyclePolicyName(), managedBy, indexMode) + ); } } diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/GetDataStreamsResponseTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/GetDataStreamsResponseTests.java index 9414943cbb439..d9efa4d458f49 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/GetDataStreamsResponseTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/GetDataStreamsResponseTests.java @@ -91,13 +91,13 @@ public void testResponseIlmAndDataStreamLifecycleRepresentation() throws Excepti String ilmPolicyName = "rollover-30days"; Map indexSettingsValues = Map.of( firstGenerationIndex, - new Response.IndexProperties(true, ilmPolicyName, ManagedBy.ILM), + new Response.IndexProperties(true, ilmPolicyName, ManagedBy.ILM, null), secondGenerationIndex, - new Response.IndexProperties(false, ilmPolicyName, ManagedBy.LIFECYCLE), + new Response.IndexProperties(false, ilmPolicyName, ManagedBy.LIFECYCLE, null), writeIndex, - new Response.IndexProperties(false, null, ManagedBy.LIFECYCLE), + new Response.IndexProperties(false, null, ManagedBy.LIFECYCLE, null), failureStoreIndex, - new Response.IndexProperties(false, null, ManagedBy.LIFECYCLE) + new Response.IndexProperties(false, null, ManagedBy.LIFECYCLE, null) ); Response.DataStreamInfo dataStreamInfo = new Response.DataStreamInfo( @@ -109,6 +109,7 @@ public void testResponseIlmAndDataStreamLifecycleRepresentation() throws Excepti null, indexSettingsValues, false, + null, null ); Response response = new Response(List.of(dataStreamInfo)); @@ -195,13 +196,13 @@ public void testResponseIlmAndDataStreamLifecycleRepresentation() throws Excepti String ilmPolicyName = "rollover-30days"; Map indexSettingsValues = Map.of( firstGenerationIndex, - new Response.IndexProperties(true, ilmPolicyName, ManagedBy.ILM), + new Response.IndexProperties(true, ilmPolicyName, ManagedBy.ILM, null), secondGenerationIndex, - new Response.IndexProperties(true, ilmPolicyName, ManagedBy.ILM), + new Response.IndexProperties(true, ilmPolicyName, ManagedBy.ILM, null), writeIndex, - new Response.IndexProperties(false, null, ManagedBy.UNMANAGED), + new Response.IndexProperties(false, null, ManagedBy.UNMANAGED, null), failureStoreIndex, - new Response.IndexProperties(false, null, ManagedBy.UNMANAGED) + new Response.IndexProperties(false, null, ManagedBy.UNMANAGED, null) ); Response.DataStreamInfo dataStreamInfo = new Response.DataStreamInfo( @@ -213,6 +214,7 @@ public void testResponseIlmAndDataStreamLifecycleRepresentation() throws Excepti null, indexSettingsValues, false, + null, null ); Response response = new Response(List.of(dataStreamInfo)); @@ -309,7 +311,8 @@ private Response.DataStreamInfo mutateInstance(Response.DataStreamInfo instance) new Response.IndexProperties( randomBoolean(), randomAlphaOfLengthBetween(50, 100), - randomBoolean() ? ManagedBy.ILM : ManagedBy.LIFECYCLE + randomBoolean() ? ManagedBy.ILM : ManagedBy.LIFECYCLE, + null ) ) ); @@ -328,7 +331,8 @@ private Response.DataStreamInfo mutateInstance(Response.DataStreamInfo instance) timeSeries, indexSettings, templatePreferIlm, - maximumTimestamp + maximumTimestamp, + null ); } @@ -349,7 +353,8 @@ private Map generateRandomIndexSettingsValues() new Response.IndexProperties( randomBoolean(), randomAlphaOfLengthBetween(50, 100), - randomBoolean() ? ManagedBy.ILM : ManagedBy.LIFECYCLE + randomBoolean() ? ManagedBy.ILM : ManagedBy.LIFECYCLE, + randomBoolean() ? randomFrom(IndexMode.values()).getName() : null ) ); } @@ -367,7 +372,8 @@ private Response.DataStreamInfo generateRandomDataStreamInfo() { timeSeries != null ? new Response.TimeSeries(timeSeries) : null, generateRandomIndexSettingsValues(), randomBoolean(), - usually() ? randomNonNegativeLong() : null + usually() ? randomNonNegativeLong() : null, + usually() ? randomFrom(IndexMode.values()).getName() : null ); } } diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsActionTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsActionTests.java index 0963647e24173..15252528e952b 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsActionTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsActionTests.java @@ -24,7 +24,9 @@ import org.elasticsearch.core.TimeValue; import org.elasticsearch.core.Tuple; import org.elasticsearch.index.Index; +import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexNotFoundException; +import org.elasticsearch.index.IndexSettingProviders; import org.elasticsearch.indices.SystemIndices; import org.elasticsearch.indices.TestIndexNameExpressionResolver; import org.elasticsearch.test.ESTestCase; @@ -32,6 +34,7 @@ import java.time.Instant; import java.time.temporal.ChronoUnit; import java.util.List; +import java.util.Set; import static org.elasticsearch.cluster.metadata.DataStreamTestHelper.getClusterStateWithDataStreams; import static org.elasticsearch.test.LambdaMatchers.transformedItemsMatch; @@ -181,6 +184,7 @@ public void testGetTimeSeriesDataStream() { ClusterSettings.createBuiltInClusterSettings(), dataStreamGlobalRetentionSettings, emptyDataStreamFailureStoreSettings, + new IndexSettingProviders(Set.of()), null ); assertThat( @@ -213,6 +217,7 @@ public void testGetTimeSeriesDataStream() { ClusterSettings.createBuiltInClusterSettings(), dataStreamGlobalRetentionSettings, emptyDataStreamFailureStoreSettings, + new IndexSettingProviders(Set.of()), null ); assertThat( @@ -266,6 +271,7 @@ public void testGetTimeSeriesDataStreamWithOutOfOrderIndices() { ClusterSettings.createBuiltInClusterSettings(), dataStreamGlobalRetentionSettings, emptyDataStreamFailureStoreSettings, + new IndexSettingProviders(Set.of()), null ); assertThat( @@ -307,6 +313,7 @@ public void testGetTimeSeriesMixedDataStream() { ClusterSettings.createBuiltInClusterSettings(), dataStreamGlobalRetentionSettings, emptyDataStreamFailureStoreSettings, + new IndexSettingProviders(Set.of()), null ); @@ -341,6 +348,7 @@ public void testPassingGlobalRetention() { ClusterSettings.createBuiltInClusterSettings(), dataStreamGlobalRetentionSettings, emptyDataStreamFailureStoreSettings, + new IndexSettingProviders(Set.of()), null ); assertThat(response.getGlobalRetention(), nullValue()); @@ -367,6 +375,7 @@ public void testPassingGlobalRetention() { ClusterSettings.createBuiltInClusterSettings(), withGlobalRetentionSettings, emptyDataStreamFailureStoreSettings, + new IndexSettingProviders(Set.of()), null ); assertThat(response.getGlobalRetention(), equalTo(globalRetention)); @@ -394,6 +403,7 @@ public void testDataStreamIsFailureStoreEffectivelyEnabled_disabled() { ClusterSettings.createBuiltInClusterSettings(), dataStreamGlobalRetentionSettings, emptyDataStreamFailureStoreSettings, + new IndexSettingProviders(Set.of()), null ); assertThat(response.getDataStreams(), hasSize(1)); @@ -423,6 +433,7 @@ public void testDataStreamIsFailureStoreEffectivelyEnabled_enabledExplicitly() { ClusterSettings.createBuiltInClusterSettings(), dataStreamGlobalRetentionSettings, emptyDataStreamFailureStoreSettings, + new IndexSettingProviders(Set.of()), null ); assertThat(response.getDataStreams(), hasSize(1)); @@ -457,9 +468,63 @@ public void testDataStreamIsFailureStoreEffectivelyEnabled_enabledByClusterSetti .build() ) ), + new IndexSettingProviders(Set.of()), null ); assertThat(response.getDataStreams(), hasSize(1)); assertThat(response.getDataStreams().getFirst().isFailureStoreEffectivelyEnabled(), is(true)); } + + public void testProvidersAffectMode() { + ClusterState state; + var projectId = randomProjectIdOrDefault(); + { + state = DataStreamTestHelper.getClusterStateWithDataStreams( + projectId, + List.of(Tuple.tuple("data-stream-1", 2)), + List.of(), + System.currentTimeMillis(), + Settings.EMPTY, + 0, + false, + false + ); + } + + var req = new GetDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] {}); + var response = TransportGetDataStreamsAction.innerOperation( + state.projectState(projectId), + req, + resolver, + systemIndices, + ClusterSettings.createBuiltInClusterSettings(), + dataStreamGlobalRetentionSettings, + emptyDataStreamFailureStoreSettings, + new IndexSettingProviders( + Set.of( + ( + indexName, + dataStreamName, + templateIndexMode, + metadata, + resolvedAt, + indexTemplateAndCreateRequestSettings, + combinedTemplateMappings) -> Settings.builder().put("index.mode", IndexMode.LOOKUP).build() + ) + ), + null + ); + assertThat(response.getDataStreams().getFirst().getIndexModeName(), equalTo("lookup")); + assertThat( + response.getDataStreams() + .getFirst() + .getIndexSettingsValues() + .values() + .stream() + .findFirst() + .map(GetDataStreamAction.Response.IndexProperties::indexMode) + .orElse("bad"), + equalTo("standard") + ); + } } diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java index 0dd856ab8c64e..ef6a0176b05fd 100644 --- a/server/src/main/java/org/elasticsearch/TransportVersions.java +++ b/server/src/main/java/org/elasticsearch/TransportVersions.java @@ -183,6 +183,7 @@ static TransportVersion def(int id) { public static final TransportVersion STORED_SCRIPT_CONTENT_LENGTH = def(9_019_0_00); public static final TransportVersion JINA_AI_EMBEDDING_TYPE_SUPPORT_ADDED = def(9_020_0_00); public static final TransportVersion RE_REMOVE_MIN_COMPATIBLE_SHARD_NODE = def(9_021_0_00); + public static final TransportVersion INCLUDE_INDEX_MODE_IN_GET_DATA_STREAM = def(9_022_0_00); /* * STOP! READ THIS FIRST! No, really, diff --git a/server/src/main/java/org/elasticsearch/action/datastreams/GetDataStreamAction.java b/server/src/main/java/org/elasticsearch/action/datastreams/GetDataStreamAction.java index 5dd60a1122bf7..c0fae14434138 100644 --- a/server/src/main/java/org/elasticsearch/action/datastreams/GetDataStreamAction.java +++ b/server/src/main/java/org/elasticsearch/action/datastreams/GetDataStreamAction.java @@ -232,6 +232,7 @@ public static class DataStreamInfo implements SimpleDiffable, To ); public static final ParseField FAILURE_STORE_ENABLED = new ParseField("enabled"); public static final ParseField MAXIMUM_TIMESTAMP = new ParseField("maximum_timestamp"); + public static final ParseField INDEX_MODE = new ParseField("index_mode"); private final DataStream dataStream; private final ClusterHealthStatus dataStreamStatus; @@ -246,6 +247,8 @@ public static class DataStreamInfo implements SimpleDiffable, To private final boolean templatePreferIlmValue; @Nullable private final Long maximumTimestamp; + @Nullable + private final String indexMode; public DataStreamInfo( DataStream dataStream, @@ -256,7 +259,8 @@ public DataStreamInfo( @Nullable TimeSeries timeSeries, Map indexSettingsValues, boolean templatePreferIlmValue, - @Nullable Long maximumTimestamp + @Nullable Long maximumTimestamp, + @Nullable String indexMode ) { this.dataStream = dataStream; this.failureStoreEffectivelyEnabled = failureStoreEffectivelyEnabled; @@ -267,6 +271,7 @@ public DataStreamInfo( this.indexSettingsValues = indexSettingsValues; this.templatePreferIlmValue = templatePreferIlmValue; this.maximumTimestamp = maximumTimestamp; + this.indexMode = indexMode; } @SuppressWarnings("unchecked") @@ -287,6 +292,9 @@ public DataStreamInfo( : Map.of(); this.templatePreferIlmValue = in.getTransportVersion().onOrAfter(V_8_11_X) ? in.readBoolean() : true; this.maximumTimestamp = in.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0) ? in.readOptionalVLong() : null; + this.indexMode = in.getTransportVersion().onOrAfter(TransportVersions.INCLUDE_INDEX_MODE_IN_GET_DATA_STREAM) + ? in.readOptionalString() + : null; } public DataStream getDataStream() { @@ -329,6 +337,11 @@ public Long getMaximumTimestamp() { return maximumTimestamp; } + @Nullable + public String getIndexModeName() { + return indexMode; + } + @Override public void writeTo(StreamOutput out) throws IOException { dataStream.writeTo(out); @@ -348,6 +361,9 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { out.writeOptionalVLong(maximumTimestamp); } + if (out.getTransportVersion().onOrAfter(TransportVersions.INCLUDE_INDEX_MODE_IN_GET_DATA_STREAM)) { + out.writeOptionalString(indexMode); + } } @Override @@ -398,6 +414,9 @@ public XContentBuilder toXContent( if (this.maximumTimestamp != null) { builder.field(MAXIMUM_TIMESTAMP.getPreferredName(), this.maximumTimestamp); } + if (this.indexMode != null) { + builder.field(INDEX_MODE.getPreferredName(), indexMode); + } addAutoShardingEvent(builder, params, dataStream.getAutoShardingEvent()); if (timeSeries != null) { builder.startObject(TIME_SERIES.getPreferredName()); @@ -441,6 +460,7 @@ private XContentBuilder indicesToXContent(XContentBuilder builder, List i builder.field(ILM_POLICY_FIELD.getPreferredName(), indexProperties.ilmPolicyName()); } builder.field(MANAGED_BY.getPreferredName(), indexProperties.managedBy.displayValue); + builder.field(INDEX_MODE.getPreferredName(), indexProperties.indexMode); } builder.endObject(); } @@ -500,7 +520,8 @@ public boolean equals(Object o) { && Objects.equals(ilmPolicyName, that.ilmPolicyName) && Objects.equals(timeSeries, that.timeSeries) && Objects.equals(indexSettingsValues, that.indexSettingsValues) - && Objects.equals(maximumTimestamp, that.maximumTimestamp); + && Objects.equals(maximumTimestamp, that.maximumTimestamp) + && Objects.equals(indexMode, that.indexMode); } @Override @@ -514,7 +535,8 @@ public int hashCode() { timeSeries, indexSettingsValues, templatePreferIlmValue, - maximumTimestamp + maximumTimestamp, + indexMode ); } } @@ -551,9 +573,18 @@ public int hashCode() { * Encapsulates the configured properties we want to display for each backing index. * They'll usually be settings values, but could also be additional properties derived from settings. */ - public record IndexProperties(boolean preferIlm, @Nullable String ilmPolicyName, ManagedBy managedBy) implements Writeable { + public record IndexProperties(boolean preferIlm, @Nullable String ilmPolicyName, ManagedBy managedBy, @Nullable String indexMode) + implements + Writeable { public IndexProperties(StreamInput in) throws IOException { - this(in.readBoolean(), in.readOptionalString(), in.readEnum(ManagedBy.class)); + this( + in.readBoolean(), + in.readOptionalString(), + in.readEnum(ManagedBy.class), + in.getTransportVersion().onOrAfter(TransportVersions.INCLUDE_INDEX_MODE_IN_GET_DATA_STREAM) + ? in.readOptionalString() + : "unknown" + ); } @Override @@ -561,6 +592,9 @@ public void writeTo(StreamOutput out) throws IOException { out.writeBoolean(preferIlm); out.writeOptionalString(ilmPolicyName); out.writeEnum(managedBy); + if (out.getTransportVersion().onOrAfter(TransportVersions.INCLUDE_INDEX_MODE_IN_GET_DATA_STREAM)) { + out.writeOptionalString(indexMode); + } } } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java index d1b91d7817927..e58f766a9781b 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java @@ -91,6 +91,9 @@ public class MetadataIndexTemplateService { public static final String DEFAULT_TIMESTAMP_FIELD = "@timestamp"; public static final CompressedXContent DEFAULT_TIMESTAMP_MAPPING_WITHOUT_ROUTING; + // Names used for validating templates when we do not know the index or data stream name + public static final String VALIDATE_INDEX_NAME = "validate-index-name"; + public static final String VALIDATE_DATA_STREAM_NAME = "validate-data-stream-name"; private static final CompressedXContent DEFAULT_TIMESTAMP_MAPPING_WITH_ROUTING; @@ -714,8 +717,8 @@ void validateIndexTemplateV2(ProjectMetadata projectMetadata, String name, Compo var finalSettings = Settings.builder(); for (var provider : indexSettingProviders) { var newAdditionalSettings = provider.getAdditionalIndexSettings( - "validate-index-name", - indexTemplate.getDataStreamTemplate() != null ? "validate-data-stream-name" : null, + VALIDATE_INDEX_NAME, + indexTemplate.getDataStreamTemplate() != null ? VALIDATE_DATA_STREAM_NAME : null, projectMetadata.retrieveIndexModeFromTemplate(indexTemplate), projectMetadata, now, diff --git a/server/src/test/java/org/elasticsearch/action/datastreams/GetDataStreamActionTests.java b/server/src/test/java/org/elasticsearch/action/datastreams/GetDataStreamActionTests.java index feb00728c858e..dfff09ffd5ddb 100644 --- a/server/src/test/java/org/elasticsearch/action/datastreams/GetDataStreamActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/datastreams/GetDataStreamActionTests.java @@ -92,6 +92,7 @@ private static GetDataStreamAction.Response.DataStreamInfo newDataStreamInfo(boo null, Map.of(), randomBoolean(), + null, null ); } diff --git a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProvider.java b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProvider.java index 28e3191ec7763..22a28e27dd243 100644 --- a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProvider.java +++ b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProvider.java @@ -12,6 +12,7 @@ import org.apache.lucene.util.SetOnce; import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.metadata.MetadataIndexTemplateService; import org.elasticsearch.cluster.metadata.ProjectMetadata; import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.compress.CompressedXContent; @@ -100,7 +101,9 @@ public Settings getAdditionalIndexSettings( ) { Settings.Builder settingsBuilder = null; boolean isLogsDB = templateIndexMode == IndexMode.LOGSDB; - boolean isTemplateValidation = "validate-index-name".equals(indexName); + // This index name is used when validating component and index templates, we should skip this check in that case. + // (See MetadataIndexTemplateService#validateIndexTemplateV2(...) method) + boolean isTemplateValidation = MetadataIndexTemplateService.VALIDATE_INDEX_NAME.equals(indexName); // Inject logsdb index mode, based on the logs pattern. if (isLogsdbEnabled @@ -118,8 +121,6 @@ && matchesLogsPattern(dataStreamName)) { if (mappingHints.hasSyntheticSourceUsage && supportFallbackToStoredSource.get() && minNodeVersion.get().get().onOrAfter(Version.V_8_17_0)) { - // This index name is used when validating component and index templates, we should skip this check in that case. - // (See MetadataIndexTemplateService#validateIndexTemplateV2(...) method) boolean legacyLicensedUsageOfSyntheticSourceAllowed = isLegacyLicensedUsageOfSyntheticSourceAllowed( templateIndexMode, indexName, @@ -216,7 +217,7 @@ MappingHints getMappingHints( Settings indexTemplateAndCreateRequestSettings, List combinedTemplateMappings ) { - if ("validate-index-name".equals(indexName)) { + if (MetadataIndexTemplateService.VALIDATE_INDEX_NAME.equals(indexName)) { // This index name is used when validating component and index templates, we should skip this check in that case. // (See MetadataIndexTemplateService#validateIndexTemplateV2(...) method) return MappingHints.EMPTY; diff --git a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProviderTests.java b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProviderTests.java index ca5246cb312c6..cf550c3a5ab72 100644 --- a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProviderTests.java +++ b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProviderTests.java @@ -14,6 +14,7 @@ import org.elasticsearch.cluster.metadata.DataStreamTestHelper; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.cluster.metadata.MetadataIndexTemplateService; import org.elasticsearch.cluster.metadata.ProjectMetadata; import org.elasticsearch.cluster.metadata.Template; import org.elasticsearch.common.compress.CompressedXContent; @@ -475,7 +476,7 @@ public void testNewIndexHasSyntheticSourceUsage() throws IOException { } public void testValidateIndexName() throws IOException { - String indexName = "validate-index-name"; + String indexName = MetadataIndexTemplateService.VALIDATE_INDEX_NAME; String mapping = """ { "_doc": {