From dc5f3cd9d50145e158b62df61b51d4409fdd37d0 Mon Sep 17 00:00:00 2001 From: gmarouli Date: Fri, 11 Apr 2025 11:54:38 +0300 Subject: [PATCH 1/6] Expose failure store lifecycle information via the `GET` data stream API --- .../rest/RestGetDataStreamsAction.java | 11 +- .../data_stream/240_failure_store_info.yml | 193 ++++++++++++++++++ .../datastreams/GetDataStreamAction.java | 32 ++- .../metadata/DataStreamFailureStore.java | 1 + 4 files changed, 228 insertions(+), 9 deletions(-) create mode 100644 modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/240_failure_store_info.yml diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestGetDataStreamsAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestGetDataStreamsAction.java index d1e04eb4072b6..39bf5f793575f 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestGetDataStreamsAction.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestGetDataStreamsAction.java @@ -11,6 +11,8 @@ import org.elasticsearch.action.datastreams.GetDataStreamAction; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.client.internal.node.NodeClient; +import org.elasticsearch.cluster.metadata.DataStream; +import org.elasticsearch.cluster.metadata.DataStreamFailureStore; import org.elasticsearch.cluster.metadata.DataStreamLifecycle; import org.elasticsearch.common.Strings; import org.elasticsearch.common.util.set.Sets; @@ -23,7 +25,10 @@ import org.elasticsearch.rest.action.RestToXContentListener; import java.util.List; +import java.util.Objects; import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; import static org.elasticsearch.rest.RestRequest.Method.GET; @@ -45,6 +50,10 @@ public class RestGetDataStreamsAction extends BaseRestHandler { ) ) ); + private static final Set CAPABILITIES = Stream.of( + DataStreamLifecycle.EFFECTIVE_RETENTION_REST_API_CAPABILITY, + DataStream.isFailureStoreFeatureFlagEnabled() ? DataStreamFailureStore.FAILURES_LIFECYCLE_API_CAPABILITY : null + ).filter(Objects::nonNull).collect(Collectors.toSet()); @Override public String getName() { @@ -79,7 +88,7 @@ public boolean allowSystemIndexAccessByDefault() { @Override public Set supportedCapabilities() { - return Set.of(DataStreamLifecycle.EFFECTIVE_RETENTION_REST_API_CAPABILITY); + return CAPABILITIES; } @Override diff --git a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/240_failure_store_info.yml b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/240_failure_store_info.yml new file mode 100644 index 0000000000000..3a4ef3f73f3da --- /dev/null +++ b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/240_failure_store_info.yml @@ -0,0 +1,193 @@ +setup: + - requires: + test_runner_features: [ capabilities, allowed_warnings ] + reason: "Exposing failures lifecycle config in templates was added in 9.1+" + capabilities: + - method: GET + path: /_data_stream/{target} + capabilities: [ 'failures_lifecycle' ] + - do: + allowed_warnings: + - "index template [my-template1] has index patterns [fs-data-stream] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template1] will take precedence during new index creation" + indices.put_index_template: + name: my-template1 + body: + index_patterns: [fs-data-stream] + template: + settings: + index.number_of_replicas: 1 + mappings: + properties: + '@timestamp': + type: date + count: + type: long + lifecycle: {} + data_stream_options: + failure_store: + enabled: true + data_stream: {} + + - do: + allowed_warnings: + - "index template [my-template2] has index patterns [fs-default-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template2] will take precedence during new index creation" + indices.put_index_template: + name: my-template2 + body: + index_patterns: [ fs-default-* ] + template: + settings: + index.number_of_replicas: 1 + mappings: + properties: + '@timestamp': + type: date + count: + type: long + data_stream: { } + + - do: + allowed_warnings: + - "index template [my-template3] has index patterns [no-fs-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template3] will take precedence during new index creation" + indices.put_index_template: + name: my-template3 + body: + index_patterns: [ no-fs-* ] + template: + settings: + index.number_of_replicas: 1 + mappings: + properties: + '@timestamp': + type: date + count: + type: long + data_stream: { } + + - do: + cluster.put_settings: + body: + persistent: + data_streams.failure_store.enabled: 'fs-default*' + +--- +teardown: + - do: + indices.delete_data_stream: + name: fs-data-stream + ignore: 404 + + - do: + indices.delete_index_template: + name: fs-default-data-stream + ignore: 404 + + - do: + indices.delete_index_template: + name: no-fs-data-stream + ignore: 404 + +--- +"Get failure store info from explicitly enabled failure store": + - do: + indices.create_data_stream: + name: fs-data-stream + - is_true: acknowledged + + - do: + indices.get_data_stream: + name: "fs-data-stream" + - match: { data_streams.0.name: fs-data-stream } + - match: { data_streams.0.timestamp_field.name: '@timestamp' } + - match: { data_streams.0.generation: 1 } + - length: { data_streams.0.indices: 1 } + - match: { data_streams.0.indices.0.index_name: '/\.ds-fs-data-stream-(\d{4}\.\d{2}\.\d{2}-)?000001/' } + - match: { data_streams.0.template: 'my-template1' } + - match: { data_streams.0.failure_store.enabled: true } + - match: { data_streams.0.failure_store.lifecycle.enabled: true} + - match: { data_streams.0.failure_store.indices: [] } + + # Initialize failure store + - do: + index: + index: fs-data-stream + refresh: true + body: + '@timestamp': '2020-12-12' + count: 'invalid value' + + - do: + indices.get_data_stream: + name: "fs-data-stream" + - match: { data_streams.0.name: fs-data-stream } + - match: { data_streams.0.timestamp_field.name: '@timestamp' } + - match: { data_streams.0.generation: 2 } + - length: { data_streams.0.indices: 1 } + - match: { data_streams.0.indices.0.index_name: '/\.ds-fs-data-stream-(\d{4}\.\d{2}\.\d{2}-)?000001/' } + - match: { data_streams.0.template: 'my-template1' } + - match: { data_streams.0.failure_store.enabled: true } + - match: { data_streams.0.failure_store.lifecycle.enabled: true } + - length: { data_streams.0.failure_store.indices: 1 } + - match: { data_streams.0.failure_store.indices.0.index_name: '/\.fs-fs-data-stream-(\d{4}\.\d{2}\.\d{2}-)?000002/' } + - is_false: data_streams.0.failure_store.indices.0.prefer_ilm + - match: { data_streams.0.failure_store.indices.0.managed_by: 'Data stream lifecycle' } + +--- +"Get failure store info from disabled failure store": + - do: + indices.create_data_stream: + name: no-fs-data-stream + - is_true: acknowledged + + - do: + indices.get_data_stream: + name: "no-fs-data-stream" + - match: { data_streams.0.name: no-fs-data-stream } + - match: { data_streams.0.timestamp_field.name: '@timestamp' } + - match: { data_streams.0.generation: 1 } + - length: { data_streams.0.indices: 1 } + - match: { data_streams.0.indices.0.index_name: '/\.ds-no-fs-data-stream-(\d{4}\.\d{2}\.\d{2}-)?000001/' } + - match: { data_streams.0.template: 'my-template3' } + - match: { data_streams.0.failure_store.enabled: false } + - is_false: data_streams.0.failure_store.lifecycle + - match: { data_streams.0.failure_store.indices: [] } + +--- +"Get failure store info from explicitly enabled failure store and disabled lifecycle": + - do: + indices.create_data_stream: + name: fs-data-stream + - is_true: acknowledged + + - do: + indices.put_data_lifecycle: + name: "fs-data-stream" + body: + enabled: false + + - is_true: acknowledged + + # Initialize failure store + - do: + index: + index: fs-data-stream + refresh: true + body: + '@timestamp': '2020-12-12' + count: 'invalid value' + + - do: + indices.get_data_stream: + name: "fs-data-stream" + - match: { data_streams.0.name: fs-data-stream } + - match: { data_streams.0.timestamp_field.name: '@timestamp' } + - match: { data_streams.0.generation: 2 } + - length: { data_streams.0.indices: 1 } + - match: { data_streams.0.indices.0.index_name: '/\.ds-fs-data-stream-(\d{4}\.\d{2}\.\d{2}-)?000001/' } + - match: { data_streams.0.template: 'my-template1' } + - match: { data_streams.0.failure_store.enabled: true } + - match: { data_streams.0.failure_store.lifecycle.enabled: false } + - length: { data_streams.0.failure_store.indices: 1 } + - match: { data_streams.0.failure_store.indices.0.index_name: '/\.fs-fs-data-stream-(\d{4}\.\d{2}\.\d{2}-)?000002/' } + - is_false: data_streams.0.failure_store.indices.0.prefer_ilm + - match: { data_streams.0.failure_store.indices.0.managed_by: 'Unmanaged' } diff --git a/server/src/main/java/org/elasticsearch/action/datastreams/GetDataStreamAction.java b/server/src/main/java/org/elasticsearch/action/datastreams/GetDataStreamAction.java index 147c34222138e..424fe2340f6f7 100644 --- a/server/src/main/java/org/elasticsearch/action/datastreams/GetDataStreamAction.java +++ b/server/src/main/java/org/elasticsearch/action/datastreams/GetDataStreamAction.java @@ -371,7 +371,7 @@ public XContentBuilder toXContent( .field(DataStream.NAME_FIELD.getPreferredName(), DataStream.TIMESTAMP_FIELD_NAME) .endObject(); - indicesToXContent(builder, dataStream.getIndices()); + indicesToXContent(builder, dataStream.getIndices(), false); builder.field(DataStream.GENERATION_FIELD.getPreferredName(), dataStream.getGeneration()); if (dataStream.getMetadata() != null) { builder.field(DataStream.METADATA_FIELD.getPreferredName(), dataStream.getMetadata()); @@ -423,15 +423,21 @@ public XContentBuilder toXContent( DataStream.ROLLOVER_ON_WRITE_FIELD.getPreferredName(), dataStream.getFailureComponent().isRolloverOnWrite() ); - indicesToXContent(builder, dataStream.getFailureIndices()); + indicesToXContent(builder, dataStream.getFailureIndices(), true); addAutoShardingEvent(builder, params, dataStream.getFailureComponent().getAutoShardingEvent()); + DataStreamLifecycle failuresLifecycle = dataStream.getFailuresLifecycle(); + if (failuresLifecycle != null) { + builder.field(LIFECYCLE_FIELD.getPreferredName()); + failuresLifecycle.toXContent(builder, params, rolloverConfiguration, globalRetention, dataStream.isInternal()); + } builder.endObject(); } builder.endObject(); return builder; } - private XContentBuilder indicesToXContent(XContentBuilder builder, List indices) throws IOException { + private XContentBuilder indicesToXContent(XContentBuilder builder, List indices, boolean failureIndices) + throws IOException { builder.field(DataStream.INDICES_FIELD.getPreferredName()); builder.startArray(); for (Index index : indices) { @@ -439,12 +445,22 @@ private XContentBuilder indicesToXContent(XContentBuilder builder, List i index.toXContentFragment(builder); IndexProperties indexProperties = indexSettingsValues.get(index); if (indexProperties != null) { - builder.field(PREFER_ILM.getPreferredName(), indexProperties.preferIlm()); - if (indexProperties.ilmPolicyName() != null) { - builder.field(ILM_POLICY_FIELD.getPreferredName(), indexProperties.ilmPolicyName()); - } builder.field(MANAGED_BY.getPreferredName(), indexProperties.managedBy.displayValue); - builder.field(INDEX_MODE.getPreferredName(), indexProperties.indexMode); + // Failure indices have more limitation than backing indices, + // so we hide some index properties that are less relevant + if (failureIndices) { + // We only display ILM info, if this index has an ILM policy + if (indexProperties.ilmPolicyName() != null) { + builder.field(PREFER_ILM.getPreferredName(), indexProperties.preferIlm()); + builder.field(ILM_POLICY_FIELD.getPreferredName(), indexProperties.ilmPolicyName()); + } + } else { + builder.field(PREFER_ILM.getPreferredName(), indexProperties.preferIlm()); + if (indexProperties.ilmPolicyName() != null) { + builder.field(ILM_POLICY_FIELD.getPreferredName(), indexProperties.ilmPolicyName()); + } + builder.field(INDEX_MODE.getPreferredName(), indexProperties.indexMode); + } } builder.endObject(); } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamFailureStore.java b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamFailureStore.java index f5ae5747e088c..cff655b7547fa 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamFailureStore.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamFailureStore.java @@ -29,6 +29,7 @@ * supports the following configurations only explicitly enabling or disabling the failure store */ public record DataStreamFailureStore(Boolean enabled) implements SimpleDiffable, ToXContentObject { + public static final String FAILURES_LIFECYCLE_API_CAPABILITY = "failures_lifecycle"; public static final String FAILURE_STORE = "failure_store"; public static final String ENABLED = "enabled"; From cd4e3faab3fef628347d4167e568fc0479f55b33 Mon Sep 17 00:00:00 2001 From: gmarouli Date: Fri, 11 Apr 2025 12:55:32 +0300 Subject: [PATCH 2/6] Add test case for when we display ilm --- .../datastreams/GetDataStreamActionTests.java | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/server/src/test/java/org/elasticsearch/action/datastreams/GetDataStreamActionTests.java b/server/src/test/java/org/elasticsearch/action/datastreams/GetDataStreamActionTests.java index a71d8ec57feea..908b1a632b9a4 100644 --- a/server/src/test/java/org/elasticsearch/action/datastreams/GetDataStreamActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/datastreams/GetDataStreamActionTests.java @@ -63,6 +63,45 @@ public void testDataStreamInfoToXContent() throws IOException { } } + @SuppressWarnings("unchecked") + public void testFailureStoreIndexManagedByIlmToXContent() throws IOException { + { + + Index failureIndex = new Index("my-fs", randomUUID()); + var failureIndexProperties = new GetDataStreamAction.Response.IndexProperties( + randomBoolean(), + "my-policy", + randomFrom(GetDataStreamAction.Response.ManagedBy.values()), + null + ); + var dataStream = DataStream.builder( + randomAlphaOfLength(50), + List.of(new Index(randomAlphaOfLength(10), randomAlphaOfLength(10))) + ) + .setGeneration(randomLongBetween(1, 1000)) + .setFailureIndices(DataStream.DataStreamIndices.failureIndicesBuilder(List.of(failureIndex)).build()) + .build(); + var dataStreamInfo = new GetDataStreamAction.Response.DataStreamInfo( + dataStream, + randomBoolean(), + randomFrom(ClusterHealthStatus.values()), + null, + null, + null, + Map.of(failureIndex, failureIndexProperties), + randomBoolean(), + null, + null + ); + Map failureStoreMap = (Map) getXContentMap(dataStreamInfo, null, null).get("failure_store"); + Map failureIndexMap = ((List>) failureStoreMap.get("indices")).get(0); + assertThat(failureIndexMap.get("index_name"), equalTo(failureIndex.getName())); + assertThat(failureIndexMap.get("managed_by"), equalTo(failureIndexProperties.managedBy().displayValue)); + assertThat(failureIndexMap.get("prefer_ilm"), equalTo(failureIndexProperties.preferIlm())); + assertThat(failureIndexMap.get("ilm_policy"), equalTo(failureIndexProperties.ilmPolicyName())); + } + } + /* * Calls toXContent on the given dataStreamInfo, and converts the response to a Map */ From 3fa43355142e213f208b2eb38896e75f96f87c61 Mon Sep 17 00:00:00 2001 From: gmarouli Date: Fri, 11 Apr 2025 14:10:28 +0300 Subject: [PATCH 3/6] Move test to a more appropriate place --- .../action/GetDataStreamsResponseTests.java | 90 ++++++++++++++++--- .../datastreams/GetDataStreamActionTests.java | 39 -------- 2 files changed, 78 insertions(+), 51 deletions(-) diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/GetDataStreamsResponseTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/GetDataStreamsResponseTests.java index f4ed40b2e0078..74c6fc0a27522 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/GetDataStreamsResponseTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/GetDataStreamsResponseTests.java @@ -31,6 +31,7 @@ import static org.elasticsearch.cluster.metadata.DataStream.getDefaultBackingIndexName; import static org.elasticsearch.cluster.metadata.DataStream.getDefaultFailureStoreName; import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; public class GetDataStreamsResponseTests extends ESTestCase { @@ -136,16 +137,16 @@ public void testResponseIlmAndDataStreamLifecycleRepresentation() throws Excepti if (DataStream.isFailureStoreFeatureFlagEnabled()) { var failureStore = (Map) dataStreamMap.get(DataStream.FAILURE_STORE_FIELD.getPreferredName()); - List failureStoresRepresentation = (List) failureStore.get(DataStream.INDICES_FIELD.getPreferredName()); - Map failureStoreRepresentation = (Map) failureStoresRepresentation.get(0); - assertThat(failureStoreRepresentation.get("index_name"), is(failureStoreIndex.getName())); - assertThat(failureStoreRepresentation.get(Response.DataStreamInfo.PREFER_ILM.getPreferredName()), is(false)); + List failureIndices = (List) failureStore.get(DataStream.INDICES_FIELD.getPreferredName()); + Map failureIndexRepresentation = (Map) failureIndices.get(0); + assertThat(failureIndexRepresentation.get("index_name"), is(failureStoreIndex.getName())); + assertThat(failureIndexRepresentation.get(Response.DataStreamInfo.PREFER_ILM.getPreferredName()), nullValue()); assertThat( - failureStoreRepresentation.get(Response.DataStreamInfo.ILM_POLICY_FIELD.getPreferredName()), + failureIndexRepresentation.get(Response.DataStreamInfo.ILM_POLICY_FIELD.getPreferredName()), is(nullValue()) ); assertThat( - failureStoreRepresentation.get(Response.DataStreamInfo.MANAGED_BY.getPreferredName()), + failureIndexRepresentation.get(Response.DataStreamInfo.MANAGED_BY.getPreferredName()), is(ManagedBy.LIFECYCLE.displayValue) ); } @@ -230,21 +231,86 @@ public void testResponseIlmAndDataStreamLifecycleRepresentation() throws Excepti if (DataStream.isFailureStoreFeatureFlagEnabled()) { var failureStore = (Map) dataStreamMap.get(DataStream.FAILURE_STORE_FIELD.getPreferredName()); - List failureStoresRepresentation = (List) failureStore.get(DataStream.INDICES_FIELD.getPreferredName()); - Map failureStoreRepresentation = (Map) failureStoresRepresentation.get(0); - assertThat(failureStoreRepresentation.get("index_name"), is(failureStoreIndex.getName())); - assertThat(failureStoreRepresentation.get(Response.DataStreamInfo.PREFER_ILM.getPreferredName()), is(false)); + List failureIndices = (List) failureStore.get(DataStream.INDICES_FIELD.getPreferredName()); + Map failureIndexRepresentation = (Map) failureIndices.get(0); + assertThat(failureIndexRepresentation.get("index_name"), is(failureStoreIndex.getName())); + assertThat(failureIndexRepresentation.get(Response.DataStreamInfo.PREFER_ILM.getPreferredName()), nullValue()); assertThat( - failureStoreRepresentation.get(Response.DataStreamInfo.ILM_POLICY_FIELD.getPreferredName()), + failureIndexRepresentation.get(Response.DataStreamInfo.ILM_POLICY_FIELD.getPreferredName()), is(nullValue()) ); assertThat( - failureStoreRepresentation.get(Response.DataStreamInfo.MANAGED_BY.getPreferredName()), + failureIndexRepresentation.get(Response.DataStreamInfo.MANAGED_BY.getPreferredName()), is(ManagedBy.UNMANAGED.displayValue) ); } } } + + { + // one failure index that have ILM policy + DataStream logs = DataStream.builder("logs", indices) + .setGeneration(3) + .setAllowCustomRouting(true) + .setIndexMode(IndexMode.STANDARD) + .setLifecycle(DataStreamLifecycle.DEFAULT_DATA_LIFECYCLE) + .setDataStreamOptions(DataStreamOptions.FAILURE_STORE_ENABLED) + .setFailureIndices(DataStream.DataStreamIndices.failureIndicesBuilder(failureStores).build()) + .build(); + + String ilmPolicyName = "rollover-30days"; + Map indexSettingsValues = Map.of( + firstGenerationIndex, + new Response.IndexProperties(true, ilmPolicyName, ManagedBy.ILM, null), + secondGenerationIndex, + new Response.IndexProperties(false, ilmPolicyName, ManagedBy.LIFECYCLE, null), + writeIndex, + new Response.IndexProperties(true, null, ManagedBy.LIFECYCLE, null), + failureStoreIndex, + new Response.IndexProperties(randomBoolean(), ilmPolicyName, ManagedBy.LIFECYCLE, null) + ); + + Response.DataStreamInfo dataStreamInfo = new Response.DataStreamInfo( + logs, + true, + ClusterHealthStatus.GREEN, + "index-template", + null, + null, + indexSettingsValues, + false, + null, + null + ); + Response response = new Response(List.of(dataStreamInfo)); + XContentBuilder contentBuilder = XContentFactory.jsonBuilder(); + response.toXContent(contentBuilder, ToXContent.EMPTY_PARAMS); + + BytesReference bytes = BytesReference.bytes(contentBuilder); + try (XContentParser parser = createParser(JsonXContent.jsonXContent, bytes)) { + Map map = parser.map(); + List dataStreams = (List) map.get(Response.DATA_STREAMS_FIELD.getPreferredName()); + assertThat(dataStreams.size(), is(1)); + Map dataStreamMap = (Map) dataStreams.get(0); + assertThat(dataStreamMap.get(DataStream.NAME_FIELD.getPreferredName()), is(dataStreamName)); + + if (DataStream.isFailureStoreFeatureFlagEnabled()) { + var failureStore = (Map) dataStreamMap.get(DataStream.FAILURE_STORE_FIELD.getPreferredName()); + List failureIndices = (List) failureStore.get(DataStream.INDICES_FIELD.getPreferredName()); + Map failureIndexRepresentation = (Map) failureIndices.get(0); + assertThat(failureIndexRepresentation.get("index_name"), is(failureStoreIndex.getName())); + assertThat(failureIndexRepresentation.get(Response.DataStreamInfo.PREFER_ILM.getPreferredName()), notNullValue()); + assertThat( + failureIndexRepresentation.get(Response.DataStreamInfo.ILM_POLICY_FIELD.getPreferredName()), + is(ilmPolicyName) + ); + assertThat( + failureIndexRepresentation.get(Response.DataStreamInfo.MANAGED_BY.getPreferredName()), + is(ManagedBy.LIFECYCLE.displayValue) + ); + } + } + } } public void testManagedByDisplayValuesDontAccidentalyChange() { diff --git a/server/src/test/java/org/elasticsearch/action/datastreams/GetDataStreamActionTests.java b/server/src/test/java/org/elasticsearch/action/datastreams/GetDataStreamActionTests.java index 908b1a632b9a4..a71d8ec57feea 100644 --- a/server/src/test/java/org/elasticsearch/action/datastreams/GetDataStreamActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/datastreams/GetDataStreamActionTests.java @@ -63,45 +63,6 @@ public void testDataStreamInfoToXContent() throws IOException { } } - @SuppressWarnings("unchecked") - public void testFailureStoreIndexManagedByIlmToXContent() throws IOException { - { - - Index failureIndex = new Index("my-fs", randomUUID()); - var failureIndexProperties = new GetDataStreamAction.Response.IndexProperties( - randomBoolean(), - "my-policy", - randomFrom(GetDataStreamAction.Response.ManagedBy.values()), - null - ); - var dataStream = DataStream.builder( - randomAlphaOfLength(50), - List.of(new Index(randomAlphaOfLength(10), randomAlphaOfLength(10))) - ) - .setGeneration(randomLongBetween(1, 1000)) - .setFailureIndices(DataStream.DataStreamIndices.failureIndicesBuilder(List.of(failureIndex)).build()) - .build(); - var dataStreamInfo = new GetDataStreamAction.Response.DataStreamInfo( - dataStream, - randomBoolean(), - randomFrom(ClusterHealthStatus.values()), - null, - null, - null, - Map.of(failureIndex, failureIndexProperties), - randomBoolean(), - null, - null - ); - Map failureStoreMap = (Map) getXContentMap(dataStreamInfo, null, null).get("failure_store"); - Map failureIndexMap = ((List>) failureStoreMap.get("indices")).get(0); - assertThat(failureIndexMap.get("index_name"), equalTo(failureIndex.getName())); - assertThat(failureIndexMap.get("managed_by"), equalTo(failureIndexProperties.managedBy().displayValue)); - assertThat(failureIndexMap.get("prefer_ilm"), equalTo(failureIndexProperties.preferIlm())); - assertThat(failureIndexMap.get("ilm_policy"), equalTo(failureIndexProperties.ilmPolicyName())); - } - } - /* * Calls toXContent on the given dataStreamInfo, and converts the response to a Map */ From 71eb3edd8005cc78f1813f88c6d567db8f5bd6db Mon Sep 17 00:00:00 2001 From: gmarouli Date: Tue, 15 Apr 2025 12:36:14 +0300 Subject: [PATCH 4/6] Add test case that covers failure store enabled by cluster setting --- .../data_stream/240_failure_store_info.yml | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/240_failure_store_info.yml b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/240_failure_store_info.yml index 3a4ef3f73f3da..292a2966381cf 100644 --- a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/240_failure_store_info.yml +++ b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/240_failure_store_info.yml @@ -44,6 +44,7 @@ setup: type: date count: type: long + lifecycle: {} data_stream: { } - do: @@ -191,3 +192,48 @@ teardown: - match: { data_streams.0.failure_store.indices.0.index_name: '/\.fs-fs-data-stream-(\d{4}\.\d{2}\.\d{2}-)?000002/' } - is_false: data_streams.0.failure_store.indices.0.prefer_ilm - match: { data_streams.0.failure_store.indices.0.managed_by: 'Unmanaged' } + +--- +"Get failure store info from cluster setting enabled failure store": + - do: + indices.create_data_stream: + name: fs-default-data-stream + - is_true: acknowledged + + - do: + indices.get_data_stream: + name: "fs-default-data-stream" + - match: { data_streams.0.name: fs-default-data-stream } + - match: { data_streams.0.timestamp_field.name: '@timestamp' } + - match: { data_streams.0.generation: 1 } + - length: { data_streams.0.indices: 1 } + - match: { data_streams.0.indices.0.index_name: '/\.ds-fs-default-data-stream-(\d{4}\.\d{2}\.\d{2}-)?000001/' } + - match: { data_streams.0.template: 'my-template2' } + - match: { data_streams.0.failure_store.enabled: true } + - match: { data_streams.0.failure_store.lifecycle.enabled: true } + - match: { data_streams.0.failure_store.indices: [] } + + # Initialize failure store + - do: + index: + index: fs-default-data-stream + refresh: true + body: + '@timestamp': '2020-12-12' + count: 'invalid value' + + - do: + indices.get_data_stream: + name: "fs-default-data-stream" + - match: { data_streams.0.name: fs-default-data-stream } + - match: { data_streams.0.timestamp_field.name: '@timestamp' } + - match: { data_streams.0.generation: 2 } + - length: { data_streams.0.indices: 1 } + - match: { data_streams.0.indices.0.index_name: '/\.ds-fs-default-data-stream-(\d{4}\.\d{2}\.\d{2}-)?000001/' } + - match: { data_streams.0.template: 'my-template2' } + - match: { data_streams.0.failure_store.enabled: true } + - match: { data_streams.0.failure_store.lifecycle.enabled: true } + - length: { data_streams.0.failure_store.indices: 1 } + - match: { data_streams.0.failure_store.indices.0.index_name: '/\.fs-fs-default-data-stream-(\d{4}\.\d{2}\.\d{2}-)?000002/' } + - is_false: data_streams.0.failure_store.indices.0.prefer_ilm + - match: { data_streams.0.failure_store.indices.0.managed_by: 'Data stream lifecycle' } From 76fc076fde34ba74b1fb3d763a478f6b02eab123 Mon Sep 17 00:00:00 2001 From: gmarouli Date: Wed, 23 Apr 2025 13:46:14 +0300 Subject: [PATCH 5/6] Address review comments --- .../datastreams/rest/RestGetDataStreamsAction.java | 4 ++-- .../rest-api-spec/test/data_stream/240_failure_store_info.yml | 2 +- .../cluster/metadata/DataStreamFailureStore.java | 1 - 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestGetDataStreamsAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestGetDataStreamsAction.java index 22f2ee77a7a7f..941f39d8545e3 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestGetDataStreamsAction.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestGetDataStreamsAction.java @@ -11,7 +11,6 @@ import org.elasticsearch.action.datastreams.GetDataStreamAction; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.client.internal.node.NodeClient; -import org.elasticsearch.cluster.metadata.DataStreamFailureStore; import org.elasticsearch.cluster.metadata.DataStreamLifecycle; import org.elasticsearch.common.Strings; import org.elasticsearch.common.util.set.Sets; @@ -46,9 +45,10 @@ public class RestGetDataStreamsAction extends BaseRestHandler { ) ) ); + public static final String FAILURES_LIFECYCLE_API_CAPABILITY = "failure_store.lifecycle"; private static final Set CAPABILITIES = Set.of( DataStreamLifecycle.EFFECTIVE_RETENTION_REST_API_CAPABILITY, - DataStreamFailureStore.FAILURES_LIFECYCLE_API_CAPABILITY + FAILURES_LIFECYCLE_API_CAPABILITY ); @Override diff --git a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/240_failure_store_info.yml b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/240_failure_store_info.yml index 292a2966381cf..6619ced833fa9 100644 --- a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/240_failure_store_info.yml +++ b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/240_failure_store_info.yml @@ -5,7 +5,7 @@ setup: capabilities: - method: GET path: /_data_stream/{target} - capabilities: [ 'failures_lifecycle' ] + capabilities: [ 'failure_store.lifecycle' ] - do: allowed_warnings: - "index template [my-template1] has index patterns [fs-data-stream] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template1] will take precedence during new index creation" diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamFailureStore.java b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamFailureStore.java index cff655b7547fa..f5ae5747e088c 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamFailureStore.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamFailureStore.java @@ -29,7 +29,6 @@ * supports the following configurations only explicitly enabling or disabling the failure store */ public record DataStreamFailureStore(Boolean enabled) implements SimpleDiffable, ToXContentObject { - public static final String FAILURES_LIFECYCLE_API_CAPABILITY = "failures_lifecycle"; public static final String FAILURE_STORE = "failure_store"; public static final String ENABLED = "enabled"; From 5e304cd9d668d759f22a7f7ab35c817a8f266bf6 Mon Sep 17 00:00:00 2001 From: gmarouli Date: Wed, 23 Apr 2025 14:44:50 +0300 Subject: [PATCH 6/6] Fix test --- .../datastreams/action/GetDataStreamsResponseTests.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/GetDataStreamsResponseTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/GetDataStreamsResponseTests.java index eb4a0af67ee47..8f6605cefae9f 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/GetDataStreamsResponseTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/GetDataStreamsResponseTests.java @@ -228,7 +228,7 @@ public void testResponseIlmAndDataStreamLifecycleRepresentation() throws Excepti List failureStoresRepresentation = (List) failureStore.get(DataStream.INDICES_FIELD.getPreferredName()); Map failureStoreRepresentation = (Map) failureStoresRepresentation.get(0); assertThat(failureStoreRepresentation.get("index_name"), is(failureStoreIndex.getName())); - assertThat(failureStoreRepresentation.get(Response.DataStreamInfo.PREFER_ILM.getPreferredName()), is(false)); + assertThat(failureStoreRepresentation.get(Response.DataStreamInfo.PREFER_ILM.getPreferredName()), nullValue()); assertThat(failureStoreRepresentation.get(Response.DataStreamInfo.ILM_POLICY_FIELD.getPreferredName()), is(nullValue())); assertThat( failureStoreRepresentation.get(Response.DataStreamInfo.MANAGED_BY.getPreferredName()),