diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestGetDataStreamsAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestGetDataStreamsAction.java index d1e04eb4072b6..941f39d8545e3 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestGetDataStreamsAction.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestGetDataStreamsAction.java @@ -45,6 +45,11 @@ public class RestGetDataStreamsAction extends BaseRestHandler { ) ) ); + public static final String FAILURES_LIFECYCLE_API_CAPABILITY = "failure_store.lifecycle"; + private static final Set CAPABILITIES = Set.of( + DataStreamLifecycle.EFFECTIVE_RETENTION_REST_API_CAPABILITY, + FAILURES_LIFECYCLE_API_CAPABILITY + ); @Override public String getName() { @@ -79,7 +84,7 @@ public boolean allowSystemIndexAccessByDefault() { @Override public Set supportedCapabilities() { - return Set.of(DataStreamLifecycle.EFFECTIVE_RETENTION_REST_API_CAPABILITY); + return CAPABILITIES; } @Override diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/GetDataStreamsResponseTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/GetDataStreamsResponseTests.java index d0b2db97e3395..8f6605cefae9f 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/GetDataStreamsResponseTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/GetDataStreamsResponseTests.java @@ -31,6 +31,7 @@ import static org.elasticsearch.cluster.metadata.DataStream.getDefaultBackingIndexName; import static org.elasticsearch.cluster.metadata.DataStream.getDefaultFailureStoreName; import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; public class GetDataStreamsResponseTests extends ESTestCase { @@ -135,13 +136,13 @@ public void testResponseIlmAndDataStreamLifecycleRepresentation() throws Excepti ); var failureStore = (Map) dataStreamMap.get(DataStream.FAILURE_STORE_FIELD.getPreferredName()); - List failureStoresRepresentation = (List) failureStore.get(DataStream.INDICES_FIELD.getPreferredName()); - Map failureStoreRepresentation = (Map) failureStoresRepresentation.get(0); - assertThat(failureStoreRepresentation.get("index_name"), is(failureStoreIndex.getName())); - assertThat(failureStoreRepresentation.get(Response.DataStreamInfo.PREFER_ILM.getPreferredName()), is(false)); - assertThat(failureStoreRepresentation.get(Response.DataStreamInfo.ILM_POLICY_FIELD.getPreferredName()), is(nullValue())); + List failureIndices = (List) failureStore.get(DataStream.INDICES_FIELD.getPreferredName()); + Map failureIndexRepresentation = (Map) failureIndices.get(0); + assertThat(failureIndexRepresentation.get("index_name"), is(failureStoreIndex.getName())); + assertThat(failureIndexRepresentation.get(Response.DataStreamInfo.PREFER_ILM.getPreferredName()), nullValue()); + assertThat(failureIndexRepresentation.get(Response.DataStreamInfo.ILM_POLICY_FIELD.getPreferredName()), is(nullValue())); assertThat( - failureStoreRepresentation.get(Response.DataStreamInfo.MANAGED_BY.getPreferredName()), + failureIndexRepresentation.get(Response.DataStreamInfo.MANAGED_BY.getPreferredName()), is(ManagedBy.LIFECYCLE.displayValue) ); } @@ -227,7 +228,7 @@ public void testResponseIlmAndDataStreamLifecycleRepresentation() throws Excepti List failureStoresRepresentation = (List) failureStore.get(DataStream.INDICES_FIELD.getPreferredName()); Map failureStoreRepresentation = (Map) failureStoresRepresentation.get(0); assertThat(failureStoreRepresentation.get("index_name"), is(failureStoreIndex.getName())); - assertThat(failureStoreRepresentation.get(Response.DataStreamInfo.PREFER_ILM.getPreferredName()), is(false)); + assertThat(failureStoreRepresentation.get(Response.DataStreamInfo.PREFER_ILM.getPreferredName()), nullValue()); assertThat(failureStoreRepresentation.get(Response.DataStreamInfo.ILM_POLICY_FIELD.getPreferredName()), is(nullValue())); assertThat( failureStoreRepresentation.get(Response.DataStreamInfo.MANAGED_BY.getPreferredName()), @@ -235,6 +236,66 @@ public void testResponseIlmAndDataStreamLifecycleRepresentation() throws Excepti ); } } + + { + // one failure index that have ILM policy + DataStream logs = DataStream.builder("logs", indices) + .setGeneration(3) + .setAllowCustomRouting(true) + .setIndexMode(IndexMode.STANDARD) + .setLifecycle(DataStreamLifecycle.DEFAULT_DATA_LIFECYCLE) + .setDataStreamOptions(DataStreamOptions.FAILURE_STORE_ENABLED) + .setFailureIndices(DataStream.DataStreamIndices.failureIndicesBuilder(failureStores).build()) + .build(); + + String ilmPolicyName = "rollover-30days"; + Map indexSettingsValues = Map.of( + firstGenerationIndex, + new Response.IndexProperties(true, ilmPolicyName, ManagedBy.ILM, null), + secondGenerationIndex, + new Response.IndexProperties(false, ilmPolicyName, ManagedBy.LIFECYCLE, null), + writeIndex, + new Response.IndexProperties(true, null, ManagedBy.LIFECYCLE, null), + failureStoreIndex, + new Response.IndexProperties(randomBoolean(), ilmPolicyName, ManagedBy.LIFECYCLE, null) + ); + + Response.DataStreamInfo dataStreamInfo = new Response.DataStreamInfo( + logs, + true, + ClusterHealthStatus.GREEN, + "index-template", + null, + null, + indexSettingsValues, + false, + null, + null + ); + Response response = new Response(List.of(dataStreamInfo)); + XContentBuilder contentBuilder = XContentFactory.jsonBuilder(); + response.toXContent(contentBuilder, ToXContent.EMPTY_PARAMS); + + BytesReference bytes = BytesReference.bytes(contentBuilder); + try (XContentParser parser = createParser(JsonXContent.jsonXContent, bytes)) { + Map map = parser.map(); + List dataStreams = (List) map.get(Response.DATA_STREAMS_FIELD.getPreferredName()); + assertThat(dataStreams.size(), is(1)); + Map dataStreamMap = (Map) dataStreams.get(0); + assertThat(dataStreamMap.get(DataStream.NAME_FIELD.getPreferredName()), is(dataStreamName)); + + var failureStore = (Map) dataStreamMap.get(DataStream.FAILURE_STORE_FIELD.getPreferredName()); + List failureIndices = (List) failureStore.get(DataStream.INDICES_FIELD.getPreferredName()); + Map failureIndexRepresentation = (Map) failureIndices.get(0); + assertThat(failureIndexRepresentation.get("index_name"), is(failureStoreIndex.getName())); + assertThat(failureIndexRepresentation.get(Response.DataStreamInfo.PREFER_ILM.getPreferredName()), notNullValue()); + assertThat(failureIndexRepresentation.get(Response.DataStreamInfo.ILM_POLICY_FIELD.getPreferredName()), is(ilmPolicyName)); + assertThat( + failureIndexRepresentation.get(Response.DataStreamInfo.MANAGED_BY.getPreferredName()), + is(ManagedBy.LIFECYCLE.displayValue) + ); + } + } } public void testManagedByDisplayValuesDontAccidentalyChange() { diff --git a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/240_failure_store_info.yml b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/240_failure_store_info.yml new file mode 100644 index 0000000000000..6619ced833fa9 --- /dev/null +++ b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/240_failure_store_info.yml @@ -0,0 +1,239 @@ +setup: + - requires: + test_runner_features: [ capabilities, allowed_warnings ] + reason: "Exposing failures lifecycle config in templates was added in 9.1+" + capabilities: + - method: GET + path: /_data_stream/{target} + capabilities: [ 'failure_store.lifecycle' ] + - do: + allowed_warnings: + - "index template [my-template1] has index patterns [fs-data-stream] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template1] will take precedence during new index creation" + indices.put_index_template: + name: my-template1 + body: + index_patterns: [fs-data-stream] + template: + settings: + index.number_of_replicas: 1 + mappings: + properties: + '@timestamp': + type: date + count: + type: long + lifecycle: {} + data_stream_options: + failure_store: + enabled: true + data_stream: {} + + - do: + allowed_warnings: + - "index template [my-template2] has index patterns [fs-default-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template2] will take precedence during new index creation" + indices.put_index_template: + name: my-template2 + body: + index_patterns: [ fs-default-* ] + template: + settings: + index.number_of_replicas: 1 + mappings: + properties: + '@timestamp': + type: date + count: + type: long + lifecycle: {} + data_stream: { } + + - do: + allowed_warnings: + - "index template [my-template3] has index patterns [no-fs-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template3] will take precedence during new index creation" + indices.put_index_template: + name: my-template3 + body: + index_patterns: [ no-fs-* ] + template: + settings: + index.number_of_replicas: 1 + mappings: + properties: + '@timestamp': + type: date + count: + type: long + data_stream: { } + + - do: + cluster.put_settings: + body: + persistent: + data_streams.failure_store.enabled: 'fs-default*' + +--- +teardown: + - do: + indices.delete_data_stream: + name: fs-data-stream + ignore: 404 + + - do: + indices.delete_index_template: + name: fs-default-data-stream + ignore: 404 + + - do: + indices.delete_index_template: + name: no-fs-data-stream + ignore: 404 + +--- +"Get failure store info from explicitly enabled failure store": + - do: + indices.create_data_stream: + name: fs-data-stream + - is_true: acknowledged + + - do: + indices.get_data_stream: + name: "fs-data-stream" + - match: { data_streams.0.name: fs-data-stream } + - match: { data_streams.0.timestamp_field.name: '@timestamp' } + - match: { data_streams.0.generation: 1 } + - length: { data_streams.0.indices: 1 } + - match: { data_streams.0.indices.0.index_name: '/\.ds-fs-data-stream-(\d{4}\.\d{2}\.\d{2}-)?000001/' } + - match: { data_streams.0.template: 'my-template1' } + - match: { data_streams.0.failure_store.enabled: true } + - match: { data_streams.0.failure_store.lifecycle.enabled: true} + - match: { data_streams.0.failure_store.indices: [] } + + # Initialize failure store + - do: + index: + index: fs-data-stream + refresh: true + body: + '@timestamp': '2020-12-12' + count: 'invalid value' + + - do: + indices.get_data_stream: + name: "fs-data-stream" + - match: { data_streams.0.name: fs-data-stream } + - match: { data_streams.0.timestamp_field.name: '@timestamp' } + - match: { data_streams.0.generation: 2 } + - length: { data_streams.0.indices: 1 } + - match: { data_streams.0.indices.0.index_name: '/\.ds-fs-data-stream-(\d{4}\.\d{2}\.\d{2}-)?000001/' } + - match: { data_streams.0.template: 'my-template1' } + - match: { data_streams.0.failure_store.enabled: true } + - match: { data_streams.0.failure_store.lifecycle.enabled: true } + - length: { data_streams.0.failure_store.indices: 1 } + - match: { data_streams.0.failure_store.indices.0.index_name: '/\.fs-fs-data-stream-(\d{4}\.\d{2}\.\d{2}-)?000002/' } + - is_false: data_streams.0.failure_store.indices.0.prefer_ilm + - match: { data_streams.0.failure_store.indices.0.managed_by: 'Data stream lifecycle' } + +--- +"Get failure store info from disabled failure store": + - do: + indices.create_data_stream: + name: no-fs-data-stream + - is_true: acknowledged + + - do: + indices.get_data_stream: + name: "no-fs-data-stream" + - match: { data_streams.0.name: no-fs-data-stream } + - match: { data_streams.0.timestamp_field.name: '@timestamp' } + - match: { data_streams.0.generation: 1 } + - length: { data_streams.0.indices: 1 } + - match: { data_streams.0.indices.0.index_name: '/\.ds-no-fs-data-stream-(\d{4}\.\d{2}\.\d{2}-)?000001/' } + - match: { data_streams.0.template: 'my-template3' } + - match: { data_streams.0.failure_store.enabled: false } + - is_false: data_streams.0.failure_store.lifecycle + - match: { data_streams.0.failure_store.indices: [] } + +--- +"Get failure store info from explicitly enabled failure store and disabled lifecycle": + - do: + indices.create_data_stream: + name: fs-data-stream + - is_true: acknowledged + + - do: + indices.put_data_lifecycle: + name: "fs-data-stream" + body: + enabled: false + + - is_true: acknowledged + + # Initialize failure store + - do: + index: + index: fs-data-stream + refresh: true + body: + '@timestamp': '2020-12-12' + count: 'invalid value' + + - do: + indices.get_data_stream: + name: "fs-data-stream" + - match: { data_streams.0.name: fs-data-stream } + - match: { data_streams.0.timestamp_field.name: '@timestamp' } + - match: { data_streams.0.generation: 2 } + - length: { data_streams.0.indices: 1 } + - match: { data_streams.0.indices.0.index_name: '/\.ds-fs-data-stream-(\d{4}\.\d{2}\.\d{2}-)?000001/' } + - match: { data_streams.0.template: 'my-template1' } + - match: { data_streams.0.failure_store.enabled: true } + - match: { data_streams.0.failure_store.lifecycle.enabled: false } + - length: { data_streams.0.failure_store.indices: 1 } + - match: { data_streams.0.failure_store.indices.0.index_name: '/\.fs-fs-data-stream-(\d{4}\.\d{2}\.\d{2}-)?000002/' } + - is_false: data_streams.0.failure_store.indices.0.prefer_ilm + - match: { data_streams.0.failure_store.indices.0.managed_by: 'Unmanaged' } + +--- +"Get failure store info from cluster setting enabled failure store": + - do: + indices.create_data_stream: + name: fs-default-data-stream + - is_true: acknowledged + + - do: + indices.get_data_stream: + name: "fs-default-data-stream" + - match: { data_streams.0.name: fs-default-data-stream } + - match: { data_streams.0.timestamp_field.name: '@timestamp' } + - match: { data_streams.0.generation: 1 } + - length: { data_streams.0.indices: 1 } + - match: { data_streams.0.indices.0.index_name: '/\.ds-fs-default-data-stream-(\d{4}\.\d{2}\.\d{2}-)?000001/' } + - match: { data_streams.0.template: 'my-template2' } + - match: { data_streams.0.failure_store.enabled: true } + - match: { data_streams.0.failure_store.lifecycle.enabled: true } + - match: { data_streams.0.failure_store.indices: [] } + + # Initialize failure store + - do: + index: + index: fs-default-data-stream + refresh: true + body: + '@timestamp': '2020-12-12' + count: 'invalid value' + + - do: + indices.get_data_stream: + name: "fs-default-data-stream" + - match: { data_streams.0.name: fs-default-data-stream } + - match: { data_streams.0.timestamp_field.name: '@timestamp' } + - match: { data_streams.0.generation: 2 } + - length: { data_streams.0.indices: 1 } + - match: { data_streams.0.indices.0.index_name: '/\.ds-fs-default-data-stream-(\d{4}\.\d{2}\.\d{2}-)?000001/' } + - match: { data_streams.0.template: 'my-template2' } + - match: { data_streams.0.failure_store.enabled: true } + - match: { data_streams.0.failure_store.lifecycle.enabled: true } + - length: { data_streams.0.failure_store.indices: 1 } + - match: { data_streams.0.failure_store.indices.0.index_name: '/\.fs-fs-default-data-stream-(\d{4}\.\d{2}\.\d{2}-)?000002/' } + - is_false: data_streams.0.failure_store.indices.0.prefer_ilm + - match: { data_streams.0.failure_store.indices.0.managed_by: 'Data stream lifecycle' } diff --git a/server/src/main/java/org/elasticsearch/action/datastreams/GetDataStreamAction.java b/server/src/main/java/org/elasticsearch/action/datastreams/GetDataStreamAction.java index 8d4fe44ece9a5..9f59c5865b484 100644 --- a/server/src/main/java/org/elasticsearch/action/datastreams/GetDataStreamAction.java +++ b/server/src/main/java/org/elasticsearch/action/datastreams/GetDataStreamAction.java @@ -371,7 +371,7 @@ public XContentBuilder toXContent( .field(DataStream.NAME_FIELD.getPreferredName(), DataStream.TIMESTAMP_FIELD_NAME) .endObject(); - indicesToXContent(builder, dataStream.getIndices()); + indicesToXContent(builder, dataStream.getIndices(), false); builder.field(DataStream.GENERATION_FIELD.getPreferredName(), dataStream.getGeneration()); if (dataStream.getMetadata() != null) { builder.field(DataStream.METADATA_FIELD.getPreferredName(), dataStream.getMetadata()); @@ -416,17 +416,24 @@ public XContentBuilder toXContent( builder.endArray(); builder.endObject(); } + builder.startObject(DataStream.FAILURE_STORE_FIELD.getPreferredName()); builder.field(FAILURE_STORE_ENABLED.getPreferredName(), failureStoreEffectivelyEnabled); builder.field(DataStream.ROLLOVER_ON_WRITE_FIELD.getPreferredName(), dataStream.getFailureComponent().isRolloverOnWrite()); - indicesToXContent(builder, dataStream.getFailureIndices()); + indicesToXContent(builder, dataStream.getFailureIndices(), true); addAutoShardingEvent(builder, params, dataStream.getFailureComponent().getAutoShardingEvent()); + DataStreamLifecycle failuresLifecycle = dataStream.getFailuresLifecycle(); + if (failuresLifecycle != null) { + builder.field(LIFECYCLE_FIELD.getPreferredName()); + failuresLifecycle.toXContent(builder, params, rolloverConfiguration, globalRetention, dataStream.isInternal()); + } builder.endObject(); builder.endObject(); return builder; } - private XContentBuilder indicesToXContent(XContentBuilder builder, List indices) throws IOException { + private XContentBuilder indicesToXContent(XContentBuilder builder, List indices, boolean failureIndices) + throws IOException { builder.field(DataStream.INDICES_FIELD.getPreferredName()); builder.startArray(); for (Index index : indices) { @@ -434,12 +441,22 @@ private XContentBuilder indicesToXContent(XContentBuilder builder, List i index.toXContentFragment(builder); IndexProperties indexProperties = indexSettingsValues.get(index); if (indexProperties != null) { - builder.field(PREFER_ILM.getPreferredName(), indexProperties.preferIlm()); - if (indexProperties.ilmPolicyName() != null) { - builder.field(ILM_POLICY_FIELD.getPreferredName(), indexProperties.ilmPolicyName()); - } builder.field(MANAGED_BY.getPreferredName(), indexProperties.managedBy.displayValue); - builder.field(INDEX_MODE.getPreferredName(), indexProperties.indexMode); + // Failure indices have more limitation than backing indices, + // so we hide some index properties that are less relevant + if (failureIndices) { + // We only display ILM info, if this index has an ILM policy + if (indexProperties.ilmPolicyName() != null) { + builder.field(PREFER_ILM.getPreferredName(), indexProperties.preferIlm()); + builder.field(ILM_POLICY_FIELD.getPreferredName(), indexProperties.ilmPolicyName()); + } + } else { + builder.field(PREFER_ILM.getPreferredName(), indexProperties.preferIlm()); + if (indexProperties.ilmPolicyName() != null) { + builder.field(ILM_POLICY_FIELD.getPreferredName(), indexProperties.ilmPolicyName()); + } + builder.field(INDEX_MODE.getPreferredName(), indexProperties.indexMode); + } } builder.endObject(); }