[Failure Store] Expose failure store lifecycle information via the GET data stream API

modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestGetDataStreamsAction.java

@@ -45,6 +45,11 @@ public class RestGetDataStreamsAction extends BaseRestHandler {
             )
         )
     );
+    public static final String FAILURES_LIFECYCLE_API_CAPABILITY = "failure_store.lifecycle";
+    private static final Set<String> CAPABILITIES = Set.of(
+        DataStreamLifecycle.EFFECTIVE_RETENTION_REST_API_CAPABILITY,
+        FAILURES_LIFECYCLE_API_CAPABILITY
+    );
 
     @Override
     public String getName() {
@@ -79,7 +84,7 @@ public boolean allowSystemIndexAccessByDefault() {
 
     @Override
     public Set<String> supportedCapabilities() {
-        return Set.of(DataStreamLifecycle.EFFECTIVE_RETENTION_REST_API_CAPABILITY);
+        return CAPABILITIES;
     }
 
     @Override

modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/GetDataStreamsResponseTests.java

@@ -31,6 +31,7 @@
 import static org.elasticsearch.cluster.metadata.DataStream.getDefaultBackingIndexName;
 import static org.elasticsearch.cluster.metadata.DataStream.getDefaultFailureStoreName;
 import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.notNullValue;
 import static org.hamcrest.Matchers.nullValue;
 
 public class GetDataStreamsResponseTests extends ESTestCase {
@@ -135,13 +136,13 @@ public void testResponseIlmAndDataStreamLifecycleRepresentation() throws Excepti
                 );
 
                 var failureStore = (Map<String, Object>) dataStreamMap.get(DataStream.FAILURE_STORE_FIELD.getPreferredName());
-                List<Object> failureStoresRepresentation = (List<Object>) failureStore.get(DataStream.INDICES_FIELD.getPreferredName());
-                Map<String, Object> failureStoreRepresentation = (Map<String, Object>) failureStoresRepresentation.get(0);
-                assertThat(failureStoreRepresentation.get("index_name"), is(failureStoreIndex.getName()));
-                assertThat(failureStoreRepresentation.get(Response.DataStreamInfo.PREFER_ILM.getPreferredName()), is(false));
-                assertThat(failureStoreRepresentation.get(Response.DataStreamInfo.ILM_POLICY_FIELD.getPreferredName()), is(nullValue()));
+                List<Object> failureIndices = (List<Object>) failureStore.get(DataStream.INDICES_FIELD.getPreferredName());
+                Map<String, Object> failureIndexRepresentation = (Map<String, Object>) failureIndices.get(0);
+                assertThat(failureIndexRepresentation.get("index_name"), is(failureStoreIndex.getName()));
+                assertThat(failureIndexRepresentation.get(Response.DataStreamInfo.PREFER_ILM.getPreferredName()), nullValue());
+                assertThat(failureIndexRepresentation.get(Response.DataStreamInfo.ILM_POLICY_FIELD.getPreferredName()), is(nullValue()));
                 assertThat(
-                    failureStoreRepresentation.get(Response.DataStreamInfo.MANAGED_BY.getPreferredName()),
+                    failureIndexRepresentation.get(Response.DataStreamInfo.MANAGED_BY.getPreferredName()),
                     is(ManagedBy.LIFECYCLE.displayValue)
                 );
             }
@@ -227,14 +228,74 @@ public void testResponseIlmAndDataStreamLifecycleRepresentation() throws Excepti
                 List<Object> failureStoresRepresentation = (List<Object>) failureStore.get(DataStream.INDICES_FIELD.getPreferredName());
                 Map<String, Object> failureStoreRepresentation = (Map<String, Object>) failureStoresRepresentation.get(0);
                 assertThat(failureStoreRepresentation.get("index_name"), is(failureStoreIndex.getName()));
-                assertThat(failureStoreRepresentation.get(Response.DataStreamInfo.PREFER_ILM.getPreferredName()), is(false));
+                assertThat(failureStoreRepresentation.get(Response.DataStreamInfo.PREFER_ILM.getPreferredName()), nullValue());
                 assertThat(failureStoreRepresentation.get(Response.DataStreamInfo.ILM_POLICY_FIELD.getPreferredName()), is(nullValue()));
                 assertThat(
                     failureStoreRepresentation.get(Response.DataStreamInfo.MANAGED_BY.getPreferredName()),
                     is(ManagedBy.UNMANAGED.displayValue)
                 );
             }
         }
+
+        {
+            // one failure index that have ILM policy
+            DataStream logs = DataStream.builder("logs", indices)
+                .setGeneration(3)
+                .setAllowCustomRouting(true)
+                .setIndexMode(IndexMode.STANDARD)
+                .setLifecycle(DataStreamLifecycle.DEFAULT_DATA_LIFECYCLE)
+                .setDataStreamOptions(DataStreamOptions.FAILURE_STORE_ENABLED)
+                .setFailureIndices(DataStream.DataStreamIndices.failureIndicesBuilder(failureStores).build())
+                .build();
+
+            String ilmPolicyName = "rollover-30days";
+            Map<Index, Response.IndexProperties> indexSettingsValues = Map.of(
+                firstGenerationIndex,
+                new Response.IndexProperties(true, ilmPolicyName, ManagedBy.ILM, null),
+                secondGenerationIndex,
+                new Response.IndexProperties(false, ilmPolicyName, ManagedBy.LIFECYCLE, null),
+                writeIndex,
+                new Response.IndexProperties(true, null, ManagedBy.LIFECYCLE, null),
+                failureStoreIndex,
+                new Response.IndexProperties(randomBoolean(), ilmPolicyName, ManagedBy.LIFECYCLE, null)
+            );
+
+            Response.DataStreamInfo dataStreamInfo = new Response.DataStreamInfo(
+                logs,
+                true,
+                ClusterHealthStatus.GREEN,
+                "index-template",
+                null,
+                null,
+                indexSettingsValues,
+                false,
+                null,
+                null
+            );
+            Response response = new Response(List.of(dataStreamInfo));
+            XContentBuilder contentBuilder = XContentFactory.jsonBuilder();
+            response.toXContent(contentBuilder, ToXContent.EMPTY_PARAMS);
+
+            BytesReference bytes = BytesReference.bytes(contentBuilder);
+            try (XContentParser parser = createParser(JsonXContent.jsonXContent, bytes)) {
+                Map<String, Object> map = parser.map();
+                List<Object> dataStreams = (List<Object>) map.get(Response.DATA_STREAMS_FIELD.getPreferredName());
+                assertThat(dataStreams.size(), is(1));
+                Map<String, Object> dataStreamMap = (Map<String, Object>) dataStreams.get(0);
+                assertThat(dataStreamMap.get(DataStream.NAME_FIELD.getPreferredName()), is(dataStreamName));
+
+                var failureStore = (Map<String, Object>) dataStreamMap.get(DataStream.FAILURE_STORE_FIELD.getPreferredName());
+                List<Object> failureIndices = (List<Object>) failureStore.get(DataStream.INDICES_FIELD.getPreferredName());
+                Map<String, Object> failureIndexRepresentation = (Map<String, Object>) failureIndices.get(0);
+                assertThat(failureIndexRepresentation.get("index_name"), is(failureStoreIndex.getName()));
+                assertThat(failureIndexRepresentation.get(Response.DataStreamInfo.PREFER_ILM.getPreferredName()), notNullValue());
+                assertThat(failureIndexRepresentation.get(Response.DataStreamInfo.ILM_POLICY_FIELD.getPreferredName()), is(ilmPolicyName));
+                assertThat(
+                    failureIndexRepresentation.get(Response.DataStreamInfo.MANAGED_BY.getPreferredName()),
+                    is(ManagedBy.LIFECYCLE.displayValue)
+                );
+            }
+        }
     }
 
     public void testManagedByDisplayValuesDontAccidentalyChange() {

modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/240_failure_store_info.yml

@@ -0,0 +1,239 @@
+setup:
+  - requires:
+      test_runner_features: [ capabilities, allowed_warnings ]
+      reason: "Exposing failures lifecycle config in templates was added in 9.1+"
+      capabilities:
+        - method: GET
+          path: /_data_stream/{target}
+          capabilities: [ 'failure_store.lifecycle' ]
+  - do:
+      allowed_warnings:
+        - "index template [my-template1] has index patterns [fs-data-stream] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template1] will take precedence during new index creation"
+      indices.put_index_template:
+        name: my-template1
+        body:
+          index_patterns: [fs-data-stream]
+          template:
+            settings:
+              index.number_of_replicas: 1
+            mappings:
+              properties:
+                '@timestamp':
+                  type: date
+                count:
+                  type: long
+            lifecycle: {}
+            data_stream_options:
+              failure_store:
+                enabled: true
+          data_stream: {}
+
+  - do:
+      allowed_warnings:
+        - "index template [my-template2] has index patterns [fs-default-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template2] will take precedence during new index creation"
+      indices.put_index_template:
+        name: my-template2
+        body:
+          index_patterns: [ fs-default-* ]
+          template:
+            settings:
+              index.number_of_replicas: 1
+            mappings:
+              properties:
+                '@timestamp':
+                  type: date
+                count:
+                  type: long
+            lifecycle: {}
+          data_stream: { }
+
+  - do:
+      allowed_warnings:
+        - "index template [my-template3] has index patterns [no-fs-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template3] will take precedence during new index creation"
+      indices.put_index_template:
+        name: my-template3
+        body:
+          index_patterns: [ no-fs-* ]
+          template:
+            settings:
+              index.number_of_replicas: 1
+            mappings:
+              properties:
+                '@timestamp':
+                  type: date
+                count:
+                  type: long
+          data_stream: { }
+
+  - do:
+      cluster.put_settings:
+        body:
+          persistent:
+            data_streams.failure_store.enabled: 'fs-default*'
+
+---
+teardown:
+  - do:
+      indices.delete_data_stream:
+        name: fs-data-stream
+        ignore: 404
+
+  - do:
+      indices.delete_index_template:
+        name: fs-default-data-stream
+        ignore: 404
+
+  - do:
+      indices.delete_index_template:
+        name: no-fs-data-stream
+        ignore: 404
+
+---
+"Get failure store info from explicitly enabled failure store":
+  - do:
+      indices.create_data_stream:
+        name: fs-data-stream
+  - is_true: acknowledged
+
+  - do:
+      indices.get_data_stream:
+        name: "fs-data-stream"
+  - match: { data_streams.0.name: fs-data-stream }
+  - match: { data_streams.0.timestamp_field.name: '@timestamp' }
+  - match: { data_streams.0.generation: 1 }
+  - length: { data_streams.0.indices: 1 }
+  - match: { data_streams.0.indices.0.index_name: '/\.ds-fs-data-stream-(\d{4}\.\d{2}\.\d{2}-)?000001/' }
+  - match: { data_streams.0.template: 'my-template1' }
+  - match: { data_streams.0.failure_store.enabled: true }
+  - match: { data_streams.0.failure_store.lifecycle.enabled: true}
+  - match: { data_streams.0.failure_store.indices: [] }
+
+  # Initialize failure store
+  - do:
+      index:
+        index: fs-data-stream
+        refresh: true
+        body:
+          '@timestamp': '2020-12-12'
+          count: 'invalid value'
+
+  - do:
+      indices.get_data_stream:
+        name: "fs-data-stream"
+  - match: { data_streams.0.name: fs-data-stream }
+  - match: { data_streams.0.timestamp_field.name: '@timestamp' }
+  - match: { data_streams.0.generation: 2 }
+  - length: { data_streams.0.indices: 1 }
+  - match: { data_streams.0.indices.0.index_name: '/\.ds-fs-data-stream-(\d{4}\.\d{2}\.\d{2}-)?000001/' }
+  - match: { data_streams.0.template: 'my-template1' }
+  - match: { data_streams.0.failure_store.enabled: true }
+  - match: { data_streams.0.failure_store.lifecycle.enabled: true }
+  - length: { data_streams.0.failure_store.indices: 1 }
+  - match: { data_streams.0.failure_store.indices.0.index_name: '/\.fs-fs-data-stream-(\d{4}\.\d{2}\.\d{2}-)?000002/' }
+  - is_false: data_streams.0.failure_store.indices.0.prefer_ilm
+  - match: { data_streams.0.failure_store.indices.0.managed_by: 'Data stream lifecycle' }
+
+---
+"Get failure store info from disabled failure store":
+  - do:
+      indices.create_data_stream:
+        name: no-fs-data-stream
+  - is_true: acknowledged
+
+  - do:
+      indices.get_data_stream:
+        name: "no-fs-data-stream"
+  - match: { data_streams.0.name: no-fs-data-stream }
+  - match: { data_streams.0.timestamp_field.name: '@timestamp' }
+  - match: { data_streams.0.generation: 1 }
+  - length: { data_streams.0.indices: 1 }
+  - match: { data_streams.0.indices.0.index_name: '/\.ds-no-fs-data-stream-(\d{4}\.\d{2}\.\d{2}-)?000001/' }
+  - match: { data_streams.0.template: 'my-template3' }
+  - match: { data_streams.0.failure_store.enabled: false }
+  - is_false: data_streams.0.failure_store.lifecycle
+  - match: { data_streams.0.failure_store.indices: [] }
+
+---
+"Get failure store info from explicitly enabled failure store and disabled lifecycle":
+  - do:
+      indices.create_data_stream:
+        name: fs-data-stream
+  - is_true: acknowledged
+
+  - do:
+      indices.put_data_lifecycle:
+        name: "fs-data-stream"
+        body:
+          enabled: false
+
+  - is_true: acknowledged
+
+  # Initialize failure store
+  - do:
+      index:
+        index: fs-data-stream
+        refresh: true
+        body:
+          '@timestamp': '2020-12-12'
+          count: 'invalid value'
+
+  - do:
+      indices.get_data_stream:
+        name: "fs-data-stream"
+  - match: { data_streams.0.name: fs-data-stream }
+  - match: { data_streams.0.timestamp_field.name: '@timestamp' }
+  - match: { data_streams.0.generation: 2 }
+  - length: { data_streams.0.indices: 1 }
+  - match: { data_streams.0.indices.0.index_name: '/\.ds-fs-data-stream-(\d{4}\.\d{2}\.\d{2}-)?000001/' }
+  - match: { data_streams.0.template: 'my-template1' }
+  - match: { data_streams.0.failure_store.enabled: true }
+  - match: { data_streams.0.failure_store.lifecycle.enabled: false }
+  - length: { data_streams.0.failure_store.indices: 1 }
+  - match: { data_streams.0.failure_store.indices.0.index_name: '/\.fs-fs-data-stream-(\d{4}\.\d{2}\.\d{2}-)?000002/' }
+  - is_false: data_streams.0.failure_store.indices.0.prefer_ilm
+  - match: { data_streams.0.failure_store.indices.0.managed_by: 'Unmanaged' }
+
+---
+"Get failure store info from cluster setting enabled failure store":
+  - do:
+      indices.create_data_stream:
+        name: fs-default-data-stream
+  - is_true: acknowledged
+
+  - do:
+      indices.get_data_stream:
+        name: "fs-default-data-stream"
+  - match: { data_streams.0.name: fs-default-data-stream }
+  - match: { data_streams.0.timestamp_field.name: '@timestamp' }
+  - match: { data_streams.0.generation: 1 }
+  - length: { data_streams.0.indices: 1 }
+  - match: { data_streams.0.indices.0.index_name: '/\.ds-fs-default-data-stream-(\d{4}\.\d{2}\.\d{2}-)?000001/' }
+  - match: { data_streams.0.template: 'my-template2' }
+  - match: { data_streams.0.failure_store.enabled: true }
+  - match: { data_streams.0.failure_store.lifecycle.enabled: true }
+  - match: { data_streams.0.failure_store.indices: [] }
+
+  # Initialize failure store
+  - do:
+      index:
+        index: fs-default-data-stream
+        refresh: true
+        body:
+          '@timestamp': '2020-12-12'
+          count: 'invalid value'
+
+  - do:
+      indices.get_data_stream:
+        name: "fs-default-data-stream"
+  - match: { data_streams.0.name: fs-default-data-stream }
+  - match: { data_streams.0.timestamp_field.name: '@timestamp' }
+  - match: { data_streams.0.generation: 2 }
+  - length: { data_streams.0.indices: 1 }
+  - match: { data_streams.0.indices.0.index_name: '/\.ds-fs-default-data-stream-(\d{4}\.\d{2}\.\d{2}-)?000001/' }
+  - match: { data_streams.0.template: 'my-template2' }
+  - match: { data_streams.0.failure_store.enabled: true }
+  - match: { data_streams.0.failure_store.lifecycle.enabled: true }
+  - length: { data_streams.0.failure_store.indices: 1 }
+  - match: { data_streams.0.failure_store.indices.0.index_name: '/\.fs-fs-default-data-stream-(\d{4}\.\d{2}\.\d{2}-)?000002/' }
+  - is_false: data_streams.0.failure_store.indices.0.prefer_ilm
+  - match: { data_streams.0.failure_store.indices.0.managed_by: 'Data stream lifecycle' }