Expose global retention settings via data stream lifecycle API (elastic#112210)

In this PR we expose the global retention via the `GET
_data_stream/{target}/_lifecycle` API.

Since the global retention is a main feature of the data stream
lifecycle we chose to expose it by default.

```
GET /_data_stream/my-data-stream/_lifecycle
{
 "global_retention": {
      "default_retention": "7d",
      "max_retention": "365d"
  }, 
  "data_streams": [...]
}
```

Author: gmarouli

docs/changelog/112210.yaml

@@ -0,0 +1,5 @@
+pr: 112210
+summary: Expose global retention settings via data stream lifecycle API
+area: Data streams
+type: enhancement
+issues: []

docs/reference/data-streams/lifecycle/apis/get-lifecycle.asciidoc

@@ -67,8 +67,18 @@ Name of the data stream.
 =====
 `data_retention`::
 (Optional, string)
+If defined, it represents the retention requested by the data stream owner for this data stream.
+
+`effective_retention`::
+(Optional, string)
 If defined, every document added to this data stream will be stored at least for this time frame. Any time after this
-duration the document could be deleted. When undefined, every document in this data stream will be stored indefinitely.
+duration the document could be deleted. When empty, every document in this data stream will be stored indefinitely.
+duration the document could be deleted. When empty, every document in this data stream will be stored indefinitely. The
+effective retention is calculated as described in the <<effective-retention-calculation, tutorial>>.
+
+`retention_determined_by`::
+(Optional, string)
+The source of the retention, it can be one of three values, `data_stream_configuration`, `default_retention` or `max_retention`.
 
 `rollover`::
 (Optional, object)
@@ -78,6 +88,21 @@ when the query param `include_defaults` is set to `true`. The contents of this f
 =====
 ====
 
+`global_retention`::
+(object)
+Contains the global max and default retention. When no global retention is configured, this will be an empty object.
++
+.Properties of `global_retention`
+[%collapsible%open]
+====
+`max_retention`::
+(Optional, string)
+The effective retention of data streams managed by the data stream lifecycle cannot exceed this value.
+`default_retention`::
+(Optional, string)
+This will be the effective retention of data streams managed by the data stream lifecycle that do not specify `data_retention`.
+====
+
 [[data-streams-get-lifecycle-example]]
 ==== {api-examples-title}
 
@@ -142,6 +167,7 @@ The response will look like the following:
         "retention_determined_by": "data_stream_configuration"
       }
     }
-  ]
+  ],
+  "global_retention": {}
 }
 --------------------------------------------------

docs/reference/data-streams/lifecycle/tutorial-manage-data-stream-retention.asciidoc

@@ -189,19 +189,29 @@ We see that it will remain the same with what the user configured:
 [source,console-result]
 ----
 {
+  "global_retention" : {
+    "max_retention" : "90d",                                   <1>
+    "default_retention" : "7d"                                 <2>
+  },
   "data_streams": [
     {
       "name": "my-data-stream",
       "lifecycle": {
         "enabled": true,
-        "data_retention": "30d",
-        "effective_retention": "30d",
-        "retention_determined_by": "data_stream_configuration"
+        "data_retention": "30d",                                <3>
+        "effective_retention": "30d",                           <4>
+        "retention_determined_by": "data_stream_configuration"  <5>
       }
     }
   ]
 }
 ----
+<1> The maximum retention configured in the cluster.
+<2> The default retention configured in the cluster.
+<3> The requested retention for this data stream.
+<4> The retention that is applied by the data stream lifecycle on this data stream.
+<5> The configuration that determined the effective retention. In this case it's the `data_configuration` because
+it is less than the `max_retention`.
 
 [discrete]
 [[effective-retention-application]]

docs/reference/data-streams/lifecycle/tutorial-manage-new-data-stream.asciidoc

@@ -99,7 +99,8 @@ The result will look like this:
         "retention_determined_by": "data_stream_configuration"
       }
     }
-  ]
+  ],
+  "global_retention": {}
 }
 --------------------------------------------------
 <1> The name of your data stream.

modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/rest/RestGetDataStreamLifecycleAction.java

@@ -59,6 +59,6 @@ public boolean allowSystemIndexAccessByDefault() {
 
     @Override
     public Set<String> supportedCapabilities() {
-        return Set.of(DataStreamLifecycle.EFFECTIVE_RETENTION_REST_API_CAPABILITY);
+        return Set.of(DataStreamLifecycle.EFFECTIVE_RETENTION_REST_API_CAPABILITY, "data_stream_global_retention");
     }
 }

modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/lifecycle/20_basic.yml

@@ -1,8 +1,8 @@
 setup:
-  - skip:
-      features: allowed_warnings
-      cluster_features: ["gte_v8.11.0"]
-      reason: "Data stream lifecycles only supported in 8.11+"
+  - requires:
+      cluster_features: [ "gte_v8.11.0" ]
+      reason: "Data stream lifecycle was released as tech preview in 8.11"
+      test_runner_features: allowed_warnings
   - do:
       allowed_warnings:
         - "index template [my-lifecycle] has index patterns [data-stream-with-lifecycle] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-lifecycle] will take precedence during new index creation"
@@ -25,6 +25,7 @@ setup:
         body:
           index_patterns: [simple-data-stream1]
           template:
+            lifecycle: {}
             mappings:
               properties:
                 '@timestamp':
@@ -39,27 +40,93 @@ setup:
         name: simple-data-stream1
 
 ---
-"Get data stream lifecycle":
+teardown:
+  - requires:
+      reason: "Global retention was exposed in 8.16+"
+      test_runner_features: [ capabilities ]
+      capabilities:
+        - method: GET
+          path: /_data_stream/{index}/_lifecycle
+          capabilities: [ 'data_stream_global_retention' ]
+  - do:
+      cluster.put_settings:
+        body:
+          persistent:
+            data_streams.lifecycle.retention.max: null
+            data_streams.lifecycle.retention.default: null
 
+---
+"Get data stream lifecycle":
+  - requires:
+      reason: "Global retention was exposed in 8.16+"
+      test_runner_features: [ capabilities ]
+      capabilities:
+        - method: GET
+          path: /_data_stream/{index}/_lifecycle
+          capabilities: [ 'data_stream_global_retention' ]
   - do:
       indices.get_data_lifecycle:
         name: "data-stream-with-lifecycle"
   - length: { data_streams: 1}
   - match: { data_streams.0.name: data-stream-with-lifecycle }
   - match: { data_streams.0.lifecycle.data_retention: '10d' }
+  - match: { data_streams.0.lifecycle.effective_retention: '10d' }
   - match: { data_streams.0.lifecycle.enabled: true}
+  - match: { global_retention: {} }
 
 ---
-"Get data stream with default lifecycle":
-  - skip:
-      awaits_fix: https://github.com/elastic/elasticsearch/pull/100187
+"Get data stream with default lifecycle configuration":
+  - requires:
+      reason: "Global retention was exposed in 8.16+"
+      test_runner_features: [ capabilities ]
+      capabilities:
+        - method: GET
+          path: /_data_stream/{index}/_lifecycle
+          capabilities: [ 'data_stream_global_retention' ]
+  - do:
+      indices.get_data_lifecycle:
+        name: "simple-data-stream1"
+  - length: { data_streams: 1}
+  - match: { data_streams.0.name: simple-data-stream1 }
+  - match: { data_streams.0.lifecycle.enabled: true}
+  - is_false: data_streams.0.lifecycle.effective_retention
+  - match: { global_retention: {} }
 
+---
+"Get data stream with global retention":
+  - requires:
+      reason: "Global retention was exposed in 8.16+"
+      test_runner_features: [ capabilities ]
+      capabilities:
+        - method: GET
+          path: /_data_stream/{index}/_lifecycle
+          capabilities: [ 'data_stream_global_retention' ]
+  - do:
+      cluster.put_settings:
+        body:
+          persistent:
+            data_streams.lifecycle.retention.default: "7d"
+            data_streams.lifecycle.retention.max: "9d"
   - do:
       indices.get_data_lifecycle:
         name: "simple-data-stream1"
   - length: { data_streams: 1}
   - match: { data_streams.0.name: simple-data-stream1 }
   - match: { data_streams.0.lifecycle.enabled: true}
+  - match: { data_streams.0.lifecycle.effective_retention: '7d'}
+  - match: { global_retention.default_retention: '7d' }
+  - match: { global_retention.max_retention: '9d' }
+
+  - do:
+      indices.get_data_lifecycle:
+        name: "data-stream-with-lifecycle"
+  - length: { data_streams: 1 }
+  - match: { data_streams.0.name: data-stream-with-lifecycle }
+  - match: { data_streams.0.lifecycle.data_retention: '10d' }
+  - match: { data_streams.0.lifecycle.effective_retention: '9d' }
+  - match: { data_streams.0.lifecycle.enabled: true }
+  - match: { global_retention.default_retention: '7d' }
+  - match: { global_retention.max_retention: '9d' }
 
 ---
 "Put data stream lifecycle":

modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/lifecycle/30_not_found.yml

@@ -1,8 +1,8 @@
 setup:
-  - skip:
-      features: allowed_warnings
-      cluster_features: ["gte_v8.11.0"]
-      reason: "Data stream lifecycle was GA in 8.11"
+  - requires:
+      cluster_features: [ "gte_v8.11.0" ]
+      reason: "Data stream lifecycle was released as tech preview in 8.11"
+      test_runner_features: allowed_warnings
   - do:
       allowed_warnings:
         - "index template [my-lifecycle] has index patterns [my-data-stream-1] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-lifecycle] will take precedence during new index creation"

server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/GetDataStreamLifecycleAction.java

@@ -254,6 +254,16 @@ public void writeTo(StreamOutput out) throws IOException {
         public Iterator<ToXContent> toXContentChunked(ToXContent.Params outerParams) {
             return Iterators.concat(Iterators.single((builder, params) -> {
                 builder.startObject();
+                builder.startObject("global_retention");
+                if (globalRetention != null) {
+                    if (globalRetention.maxRetention() != null) {
+                        builder.field("max_retention", globalRetention.maxRetention().getStringRep());
+                    }
+                    if (globalRetention.defaultRetention() != null) {
+                        builder.field("default_retention", globalRetention.defaultRetention().getStringRep());
+                    }
+                }
+                builder.endObject();
                 builder.startArray(DATA_STREAMS_FIELD.getPreferredName());
                 return builder;
             }),

server/src/test/java/org/elasticsearch/action/datastreams/lifecycle/GetDataStreamLifecycleActionTests.java

@@ -12,6 +12,7 @@
 import org.elasticsearch.cluster.metadata.DataStreamGlobalRetention;
 import org.elasticsearch.cluster.metadata.DataStreamLifecycle;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.test.ESTestCase;
@@ -20,26 +21,88 @@
 import org.elasticsearch.xcontent.XContentType;
 
 import java.io.IOException;
+import java.util.List;
 import java.util.Map;
 
-import static org.elasticsearch.rest.RestRequest.PATH_RESTRICTED;
+import static org.elasticsearch.xcontent.ToXContent.EMPTY_PARAMS;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.nullValue;
 
 public class GetDataStreamLifecycleActionTests extends ESTestCase {
 
+    @SuppressWarnings("unchecked")
+    public void testDefaultLifecycleResponseToXContent() throws Exception {
+        boolean isInternalDataStream = randomBoolean();
+        GetDataStreamLifecycleAction.Response.DataStreamLifecycle dataStreamLifecycle = createDataStreamLifecycle(
+            DataStreamLifecycle.DEFAULT,
+            isInternalDataStream
+        );
+        GetDataStreamLifecycleAction.Response response = new GetDataStreamLifecycleAction.Response(List.of(dataStreamLifecycle));
+        try (XContentBuilder builder = XContentBuilder.builder(XContentType.JSON.xContent())) {
+            builder.humanReadable(true);
+            response.toXContentChunked(ToXContent.EMPTY_PARAMS).forEachRemaining(xcontent -> {
+                try {
+                    xcontent.toXContent(builder, EMPTY_PARAMS);
+                } catch (IOException e) {
+                    logger.error(e.getMessage(), e);
+                    fail(e.getMessage());
+                }
+            });
+            Map<String, Object> resultMap = XContentHelper.convertToMap(BytesReference.bytes(builder), false, builder.contentType()).v2();
+            assertThat(resultMap.get("global_retention"), equalTo(Map.of()));
+            assertThat(resultMap.containsKey("data_streams"), equalTo(true));
+            List<Map<String, Object>> dataStreams = (List<Map<String, Object>>) resultMap.get("data_streams");
+            Map<String, Object> firstDataStream = dataStreams.get(0);
+            assertThat(firstDataStream.containsKey("lifecycle"), equalTo(true));
+            Map<String, Object> lifecycleResult = (Map<String, Object>) firstDataStream.get("lifecycle");
+            assertThat(lifecycleResult.get("enabled"), equalTo(true));
+            assertThat(lifecycleResult.get("data_retention"), nullValue());
+            assertThat(lifecycleResult.get("effective_retention"), nullValue());
+            assertThat(lifecycleResult.get("retention_determined_by"), nullValue());
+        }
+    }
+
+    @SuppressWarnings("unchecked")
+    public void testGlobalRetentionToXContent() {
+        TimeValue globalDefaultRetention = TimeValue.timeValueDays(10);
+        TimeValue globalMaxRetention = TimeValue.timeValueDays(50);
+        DataStreamGlobalRetention globalRetention = new DataStreamGlobalRetention(globalDefaultRetention, globalMaxRetention);
+        GetDataStreamLifecycleAction.Response response = new GetDataStreamLifecycleAction.Response(List.of(), null, globalRetention);
+        try (XContentBuilder builder = XContentBuilder.builder(XContentType.JSON.xContent())) {
+            builder.humanReadable(true);
+            response.toXContentChunked(ToXContent.EMPTY_PARAMS).forEachRemaining(xcontent -> {
+                try {
+                    xcontent.toXContent(builder, EMPTY_PARAMS);
+                } catch (IOException e) {
+                    logger.error(e.getMessage(), e);
+                    fail(e.getMessage());
+                }
+            });
+            Map<String, Object> resultMap = XContentHelper.convertToMap(BytesReference.bytes(builder), false, builder.contentType()).v2();
+            assertThat(resultMap.containsKey("global_retention"), equalTo(true));
+            Map<String, String> globalRetentionMap = (Map<String, String>) resultMap.get("global_retention");
+            assertThat(globalRetentionMap.get("max_retention"), equalTo(globalMaxRetention.getStringRep()));
+            assertThat(globalRetentionMap.get("default_retention"), equalTo(globalDefaultRetention.getStringRep()));
+            assertThat(resultMap.containsKey("data_streams"), equalTo(true));
+        } catch (Exception e) {
+            fail(e);
+        }
+    }
+
     @SuppressWarnings("unchecked")
     public void testDataStreamLifecycleToXContent() throws Exception {
         TimeValue configuredRetention = TimeValue.timeValueDays(100);
         TimeValue globalDefaultRetention = TimeValue.timeValueDays(10);
         TimeValue globalMaxRetention = TimeValue.timeValueDays(50);
+        DataStreamGlobalRetention globalRetention = new DataStreamGlobalRetention(globalDefaultRetention, globalMaxRetention);
         DataStreamLifecycle lifecycle = new DataStreamLifecycle(new DataStreamLifecycle.Retention(configuredRetention), null, null);
         {
             boolean isInternalDataStream = true;
             GetDataStreamLifecycleAction.Response.DataStreamLifecycle explainIndexDataStreamLifecycle = createDataStreamLifecycle(
                 lifecycle,
                 isInternalDataStream
             );
-            Map<String, Object> resultMap = getXContentMap(explainIndexDataStreamLifecycle, globalDefaultRetention, globalMaxRetention);
+            Map<String, Object> resultMap = getXContentMap(explainIndexDataStreamLifecycle, globalRetention);
             Map<String, Object> lifecycleResult = (Map<String, Object>) resultMap.get("lifecycle");
             assertThat(lifecycleResult.get("data_retention"), equalTo(configuredRetention.getStringRep()));
             assertThat(lifecycleResult.get("effective_retention"), equalTo(configuredRetention.getStringRep()));
@@ -51,7 +114,7 @@ public void testDataStreamLifecycleToXContent() throws Exception {
                 lifecycle,
                 isInternalDataStream
             );
-            Map<String, Object> resultMap = getXContentMap(explainIndexDataStreamLifecycle, globalDefaultRetention, globalMaxRetention);
+            Map<String, Object> resultMap = getXContentMap(explainIndexDataStreamLifecycle, globalRetention);
             Map<String, Object> lifecycleResult = (Map<String, Object>) resultMap.get("lifecycle");
             assertThat(lifecycleResult.get("data_retention"), equalTo(configuredRetention.getStringRep()));
             assertThat(lifecycleResult.get("effective_retention"), equalTo(globalMaxRetention.getStringRep()));
@@ -71,14 +134,11 @@ private GetDataStreamLifecycleAction.Response.DataStreamLifecycle createDataStre
      */
     private Map<String, Object> getXContentMap(
         GetDataStreamLifecycleAction.Response.DataStreamLifecycle dataStreamLifecycle,
-        TimeValue globalDefaultRetention,
-        TimeValue globalMaxRetention
+        DataStreamGlobalRetention globalRetention
     ) throws IOException {
         try (XContentBuilder builder = XContentBuilder.builder(XContentType.JSON.xContent())) {
-            ToXContent.Params params = new ToXContent.MapParams(Map.of(PATH_RESTRICTED, "serverless"));
             RolloverConfiguration rolloverConfiguration = null;
-            DataStreamGlobalRetention globalRetention = new DataStreamGlobalRetention(globalDefaultRetention, globalMaxRetention);
-            dataStreamLifecycle.toXContent(builder, params, rolloverConfiguration, globalRetention);
+            dataStreamLifecycle.toXContent(builder, ToXContent.EMPTY_PARAMS, rolloverConfiguration, globalRetention);
             String serialized = Strings.toString(builder);
             return XContentHelper.convertToMap(XContentType.JSON.xContent(), serialized, randomBoolean());
         }