diff --git a/docs/changelog/112210.yaml b/docs/changelog/112210.yaml new file mode 100644 index 0000000000000..6483b8b01315c --- /dev/null +++ b/docs/changelog/112210.yaml @@ -0,0 +1,5 @@ +pr: 112210 +summary: Expose global retention settings via data stream lifecycle API +area: Data streams +type: enhancement +issues: [] diff --git a/docs/reference/data-streams/lifecycle/apis/get-lifecycle.asciidoc b/docs/reference/data-streams/lifecycle/apis/get-lifecycle.asciidoc index 6bac1c7f7cc75..6323fac1eac2f 100644 --- a/docs/reference/data-streams/lifecycle/apis/get-lifecycle.asciidoc +++ b/docs/reference/data-streams/lifecycle/apis/get-lifecycle.asciidoc @@ -67,8 +67,18 @@ Name of the data stream. ===== `data_retention`:: (Optional, string) +If defined, it represents the retention requested by the data stream owner for this data stream. + +`effective_retention`:: +(Optional, string) If defined, every document added to this data stream will be stored at least for this time frame. Any time after this -duration the document could be deleted. When undefined, every document in this data stream will be stored indefinitely. +duration the document could be deleted. When empty, every document in this data stream will be stored indefinitely. +duration the document could be deleted. When empty, every document in this data stream will be stored indefinitely. The +effective retention is calculated as described in the <>. + +`retention_determined_by`:: +(Optional, string) +The source of the retention, it can be one of three values, `data_stream_configuration`, `default_retention` or `max_retention`. `rollover`:: (Optional, object) @@ -78,6 +88,21 @@ when the query param `include_defaults` is set to `true`. The contents of this f ===== ==== +`global_retention`:: +(object) +Contains the global max and default retention. When no global retention is configured, this will be an empty object. ++ +.Properties of `global_retention` +[%collapsible%open] +==== +`max_retention`:: +(Optional, string) +The effective retention of data streams managed by the data stream lifecycle cannot exceed this value. +`default_retention`:: +(Optional, string) +This will be the effective retention of data streams managed by the data stream lifecycle that do not specify `data_retention`. +==== + [[data-streams-get-lifecycle-example]] ==== {api-examples-title} @@ -142,6 +167,7 @@ The response will look like the following: "retention_determined_by": "data_stream_configuration" } } - ] + ], + "global_retention": {} } -------------------------------------------------- diff --git a/docs/reference/data-streams/lifecycle/tutorial-manage-data-stream-retention.asciidoc b/docs/reference/data-streams/lifecycle/tutorial-manage-data-stream-retention.asciidoc index 1b2996c62e2df..a7f0379a45167 100644 --- a/docs/reference/data-streams/lifecycle/tutorial-manage-data-stream-retention.asciidoc +++ b/docs/reference/data-streams/lifecycle/tutorial-manage-data-stream-retention.asciidoc @@ -189,19 +189,29 @@ We see that it will remain the same with what the user configured: [source,console-result] ---- { + "global_retention" : { + "max_retention" : "90d", <1> + "default_retention" : "7d" <2> + }, "data_streams": [ { "name": "my-data-stream", "lifecycle": { "enabled": true, - "data_retention": "30d", - "effective_retention": "30d", - "retention_determined_by": "data_stream_configuration" + "data_retention": "30d", <3> + "effective_retention": "30d", <4> + "retention_determined_by": "data_stream_configuration" <5> } } ] } ---- +<1> The maximum retention configured in the cluster. +<2> The default retention configured in the cluster. +<3> The requested retention for this data stream. +<4> The retention that is applied by the data stream lifecycle on this data stream. +<5> The configuration that determined the effective retention. In this case it's the `data_configuration` because +it is less than the `max_retention`. [discrete] [[effective-retention-application]] diff --git a/docs/reference/data-streams/lifecycle/tutorial-manage-new-data-stream.asciidoc b/docs/reference/data-streams/lifecycle/tutorial-manage-new-data-stream.asciidoc index 01d51cdde3167..173b7a75dd28e 100644 --- a/docs/reference/data-streams/lifecycle/tutorial-manage-new-data-stream.asciidoc +++ b/docs/reference/data-streams/lifecycle/tutorial-manage-new-data-stream.asciidoc @@ -99,7 +99,8 @@ The result will look like this: "retention_determined_by": "data_stream_configuration" } } - ] + ], + "global_retention": {} } -------------------------------------------------- <1> The name of your data stream. diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/rest/RestGetDataStreamLifecycleAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/rest/RestGetDataStreamLifecycleAction.java index 00f9d4da88301..ee325ed9655be 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/rest/RestGetDataStreamLifecycleAction.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/rest/RestGetDataStreamLifecycleAction.java @@ -59,6 +59,6 @@ public boolean allowSystemIndexAccessByDefault() { @Override public Set supportedCapabilities() { - return Set.of(DataStreamLifecycle.EFFECTIVE_RETENTION_REST_API_CAPABILITY); + return Set.of(DataStreamLifecycle.EFFECTIVE_RETENTION_REST_API_CAPABILITY, "data_stream_global_retention"); } } diff --git a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/lifecycle/20_basic.yml b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/lifecycle/20_basic.yml index 1cf44312ae7d5..4bf6ccfbfa7ce 100644 --- a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/lifecycle/20_basic.yml +++ b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/lifecycle/20_basic.yml @@ -1,8 +1,8 @@ setup: - - skip: - features: allowed_warnings - cluster_features: ["gte_v8.11.0"] - reason: "Data stream lifecycles only supported in 8.11+" + - requires: + cluster_features: [ "gte_v8.11.0" ] + reason: "Data stream lifecycle was released as tech preview in 8.11" + test_runner_features: allowed_warnings - do: allowed_warnings: - "index template [my-lifecycle] has index patterns [data-stream-with-lifecycle] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-lifecycle] will take precedence during new index creation" @@ -25,6 +25,7 @@ setup: body: index_patterns: [simple-data-stream1] template: + lifecycle: {} mappings: properties: '@timestamp': @@ -39,27 +40,93 @@ setup: name: simple-data-stream1 --- -"Get data stream lifecycle": +teardown: + - requires: + reason: "Global retention was exposed in 8.16+" + test_runner_features: [ capabilities ] + capabilities: + - method: GET + path: /_data_stream/{index}/_lifecycle + capabilities: [ 'data_stream_global_retention' ] + - do: + cluster.put_settings: + body: + persistent: + data_streams.lifecycle.retention.max: null + data_streams.lifecycle.retention.default: null +--- +"Get data stream lifecycle": + - requires: + reason: "Global retention was exposed in 8.16+" + test_runner_features: [ capabilities ] + capabilities: + - method: GET + path: /_data_stream/{index}/_lifecycle + capabilities: [ 'data_stream_global_retention' ] - do: indices.get_data_lifecycle: name: "data-stream-with-lifecycle" - length: { data_streams: 1} - match: { data_streams.0.name: data-stream-with-lifecycle } - match: { data_streams.0.lifecycle.data_retention: '10d' } + - match: { data_streams.0.lifecycle.effective_retention: '10d' } - match: { data_streams.0.lifecycle.enabled: true} + - match: { global_retention: {} } --- -"Get data stream with default lifecycle": - - skip: - awaits_fix: https://github.com/elastic/elasticsearch/pull/100187 +"Get data stream with default lifecycle configuration": + - requires: + reason: "Global retention was exposed in 8.16+" + test_runner_features: [ capabilities ] + capabilities: + - method: GET + path: /_data_stream/{index}/_lifecycle + capabilities: [ 'data_stream_global_retention' ] + - do: + indices.get_data_lifecycle: + name: "simple-data-stream1" + - length: { data_streams: 1} + - match: { data_streams.0.name: simple-data-stream1 } + - match: { data_streams.0.lifecycle.enabled: true} + - is_false: data_streams.0.lifecycle.effective_retention + - match: { global_retention: {} } +--- +"Get data stream with global retention": + - requires: + reason: "Global retention was exposed in 8.16+" + test_runner_features: [ capabilities ] + capabilities: + - method: GET + path: /_data_stream/{index}/_lifecycle + capabilities: [ 'data_stream_global_retention' ] + - do: + cluster.put_settings: + body: + persistent: + data_streams.lifecycle.retention.default: "7d" + data_streams.lifecycle.retention.max: "9d" - do: indices.get_data_lifecycle: name: "simple-data-stream1" - length: { data_streams: 1} - match: { data_streams.0.name: simple-data-stream1 } - match: { data_streams.0.lifecycle.enabled: true} + - match: { data_streams.0.lifecycle.effective_retention: '7d'} + - match: { global_retention.default_retention: '7d' } + - match: { global_retention.max_retention: '9d' } + + - do: + indices.get_data_lifecycle: + name: "data-stream-with-lifecycle" + - length: { data_streams: 1 } + - match: { data_streams.0.name: data-stream-with-lifecycle } + - match: { data_streams.0.lifecycle.data_retention: '10d' } + - match: { data_streams.0.lifecycle.effective_retention: '9d' } + - match: { data_streams.0.lifecycle.enabled: true } + - match: { global_retention.default_retention: '7d' } + - match: { global_retention.max_retention: '9d' } --- "Put data stream lifecycle": diff --git a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/lifecycle/30_not_found.yml b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/lifecycle/30_not_found.yml index 24d0a5649a619..a3252496ef592 100644 --- a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/lifecycle/30_not_found.yml +++ b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/lifecycle/30_not_found.yml @@ -1,8 +1,8 @@ setup: - - skip: - features: allowed_warnings - cluster_features: ["gte_v8.11.0"] - reason: "Data stream lifecycle was GA in 8.11" + - requires: + cluster_features: [ "gte_v8.11.0" ] + reason: "Data stream lifecycle was released as tech preview in 8.11" + test_runner_features: allowed_warnings - do: allowed_warnings: - "index template [my-lifecycle] has index patterns [my-data-stream-1] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-lifecycle] will take precedence during new index creation" diff --git a/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/GetDataStreamLifecycleAction.java b/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/GetDataStreamLifecycleAction.java index 52af1341692eb..2ae6e544b3f53 100644 --- a/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/GetDataStreamLifecycleAction.java +++ b/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/GetDataStreamLifecycleAction.java @@ -254,6 +254,16 @@ public void writeTo(StreamOutput out) throws IOException { public Iterator toXContentChunked(ToXContent.Params outerParams) { return Iterators.concat(Iterators.single((builder, params) -> { builder.startObject(); + builder.startObject("global_retention"); + if (globalRetention != null) { + if (globalRetention.maxRetention() != null) { + builder.field("max_retention", globalRetention.maxRetention().getStringRep()); + } + if (globalRetention.defaultRetention() != null) { + builder.field("default_retention", globalRetention.defaultRetention().getStringRep()); + } + } + builder.endObject(); builder.startArray(DATA_STREAMS_FIELD.getPreferredName()); return builder; }), diff --git a/server/src/test/java/org/elasticsearch/action/datastreams/lifecycle/GetDataStreamLifecycleActionTests.java b/server/src/test/java/org/elasticsearch/action/datastreams/lifecycle/GetDataStreamLifecycleActionTests.java index 920e844055f6c..0edf5fd8b4ce7 100644 --- a/server/src/test/java/org/elasticsearch/action/datastreams/lifecycle/GetDataStreamLifecycleActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/datastreams/lifecycle/GetDataStreamLifecycleActionTests.java @@ -12,6 +12,7 @@ import org.elasticsearch.cluster.metadata.DataStreamGlobalRetention; import org.elasticsearch.cluster.metadata.DataStreamLifecycle; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.core.TimeValue; import org.elasticsearch.test.ESTestCase; @@ -20,18 +21,80 @@ import org.elasticsearch.xcontent.XContentType; import java.io.IOException; +import java.util.List; import java.util.Map; -import static org.elasticsearch.rest.RestRequest.PATH_RESTRICTED; +import static org.elasticsearch.xcontent.ToXContent.EMPTY_PARAMS; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.nullValue; public class GetDataStreamLifecycleActionTests extends ESTestCase { + @SuppressWarnings("unchecked") + public void testDefaultLifecycleResponseToXContent() throws Exception { + boolean isInternalDataStream = randomBoolean(); + GetDataStreamLifecycleAction.Response.DataStreamLifecycle dataStreamLifecycle = createDataStreamLifecycle( + DataStreamLifecycle.DEFAULT, + isInternalDataStream + ); + GetDataStreamLifecycleAction.Response response = new GetDataStreamLifecycleAction.Response(List.of(dataStreamLifecycle)); + try (XContentBuilder builder = XContentBuilder.builder(XContentType.JSON.xContent())) { + builder.humanReadable(true); + response.toXContentChunked(ToXContent.EMPTY_PARAMS).forEachRemaining(xcontent -> { + try { + xcontent.toXContent(builder, EMPTY_PARAMS); + } catch (IOException e) { + logger.error(e.getMessage(), e); + fail(e.getMessage()); + } + }); + Map resultMap = XContentHelper.convertToMap(BytesReference.bytes(builder), false, builder.contentType()).v2(); + assertThat(resultMap.get("global_retention"), equalTo(Map.of())); + assertThat(resultMap.containsKey("data_streams"), equalTo(true)); + List> dataStreams = (List>) resultMap.get("data_streams"); + Map firstDataStream = dataStreams.get(0); + assertThat(firstDataStream.containsKey("lifecycle"), equalTo(true)); + Map lifecycleResult = (Map) firstDataStream.get("lifecycle"); + assertThat(lifecycleResult.get("enabled"), equalTo(true)); + assertThat(lifecycleResult.get("data_retention"), nullValue()); + assertThat(lifecycleResult.get("effective_retention"), nullValue()); + assertThat(lifecycleResult.get("retention_determined_by"), nullValue()); + } + } + + @SuppressWarnings("unchecked") + public void testGlobalRetentionToXContent() { + TimeValue globalDefaultRetention = TimeValue.timeValueDays(10); + TimeValue globalMaxRetention = TimeValue.timeValueDays(50); + DataStreamGlobalRetention globalRetention = new DataStreamGlobalRetention(globalDefaultRetention, globalMaxRetention); + GetDataStreamLifecycleAction.Response response = new GetDataStreamLifecycleAction.Response(List.of(), null, globalRetention); + try (XContentBuilder builder = XContentBuilder.builder(XContentType.JSON.xContent())) { + builder.humanReadable(true); + response.toXContentChunked(ToXContent.EMPTY_PARAMS).forEachRemaining(xcontent -> { + try { + xcontent.toXContent(builder, EMPTY_PARAMS); + } catch (IOException e) { + logger.error(e.getMessage(), e); + fail(e.getMessage()); + } + }); + Map resultMap = XContentHelper.convertToMap(BytesReference.bytes(builder), false, builder.contentType()).v2(); + assertThat(resultMap.containsKey("global_retention"), equalTo(true)); + Map globalRetentionMap = (Map) resultMap.get("global_retention"); + assertThat(globalRetentionMap.get("max_retention"), equalTo(globalMaxRetention.getStringRep())); + assertThat(globalRetentionMap.get("default_retention"), equalTo(globalDefaultRetention.getStringRep())); + assertThat(resultMap.containsKey("data_streams"), equalTo(true)); + } catch (Exception e) { + fail(e); + } + } + @SuppressWarnings("unchecked") public void testDataStreamLifecycleToXContent() throws Exception { TimeValue configuredRetention = TimeValue.timeValueDays(100); TimeValue globalDefaultRetention = TimeValue.timeValueDays(10); TimeValue globalMaxRetention = TimeValue.timeValueDays(50); + DataStreamGlobalRetention globalRetention = new DataStreamGlobalRetention(globalDefaultRetention, globalMaxRetention); DataStreamLifecycle lifecycle = new DataStreamLifecycle(new DataStreamLifecycle.Retention(configuredRetention), null, null); { boolean isInternalDataStream = true; @@ -39,7 +102,7 @@ public void testDataStreamLifecycleToXContent() throws Exception { lifecycle, isInternalDataStream ); - Map resultMap = getXContentMap(explainIndexDataStreamLifecycle, globalDefaultRetention, globalMaxRetention); + Map resultMap = getXContentMap(explainIndexDataStreamLifecycle, globalRetention); Map lifecycleResult = (Map) resultMap.get("lifecycle"); assertThat(lifecycleResult.get("data_retention"), equalTo(configuredRetention.getStringRep())); assertThat(lifecycleResult.get("effective_retention"), equalTo(configuredRetention.getStringRep())); @@ -51,7 +114,7 @@ public void testDataStreamLifecycleToXContent() throws Exception { lifecycle, isInternalDataStream ); - Map resultMap = getXContentMap(explainIndexDataStreamLifecycle, globalDefaultRetention, globalMaxRetention); + Map resultMap = getXContentMap(explainIndexDataStreamLifecycle, globalRetention); Map lifecycleResult = (Map) resultMap.get("lifecycle"); assertThat(lifecycleResult.get("data_retention"), equalTo(configuredRetention.getStringRep())); assertThat(lifecycleResult.get("effective_retention"), equalTo(globalMaxRetention.getStringRep())); @@ -71,14 +134,11 @@ private GetDataStreamLifecycleAction.Response.DataStreamLifecycle createDataStre */ private Map getXContentMap( GetDataStreamLifecycleAction.Response.DataStreamLifecycle dataStreamLifecycle, - TimeValue globalDefaultRetention, - TimeValue globalMaxRetention + DataStreamGlobalRetention globalRetention ) throws IOException { try (XContentBuilder builder = XContentBuilder.builder(XContentType.JSON.xContent())) { - ToXContent.Params params = new ToXContent.MapParams(Map.of(PATH_RESTRICTED, "serverless")); RolloverConfiguration rolloverConfiguration = null; - DataStreamGlobalRetention globalRetention = new DataStreamGlobalRetention(globalDefaultRetention, globalMaxRetention); - dataStreamLifecycle.toXContent(builder, params, rolloverConfiguration, globalRetention); + dataStreamLifecycle.toXContent(builder, ToXContent.EMPTY_PARAMS, rolloverConfiguration, globalRetention); String serialized = Strings.toString(builder); return XContentHelper.convertToMap(XContentType.JSON.xContent(), serialized, randomBoolean()); }