Skip to content
Merged
Show file tree
Hide file tree
Changes from 13 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions docs/changelog/112210.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
pr: 112210
summary: Expose global retention settings via data stream lifecycle API
area: Data streams
type: enhancement
issues: []
30 changes: 28 additions & 2 deletions docs/reference/data-streams/lifecycle/apis/get-lifecycle.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -67,8 +67,18 @@ Name of the data stream.
=====
`data_retention`::
(Optional, string)
If defined, it represents the retention requested by the data stream owner for this data stream.

`effective_retention`::
(Optional, string)
If defined, every document added to this data stream will be stored at least for this time frame. Any time after this
duration the document could be deleted. When undefined, every document in this data stream will be stored indefinitely.
duration the document could be deleted. When empty, every document in this data stream will be stored indefinitely.
duration the document could be deleted. When empty, every document in this data stream will be stored indefinitely. The
effective retention is calculated as described in the <<effective-retention-calculation, tutorial>>.

`retention_determined_by`::
(Optional, string)
The source of the retention, it can be one of three values, `data_stream_configuration`, `default_retention` or `max_retention`.

`rollover`::
(Optional, object)
Expand All @@ -78,6 +88,21 @@ when the query param `include_defaults` is set to `true`. The contents of this f
=====
====

`global_retention`::
(object)
Contains the global max and default retention. When no global retention is configured, this will be an empty object.
+
.Properties of `global_retention`
[%collapsible%open]
====
`max_retention`::
(Optional, string)
The effective retention of data streams managed by the data stream lifecycle cannot exceed this value.
`default_retention`::
(Optional, string)
This will be the effective retention of data streams managed by the data stream lifecycle that do not specify `data_retention`.
====

[[data-streams-get-lifecycle-example]]
==== {api-examples-title}

Expand Down Expand Up @@ -142,6 +167,7 @@ The response will look like the following:
"retention_determined_by": "data_stream_configuration"
}
}
]
],
"global_retention": {}
}
--------------------------------------------------
Original file line number Diff line number Diff line change
Expand Up @@ -189,19 +189,29 @@ We see that it will remain the same with what the user configured:
[source,console-result]
----
{
"global_retention" : {
"max_retention" : "90d", <1>
"default_retention" : "7d" <2>
},
"data_streams": [
{
"name": "my-data-stream",
"lifecycle": {
"enabled": true,
"data_retention": "30d",
"effective_retention": "30d",
"retention_determined_by": "data_stream_configuration"
"data_retention": "30d", <3>
"effective_retention": "30d", <4>
"retention_determined_by": "data_stream_configuration" <5>
}
}
]
}
----
<1> The maximum retention configured in the cluster.
<2> The default retention configured in the cluster.
<3> The requested retention for this data stream.
<4> The retention that is applied by the data stream lifecycle on this data stream.
<5> The configuration that determined the effective retention. In this case it's the `data_configuration` because
it is less than the `max_retention`.

[discrete]
[[effective-retention-application]]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -99,7 +99,8 @@ The result will look like this:
"retention_determined_by": "data_stream_configuration"
}
}
]
],
"global_retention": {}
}
--------------------------------------------------
<1> The name of your data stream.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,6 @@ public boolean allowSystemIndexAccessByDefault() {

@Override
public Set<String> supportedCapabilities() {
return Set.of(DataStreamLifecycle.EFFECTIVE_RETENTION_REST_API_CAPABILITY);
return Set.of(DataStreamLifecycle.EFFECTIVE_RETENTION_REST_API_CAPABILITY, "data_stream_global_retention");
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
setup:
- requires:
reason: "Global retention was exposed in 8.16+"
test_runner_features: [ capabilities, allowed_warnings ]
capabilities:
- method: GET
path: /_data_stream/{index}/_lifecycle
capabilities: [ 'data_stream_global_retention' ]
- do:
allowed_warnings:
- "index template [my-template] has index patterns [my-data-stream] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template] will take precedence during new index creation"
indices.put_index_template:
name: my-template
body:
index_patterns: [my-data-stream]
template:
settings:
index.number_of_replicas: 1
data_stream: {}
---
teardown:
- do:
cluster.put_settings:
body:
persistent:
data_streams.lifecycle.retention.max: null
data_streams.lifecycle.retention.default: null

---
"Get data stream api check the exposure of global retention":
- do:
indices.get_data_lifecycle:
name: "*"
- match: { global_retention: {} }

- do:
cluster.put_settings:
body:
persistent:
data_streams.lifecycle.retention.default: "7d"
data_streams.lifecycle.retention.max: "90d"

- do:
indices.get_data_lifecycle:
name: "*"
- match: { global_retention.default_retention: "7d" }
- match: { global_retention.max_retention: "90d" }
Original file line number Diff line number Diff line change
Expand Up @@ -254,6 +254,16 @@ public void writeTo(StreamOutput out) throws IOException {
public Iterator<ToXContent> toXContentChunked(ToXContent.Params outerParams) {
return Iterators.concat(Iterators.single((builder, params) -> {
builder.startObject();
builder.startObject("global_retention");
if (globalRetention != null) {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do we not want to show empty global configuration if none is configured? I'm trying to think of something analogous in an API we have (to see if it does this, or shows empty) but haven't thought of anything yet.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I have been going back and forth on this too. In the same when we have no data streams we show an empty list. "data_streams": []. But because this is an object and not a list, I wasn't sure how to approach it.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do you think that having global_retention: {} is more descriptive? Like showing that, yes we do support it and it's non set?

Copy link
Contributor Author

@gmarouli gmarouli Aug 28, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I changed the implementation to reflect this option.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm also on the fence. I have no strong feeling either way. But maybe it's nice having it always present -- that way you know it's not set, rather than that you're just not looking for it in the right place.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Maybe @dakrone has an opinion or knows of some relevant precedent.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

One argument against this is that now we need to update every yaml rest test that returns a data stream to require the data_stream_global_retention, don't we? Otherwise backwards compatibility test runs will fail.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, we do not have that many though. I will double check to ensure they are correct.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

While checking on the global retention in the existing get lifecycle tests, I noticed..... they were not even running :(. I fixed that in 15fa8e1.

I also decided to merge the two files into one. It seems to me that since the global retention is part of the response it should be part of the basic tests.

if (globalRetention.maxRetention() != null) {
builder.field("max_retention", globalRetention.maxRetention().getStringRep());
}
if (globalRetention.defaultRetention() != null) {
builder.field("default_retention", globalRetention.defaultRetention().getStringRep());
}
}
builder.endObject();
builder.startArray(DATA_STREAMS_FIELD.getPreferredName());
return builder;
}),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
import org.elasticsearch.cluster.metadata.DataStreamGlobalRetention;
import org.elasticsearch.cluster.metadata.DataStreamLifecycle;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.xcontent.XContentHelper;
import org.elasticsearch.core.TimeValue;
import org.elasticsearch.test.ESTestCase;
Expand All @@ -20,26 +21,88 @@
import org.elasticsearch.xcontent.XContentType;

import java.io.IOException;
import java.util.List;
import java.util.Map;

import static org.elasticsearch.rest.RestRequest.PATH_RESTRICTED;
import static org.elasticsearch.xcontent.ToXContent.EMPTY_PARAMS;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.nullValue;

public class GetDataStreamLifecycleActionTests extends ESTestCase {

@SuppressWarnings("unchecked")
public void testDefaultLifecycleResponseToXContent() throws Exception {
boolean isInternalDataStream = randomBoolean();
GetDataStreamLifecycleAction.Response.DataStreamLifecycle dataStreamLifecycle = createDataStreamLifecycle(
DataStreamLifecycle.DEFAULT,
isInternalDataStream
);
GetDataStreamLifecycleAction.Response response = new GetDataStreamLifecycleAction.Response(List.of(dataStreamLifecycle));
try (XContentBuilder builder = XContentBuilder.builder(XContentType.JSON.xContent())) {
builder.humanReadable(true);
response.toXContentChunked(ToXContent.EMPTY_PARAMS).forEachRemaining(xcontent -> {
try {
xcontent.toXContent(builder, EMPTY_PARAMS);
} catch (IOException e) {
logger.error(e.getMessage(), e);
fail(e.getMessage());
}
});
Map<String, Object> resultMap = XContentHelper.convertToMap(BytesReference.bytes(builder), false, builder.contentType()).v2();
assertThat(resultMap.get("global_retention"), equalTo(Map.of()));
assertThat(resultMap.containsKey("data_streams"), equalTo(true));
List<Map<String, Object>> dataStreams = (List<Map<String, Object>>) resultMap.get("data_streams");
Map<String, Object> firstDataStream = dataStreams.get(0);
assertThat(firstDataStream.containsKey("lifecycle"), equalTo(true));
Map<String, Object> lifecycleResult = (Map<String, Object>) firstDataStream.get("lifecycle");
assertThat(lifecycleResult.get("enabled"), equalTo(true));
assertThat(lifecycleResult.get("data_retention"), nullValue());
assertThat(lifecycleResult.get("effective_retention"), nullValue());
assertThat(lifecycleResult.get("retention_determined_by"), nullValue());
}
}

@SuppressWarnings("unchecked")
public void testGlobalRetentionToXContent() {
TimeValue globalDefaultRetention = TimeValue.timeValueDays(10);
TimeValue globalMaxRetention = TimeValue.timeValueDays(50);
DataStreamGlobalRetention globalRetention = new DataStreamGlobalRetention(globalDefaultRetention, globalMaxRetention);
GetDataStreamLifecycleAction.Response response = new GetDataStreamLifecycleAction.Response(List.of(), null, globalRetention);
try (XContentBuilder builder = XContentBuilder.builder(XContentType.JSON.xContent())) {
builder.humanReadable(true);
response.toXContentChunked(ToXContent.EMPTY_PARAMS).forEachRemaining(xcontent -> {
try {
xcontent.toXContent(builder, EMPTY_PARAMS);
} catch (IOException e) {
logger.error(e.getMessage(), e);
fail(e.getMessage());
}
});
Map<String, Object> resultMap = XContentHelper.convertToMap(BytesReference.bytes(builder), false, builder.contentType()).v2();
assertThat(resultMap.containsKey("global_retention"), equalTo(true));
Map<String, String> globalRetentionMap = (Map<String, String>) resultMap.get("global_retention");
assertThat(globalRetentionMap.get("max_retention"), equalTo(globalMaxRetention.getStringRep()));
assertThat(globalRetentionMap.get("default_retention"), equalTo(globalDefaultRetention.getStringRep()));
assertThat(resultMap.containsKey("data_streams"), equalTo(true));
} catch (Exception e) {
fail(e);
}
}

@SuppressWarnings("unchecked")
public void testDataStreamLifecycleToXContent() throws Exception {
TimeValue configuredRetention = TimeValue.timeValueDays(100);
TimeValue globalDefaultRetention = TimeValue.timeValueDays(10);
TimeValue globalMaxRetention = TimeValue.timeValueDays(50);
DataStreamGlobalRetention globalRetention = new DataStreamGlobalRetention(globalDefaultRetention, globalMaxRetention);
DataStreamLifecycle lifecycle = new DataStreamLifecycle(new DataStreamLifecycle.Retention(configuredRetention), null, null);
{
boolean isInternalDataStream = true;
GetDataStreamLifecycleAction.Response.DataStreamLifecycle explainIndexDataStreamLifecycle = createDataStreamLifecycle(
lifecycle,
isInternalDataStream
);
Map<String, Object> resultMap = getXContentMap(explainIndexDataStreamLifecycle, globalDefaultRetention, globalMaxRetention);
Map<String, Object> resultMap = getXContentMap(explainIndexDataStreamLifecycle, globalRetention);
Map<String, Object> lifecycleResult = (Map<String, Object>) resultMap.get("lifecycle");
assertThat(lifecycleResult.get("data_retention"), equalTo(configuredRetention.getStringRep()));
assertThat(lifecycleResult.get("effective_retention"), equalTo(configuredRetention.getStringRep()));
Expand All @@ -51,7 +114,7 @@ public void testDataStreamLifecycleToXContent() throws Exception {
lifecycle,
isInternalDataStream
);
Map<String, Object> resultMap = getXContentMap(explainIndexDataStreamLifecycle, globalDefaultRetention, globalMaxRetention);
Map<String, Object> resultMap = getXContentMap(explainIndexDataStreamLifecycle, globalRetention);
Map<String, Object> lifecycleResult = (Map<String, Object>) resultMap.get("lifecycle");
assertThat(lifecycleResult.get("data_retention"), equalTo(configuredRetention.getStringRep()));
assertThat(lifecycleResult.get("effective_retention"), equalTo(globalMaxRetention.getStringRep()));
Expand All @@ -71,14 +134,11 @@ private GetDataStreamLifecycleAction.Response.DataStreamLifecycle createDataStre
*/
private Map<String, Object> getXContentMap(
GetDataStreamLifecycleAction.Response.DataStreamLifecycle dataStreamLifecycle,
TimeValue globalDefaultRetention,
TimeValue globalMaxRetention
DataStreamGlobalRetention globalRetention
) throws IOException {
try (XContentBuilder builder = XContentBuilder.builder(XContentType.JSON.xContent())) {
ToXContent.Params params = new ToXContent.MapParams(Map.of(PATH_RESTRICTED, "serverless"));
RolloverConfiguration rolloverConfiguration = null;
DataStreamGlobalRetention globalRetention = new DataStreamGlobalRetention(globalDefaultRetention, globalMaxRetention);
dataStreamLifecycle.toXContent(builder, params, rolloverConfiguration, globalRetention);
dataStreamLifecycle.toXContent(builder, ToXContent.EMPTY_PARAMS, rolloverConfiguration, globalRetention);
String serialized = Strings.toString(builder);
return XContentHelper.convertToMap(XContentType.JSON.xContent(), serialized, randomBoolean());
}
Expand Down