diff --git a/docs/reference/indices/get-data-stream.asciidoc b/docs/reference/indices/get-data-stream.asciidoc index b88a1a1be2a7e..09d64a5b80c48 100644 --- a/docs/reference/indices/get-data-stream.asciidoc +++ b/docs/reference/indices/get-data-stream.asciidoc @@ -253,8 +253,17 @@ Functionality in preview:[]. Contains the configuration for the data stream life ===== `data_retention`:: (string) +If defined, it represents the retention requested by the data stream owner for this data stream. + +`effective_retention`:: +(string) If defined, every document added to this data stream will be stored at least for this time frame. Any time after this -duration the document could be deleted. When empty, every document in this data stream will be stored indefinitely. +duration the document could be deleted. When empty, every document in this data stream will be stored indefinitely. The +effective retention is calculated as described in the <>. + +`retention_determined_by`:: +(string) +The source of the retention, it can be one of three values, `data_stream_configuration`, `default_retention` or `max_retention`. `rollover`:: (object) @@ -269,6 +278,22 @@ If `true`, the next write to this data stream will trigger a rollover first and indexed in the new backing index. If the rollover fails the indexing request will fail too. ==== +`global_retention`:: +(object) +Contains the global max and default retention. Global retention is only retrieved when the request sets `include_defaults` to `true`. ++ +.Properties of `global_retention` +[%collapsible%open] +==== +`max_retention`:: +(string) +The effective retention of data streams managed by the data stream lifecycle cannot exceed this value. + +`default_retention`:: +(string) +This will be the effective retention of data streams managed by the data stream lifecycle that do not specify `data_retention`. +===== + [[get-data-stream-api-example]] ==== {api-examples-title} diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/GetDataStreamsTransportAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/GetDataStreamsTransportAction.java index dcca32355082b..38088e62c6c92 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/GetDataStreamsTransportAction.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/GetDataStreamsTransportAction.java @@ -223,7 +223,8 @@ public int compareTo(IndexInfo o) { return new GetDataStreamAction.Response( dataStreamInfos, request.includeDefaults() ? clusterSettings.get(DataStreamLifecycle.CLUSTER_LIFECYCLE_DEFAULT_ROLLOVER_SETTING) : null, - globalRetentionSettings.get() + globalRetentionSettings.get(), + request.includeDefaults() ); } diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestGetDataStreamsAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestGetDataStreamsAction.java index c3178208d51c2..f4d90ca1960d7 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestGetDataStreamsAction.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestGetDataStreamsAction.java @@ -55,6 +55,6 @@ public boolean allowSystemIndexAccessByDefault() { @Override public Set supportedCapabilities() { - return Set.of(DataStreamLifecycle.EFFECTIVE_RETENTION_REST_API_CAPABILITY); + return Set.of(DataStreamLifecycle.EFFECTIVE_RETENTION_REST_API_CAPABILITY, "data_stream_global_retention"); } } diff --git a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/10_basic.yml b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/10_basic.yml index ec0c82365a681..e617f98b055da 100644 --- a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/10_basic.yml +++ b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/10_basic.yml @@ -1150,11 +1150,53 @@ setup: name: simple-data-stream2 - is_true: acknowledged +--- +"Get data stream api check the exposure of global retention": + - requires: + reason: "Global retention was exposed in 8.16+" + test_runner_features: [ capabilities ] + capabilities: + - method: GET + path: /_data_stream/{index} + capabilities: [ 'data_stream_global_retention' ] + + - do: + indices.get_data_stream: + name: "*" + include_defaults: true + - match: { global_retention: { } } + + - do: + cluster.put_settings: + body: + persistent: + data_streams.lifecycle.retention.default: "7d" + data_streams.lifecycle.retention.max: "90d" + + - do: + indices.get_data_stream: + name: "*" + include_defaults: true + - match: { global_retention.default_retention: "7d" } + - match: { global_retention.max_retention: "90d" } + + - do: + indices.get_data_stream: + name: "*" + - is_false: global_retention + + - do: + cluster.put_settings: + body: + persistent: + data_streams.lifecycle.retention.max: null + data_streams.lifecycle.retention.default: null + --- "Create data stream with match all template": - requires: cluster_features: ["gte_v8.16.0"] - reason: "data streams supoprt for match all templates only supported in 8.16" + reason: "data streams support for match all templates only supported in 8.16" - do: allowed_warnings: diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java index 9fe270e933785..c6b48a213d741 100644 --- a/server/src/main/java/org/elasticsearch/TransportVersions.java +++ b/server/src/main/java/org/elasticsearch/TransportVersions.java @@ -195,6 +195,7 @@ static TransportVersion def(int id) { public static final TransportVersion ESQL_PROFILE_SLEEPS = def(8_725_00_0); public static final TransportVersion ZDT_NANOS_SUPPORT = def(8_726_00_0); public static final TransportVersion LTR_SERVERLESS_RELEASE = def(8_727_00_0); + public static final TransportVersion EXPOSE_DATA_STREAM_GLOBAL_RETENTION = def(8_728_00_0); /* * STOP! READ THIS FIRST! No, really, * ____ _____ ___ ____ _ ____ _____ _ ____ _____ _ _ ___ ____ _____ ___ ____ ____ _____ _ diff --git a/server/src/main/java/org/elasticsearch/action/datastreams/GetDataStreamAction.java b/server/src/main/java/org/elasticsearch/action/datastreams/GetDataStreamAction.java index 2fcc5ce3702c1..af1f2608fe59d 100644 --- a/server/src/main/java/org/elasticsearch/action/datastreams/GetDataStreamAction.java +++ b/server/src/main/java/org/elasticsearch/action/datastreams/GetDataStreamAction.java @@ -499,29 +499,41 @@ public void writeTo(StreamOutput out) throws IOException { private final RolloverConfiguration rolloverConfiguration; @Nullable private final DataStreamGlobalRetention globalRetention; + private final boolean includeGlobalRetention; public Response(List dataStreams) { - this(dataStreams, null, null); + this(dataStreams, null, null, false); } public Response( List dataStreams, @Nullable RolloverConfiguration rolloverConfiguration, - @Nullable DataStreamGlobalRetention globalRetention + @Nullable DataStreamGlobalRetention globalRetention, + boolean includeGlobalRetention ) { this.dataStreams = dataStreams; this.rolloverConfiguration = rolloverConfiguration; this.globalRetention = globalRetention; + this.includeGlobalRetention = includeGlobalRetention; } public Response(StreamInput in) throws IOException { - this( - in.readCollectionAsList(DataStreamInfo::new), - in.getTransportVersion().onOrAfter(TransportVersions.V_8_9_X) ? in.readOptionalWriteable(RolloverConfiguration::new) : null, - in.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0) - ? in.readOptionalWriteable(DataStreamGlobalRetention::read) - : null - ); + this.dataStreams = in.readCollectionAsList(DataStreamInfo::new); + if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_9_X)) { + this.rolloverConfiguration = in.readOptionalWriteable(RolloverConfiguration::new); + } else { + this.rolloverConfiguration = null; + } + if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)) { + this.globalRetention = in.readOptionalWriteable(DataStreamGlobalRetention::read); + } else { + this.globalRetention = null; + } + if (in.getTransportVersion().onOrAfter(TransportVersions.EXPOSE_DATA_STREAM_GLOBAL_RETENTION)) { + this.includeGlobalRetention = in.readBoolean(); + } else { + this.includeGlobalRetention = rolloverConfiguration != null; + } } public List getDataStreams() { @@ -547,11 +559,26 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)) { out.writeOptionalWriteable(globalRetention); } + if (out.getTransportVersion().onOrAfter(TransportVersions.EXPOSE_DATA_STREAM_GLOBAL_RETENTION)) { + out.writeBoolean(includeGlobalRetention); + } } @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); + if (includeGlobalRetention) { + builder.startObject("global_retention"); + if (globalRetention != null) { + if (globalRetention.maxRetention() != null) { + builder.field("max_retention", globalRetention.maxRetention().getStringRep()); + } + if (globalRetention.defaultRetention() != null) { + builder.field("default_retention", globalRetention.defaultRetention().getStringRep()); + } + } + builder.endObject(); + } builder.startArray(DATA_STREAMS_FIELD.getPreferredName()); for (DataStreamInfo dataStream : dataStreams) { dataStream.toXContent( @@ -573,12 +600,13 @@ public boolean equals(Object o) { Response response = (Response) o; return dataStreams.equals(response.dataStreams) && Objects.equals(rolloverConfiguration, response.rolloverConfiguration) - && Objects.equals(globalRetention, response.globalRetention); + && Objects.equals(globalRetention, response.globalRetention) + && includeGlobalRetention == response.includeGlobalRetention; } @Override public int hashCode() { - return Objects.hash(dataStreams, rolloverConfiguration, globalRetention); + return Objects.hash(dataStreams, rolloverConfiguration, globalRetention, includeGlobalRetention); } }