Exclude internal data streams from global retention (elastic#112100)

With elastic#111972 we enable users to set up global retention for data streams that are managed by the data stream lifecycle. This will allow users of elasticsearch to have a more control over their data retention, and consequently better resource management of their clusters.

However, there is a small number of data streams that are necessary for the good operation of elasticsearch and should not follow user defined retention to avoid surprises.

For this reason, we put forth the following definition of internal data streams.

A data stream is internal if it's either a system index (system flag is true) or if its name starts with a dot.

This PR adds the `isInternalDataStream` param in the effective retention calculation making explicit that this is also used to determine the effective retention.

Author: gmarouli

docs/changelog/112100.yaml

@@ -0,0 +1,5 @@
+pr: 112100
+summary: Exclude internal data streams from global retention
+area: Data streams
+type: bug
+issues: []

docs/reference/data-streams/lifecycle/tutorial-manage-data-stream-retention.asciidoc

@@ -96,6 +96,9 @@ will exceed this time period. This can be set via the <<cluster-update-settings,
 Effective retention cannot be set, it is derived by taking into account all the configured retention listed above and is
 calculated as it is described <<effective-retention-calculation,here>>.
 
+NOTE: Global default and max retention do not apply to data streams internal to elastic. Internal data streams are recognised
+ either by having the `system` flag set to `true` or if their name is prefixed with a dot (`.`).
+
 [discrete]
 [[retention-configuration]]
 ==== How to configure retention?

modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleService.java

@@ -819,7 +819,7 @@ private Index maybeExecuteRollover(ClusterState state, DataStream dataStream, bo
                 RolloverRequest rolloverRequest = getDefaultRolloverRequest(
                     rolloverConfiguration,
                     dataStream.getName(),
-                    dataStream.getLifecycle().getEffectiveDataRetention(dataStream.isSystem() ? null : globalRetentionSettings.get()),
+                    dataStream.getLifecycle().getEffectiveDataRetention(globalRetentionSettings.get(), dataStream.isInternal()),
                     rolloverFailureStore
                 );
                 transportActionsDeduplicator.executeOnce(
@@ -879,7 +879,7 @@ Set<Index> maybeExecuteRetention(ClusterState state, DataStream dataStream, Set<
         Set<Index> indicesToBeRemoved = new HashSet<>();
         // We know that there is lifecycle and retention because there are indices to be deleted
         assert dataStream.getLifecycle() != null;
-        TimeValue effectiveDataRetention = dataStream.getLifecycle().getEffectiveDataRetention(globalRetention);
+        TimeValue effectiveDataRetention = dataStream.getLifecycle().getEffectiveDataRetention(globalRetention, dataStream.isInternal());
         for (Index index : backingIndicesOlderThanRetention) {
             if (indicesToExcludeForRemainingRun.contains(index) == false) {
                 IndexMetadata backingIndex = metadata.index(index);

modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/TransportExplainDataStreamLifecycleAction.java

@@ -103,7 +103,7 @@ protected void masterOperation(
             ExplainIndexDataStreamLifecycle explainIndexDataStreamLifecycle = new ExplainIndexDataStreamLifecycle(
                 index,
                 true,
-                parentDataStream.isSystem(),
+                parentDataStream.isInternal(),
                 idxMetadata.getCreationDate(),
                 rolloverInfo == null ? null : rolloverInfo.getTime(),
                 generationDate,

server/src/main/java/org/elasticsearch/action/datastreams/GetDataStreamAction.java

@@ -319,8 +319,7 @@ public XContentBuilder toXContent(
                 }
                 if (dataStream.getLifecycle() != null) {
                     builder.field(LIFECYCLE_FIELD.getPreferredName());
-                    dataStream.getLifecycle()
-                        .toXContent(builder, params, rolloverConfiguration, dataStream.isSystem() ? null : globalRetention);
+                    dataStream.getLifecycle().toXContent(builder, params, rolloverConfiguration, globalRetention, dataStream.isInternal());
                 }
                 if (ilmPolicyName != null) {
                     builder.field(ILM_POLICY_FIELD.getPreferredName(), ilmPolicyName);

server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/ExplainIndexDataStreamLifecycle.java

@@ -45,7 +45,7 @@ public class ExplainIndexDataStreamLifecycle implements Writeable, ToXContentObj
 
     private final String index;
     private final boolean managedByLifecycle;
-    private final boolean isSystemDataStream;
+    private final boolean isInternalDataStream;
     @Nullable
     private final Long indexCreationDate;
     @Nullable
@@ -61,7 +61,7 @@ public class ExplainIndexDataStreamLifecycle implements Writeable, ToXContentObj
     public ExplainIndexDataStreamLifecycle(
         String index,
         boolean managedByLifecycle,
-        boolean isSystemDataStream,
+        boolean isInternalDataStream,
         @Nullable Long indexCreationDate,
         @Nullable Long rolloverDate,
         @Nullable TimeValue generationDate,
@@ -70,7 +70,7 @@ public ExplainIndexDataStreamLifecycle(
     ) {
         this.index = index;
         this.managedByLifecycle = managedByLifecycle;
-        this.isSystemDataStream = isSystemDataStream;
+        this.isInternalDataStream = isInternalDataStream;
         this.indexCreationDate = indexCreationDate;
         this.rolloverDate = rolloverDate;
         this.generationDateMillis = generationDate == null ? null : generationDate.millis();
@@ -82,9 +82,9 @@ public ExplainIndexDataStreamLifecycle(StreamInput in) throws IOException {
         this.index = in.readString();
         this.managedByLifecycle = in.readBoolean();
         if (in.getTransportVersion().onOrAfter(TransportVersions.NO_GLOBAL_RETENTION_FOR_SYSTEM_DATA_STREAMS)) {
-            this.isSystemDataStream = in.readBoolean();
+            this.isInternalDataStream = in.readBoolean();
         } else {
-            this.isSystemDataStream = false;
+            this.isInternalDataStream = false;
         }
         if (managedByLifecycle) {
             this.indexCreationDate = in.readOptionalLong();
@@ -141,7 +141,7 @@ public XContentBuilder toXContent(
             }
             if (this.lifecycle != null) {
                 builder.field(LIFECYCLE_FIELD.getPreferredName());
-                lifecycle.toXContent(builder, params, rolloverConfiguration, isSystemDataStream ? null : globalRetention);
+                lifecycle.toXContent(builder, params, rolloverConfiguration, globalRetention, isInternalDataStream);
             }
             if (this.error != null) {
                 if (error.firstOccurrenceTimestamp() != -1L && error.recordedTimestamp() != -1L && error.retryCount() != -1) {
@@ -161,7 +161,7 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeString(index);
         out.writeBoolean(managedByLifecycle);
         if (out.getTransportVersion().onOrAfter(TransportVersions.NO_GLOBAL_RETENTION_FOR_SYSTEM_DATA_STREAMS)) {
-            out.writeBoolean(isSystemDataStream);
+            out.writeBoolean(isInternalDataStream);
         }
         if (managedByLifecycle) {
             out.writeOptionalLong(indexCreationDate);

server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/GetDataStreamLifecycleAction.java

@@ -143,7 +143,7 @@ public static class Response extends ActionResponse implements ChunkedToXContent
         public record DataStreamLifecycle(
             String dataStreamName,
             @Nullable org.elasticsearch.cluster.metadata.DataStreamLifecycle lifecycle,
-            boolean isSystemDataStream
+            boolean isInternalDataStream
         ) implements Writeable, ToXContentObject {
 
             public static final ParseField NAME_FIELD = new ParseField("name");
@@ -162,7 +162,7 @@ public void writeTo(StreamOutput out) throws IOException {
                 out.writeString(dataStreamName);
                 out.writeOptionalWriteable(lifecycle);
                 if (out.getTransportVersion().onOrAfter(TransportVersions.NO_GLOBAL_RETENTION_FOR_SYSTEM_DATA_STREAMS)) {
-                    out.writeBoolean(isSystemDataStream);
+                    out.writeBoolean(isInternalDataStream);
                 }
             }
 
@@ -189,7 +189,8 @@ public XContentBuilder toXContent(
                         builder,
                         org.elasticsearch.cluster.metadata.DataStreamLifecycle.addEffectiveRetentionParams(params),
                         rolloverConfiguration,
-                        isSystemDataStream ? null : globalRetention
+                        globalRetention,
+                        isInternalDataStream
                     );
                 }
                 builder.endObject();

server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java

@@ -277,6 +277,18 @@ public boolean rolloverOnWrite() {
         return backingIndices.rolloverOnWrite;
     }
 
+    /**
+     * We define that a data stream is considered internal either if it is a system index or if
+     * its name starts with a dot.
+     *
+     * Note: Dot-prefixed internal data streams is a naming convention for internal data streams,
+     * but it's not yet enforced.
+     * @return true if it's a system index or has a dot-prefixed name.
+     */
+    public boolean isInternal() {
+        return isSystem() || name.charAt(0) == '.';
+    }
+
     /**
      * @param timestamp The timestamp used to select a backing index based on its start and end time.
      * @param metadata  The metadata that is used to fetch the start and end times for backing indices of this data stream.
@@ -796,12 +808,12 @@ public List<Index> getIndicesPastRetention(
     ) {
         if (lifecycle == null
             || lifecycle.isEnabled() == false
-            || lifecycle.getEffectiveDataRetention(isSystem() ? null : globalRetention) == null) {
+            || lifecycle.getEffectiveDataRetention(globalRetention, isInternal()) == null) {
             return List.of();
         }
 
         List<Index> indicesPastRetention = getNonWriteIndicesOlderThan(
-            lifecycle.getEffectiveDataRetention(isSystem() ? null : globalRetention),
+            lifecycle.getEffectiveDataRetention(globalRetention, isInternal()),
             indexMetadataSupplier,
             this::isIndexManagedByDataStreamLifecycle,
             nowSupplier
@@ -1202,7 +1214,7 @@ public XContentBuilder toXContent(
         }
         if (lifecycle != null) {
             builder.field(LIFECYCLE.getPreferredName());
-            lifecycle.toXContent(builder, params, rolloverConfiguration, isSystem() ? null : globalRetention);
+            lifecycle.toXContent(builder, params, rolloverConfiguration, globalRetention, isInternal());
         }
         builder.field(ROLLOVER_ON_WRITE_FIELD.getPreferredName(), backingIndices.rolloverOnWrite);
         if (backingIndices.autoShardingEvent != null) {

server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamLifecycle.java

@@ -65,6 +65,7 @@ public class DataStreamLifecycle implements SimpleDiffable<DataStreamLifecycle>,
         DataStreamLifecycle.INCLUDE_EFFECTIVE_RETENTION_PARAM_NAME,
         "true"
     );
+    public static final Tuple<TimeValue, RetentionSource> INFINITE_RETENTION = Tuple.tuple(null, RetentionSource.DATA_STREAM_CONFIGURATION);
 
     /**
      * Check if {@link #DATA_STREAMS_LIFECYCLE_ONLY_SETTING_NAME} is present and set to {@code true}, indicating that
@@ -145,32 +146,37 @@ public boolean isEnabled() {
     }
 
     /**
-     * The least amount of time data should be kept by elasticsearch. If a caller does not want the global retention considered (for
-     * example, when evaluating the effective retention for a system data stream or a template) then null should be given for
-     * globalRetention.
-     * @param globalRetention The global retention, or null if global retention does not exist or should not be applied
+     * The least amount of time data should be kept by elasticsearch. The effective retention is a function with three parameters,
+     * the {@link DataStreamLifecycle#dataRetention}, the global retention and whether this lifecycle is associated with an internal
+     * data stream.
+     * @param globalRetention The global retention, or null if global retention does not exist.
+     * @param isInternalDataStream A flag denoting if this lifecycle is associated with an internal data stream or not
      * @return the time period or null, null represents that data should never be deleted.
      */
     @Nullable
-    public TimeValue getEffectiveDataRetention(@Nullable DataStreamGlobalRetention globalRetention) {
-        return getEffectiveDataRetentionWithSource(globalRetention).v1();
+    public TimeValue getEffectiveDataRetention(@Nullable DataStreamGlobalRetention globalRetention, boolean isInternalDataStream) {
+        return getEffectiveDataRetentionWithSource(globalRetention, isInternalDataStream).v1();
     }
 
     /**
-     * The least amount of time data should be kept by elasticsearch. If a caller does not want the global retention considered (for
-     * example, when evaluating the effective retention for a system data stream or a template) then null should be given for
-     * globalRetention.
-     * @param globalRetention The global retention, or null if global retention does not exist or should not be applied
+     * The least amount of time data should be kept by elasticsearch.. The effective retention is a function with three parameters,
+     * the {@link DataStreamLifecycle#dataRetention}, the global retention and whether this lifecycle is associated with an internal
+     * data stream.
+     * @param globalRetention The global retention, or null if global retention does not exist.
+     * @param isInternalDataStream A flag denoting if this lifecycle is associated with an internal data stream or not
      * @return A tuple containing the time period or null as v1 (where null represents that data should never be deleted), and the non-null
      * retention source as v2.
      */
-    public Tuple<TimeValue, RetentionSource> getEffectiveDataRetentionWithSource(@Nullable DataStreamGlobalRetention globalRetention) {
+    public Tuple<TimeValue, RetentionSource> getEffectiveDataRetentionWithSource(
+        @Nullable DataStreamGlobalRetention globalRetention,
+        boolean isInternalDataStream
+    ) {
         // If lifecycle is disabled there is no effective retention
         if (enabled == false) {
-            return Tuple.tuple(null, RetentionSource.DATA_STREAM_CONFIGURATION);
+            return INFINITE_RETENTION;
         }
         var dataStreamRetention = getDataStreamRetention();
-        if (globalRetention == null) {
+        if (globalRetention == null || isInternalDataStream) {
             return Tuple.tuple(dataStreamRetention, RetentionSource.DATA_STREAM_CONFIGURATION);
         }
         if (dataStreamRetention == null) {
@@ -187,7 +193,7 @@ public Tuple<TimeValue, RetentionSource> getEffectiveDataRetentionWithSource(@Nu
 
     /**
      * The least amount of time data the data stream is requesting es to keep the data.
-     * NOTE: this can be overridden by the {@link DataStreamLifecycle#getEffectiveDataRetention(DataStreamGlobalRetention)}.
+     * NOTE: this can be overridden by the {@link DataStreamLifecycle#getEffectiveDataRetention(DataStreamGlobalRetention,boolean)}.
      * @return the time period or null, null represents that data should never be deleted.
      */
     @Nullable
@@ -199,12 +205,16 @@ public TimeValue getDataStreamRetention() {
      * This method checks if the effective retention is matching what the user has configured; if the effective retention
      * does not match then it adds a warning informing the user about the effective retention and the source.
      */
-    public void addWarningHeaderIfDataRetentionNotEffective(@Nullable DataStreamGlobalRetention globalRetention) {
-        if (globalRetention == null) {
+    public void addWarningHeaderIfDataRetentionNotEffective(
+        @Nullable DataStreamGlobalRetention globalRetention,
+        boolean isInternalDataStream
+    ) {
+        if (globalRetention == null || isInternalDataStream) {
             return;
         }
         Tuple<TimeValue, DataStreamLifecycle.RetentionSource> effectiveDataRetentionWithSource = getEffectiveDataRetentionWithSource(
-            globalRetention
+            globalRetention,
+            isInternalDataStream
         );
         if (effectiveDataRetentionWithSource.v1() == null) {
             return;
@@ -318,20 +328,22 @@ public String toString() {
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        return toXContent(builder, params, null, null);
+        return toXContent(builder, params, null, null, false);
     }
 
     /**
      * Converts the data stream lifecycle to XContent, enriches it with effective retention information when requested
      * and injects the RolloverConditions if they exist.
      * In order to request the effective retention you need to set {@link #INCLUDE_EFFECTIVE_RETENTION_PARAM_NAME} to true
      * in the XContent params.
+     * NOTE: this is used for serialising user output and the result is never deserialised in elasticsearch.
      */
     public XContentBuilder toXContent(
         XContentBuilder builder,
         Params params,
         @Nullable RolloverConfiguration rolloverConfiguration,
-        @Nullable DataStreamGlobalRetention globalRetention
+        @Nullable DataStreamGlobalRetention globalRetention,
+        boolean isInternalDataStream
     ) throws IOException {
         builder.startObject();
         builder.field(ENABLED_FIELD.getPreferredName(), enabled);
@@ -342,11 +354,14 @@ public XContentBuilder toXContent(
                 builder.field(DATA_RETENTION_FIELD.getPreferredName(), dataRetention.value().getStringRep());
             }
         }
+        Tuple<TimeValue, RetentionSource> effectiveDataRetentionWithSource = getEffectiveDataRetentionWithSource(
+            globalRetention,
+            isInternalDataStream
+        );
         if (params.paramAsBoolean(INCLUDE_EFFECTIVE_RETENTION_PARAM_NAME, false)) {
-            Tuple<TimeValue, RetentionSource> effectiveRetention = getEffectiveDataRetentionWithSource(globalRetention);
-            if (effectiveRetention.v1() != null) {
-                builder.field(EFFECTIVE_RETENTION_FIELD.getPreferredName(), effectiveRetention.v1().getStringRep());
-                builder.field(RETENTION_SOURCE_FIELD.getPreferredName(), effectiveRetention.v2().displayName());
+            if (effectiveDataRetentionWithSource.v1() != null) {
+                builder.field(EFFECTIVE_RETENTION_FIELD.getPreferredName(), effectiveDataRetentionWithSource.v1().getStringRep());
+                builder.field(RETENTION_SOURCE_FIELD.getPreferredName(), effectiveDataRetentionWithSource.v2().displayName());
             }
         }
 
@@ -356,12 +371,18 @@ public XContentBuilder toXContent(
         }
         if (rolloverConfiguration != null) {
             builder.field(ROLLOVER_FIELD.getPreferredName());
-            rolloverConfiguration.evaluateAndConvertToXContent(builder, params, getEffectiveDataRetention(globalRetention));
+            rolloverConfiguration.evaluateAndConvertToXContent(builder, params, effectiveDataRetentionWithSource.v1());
         }
         builder.endObject();
         return builder;
     }
 
+    /**
+     * This method deserialises XContent format as it was generated ONLY by {@link DataStreamLifecycle#toXContent(XContentBuilder, Params)}.
+     * It does not support the output of
+     * {@link DataStreamLifecycle#toXContent(XContentBuilder, Params, RolloverConfiguration, DataStreamGlobalRetention, boolean)} because
+     * this output is enriched with derived fields we do not handle in this deserialisation.
+     */
     public static DataStreamLifecycle fromXContent(XContentParser parser) throws IOException {
         return PARSER.parse(parser, null);
     }

server/src/main/java/org/elasticsearch/cluster/metadata/MetadataDataStreamsService.java

@@ -214,17 +214,15 @@ static ClusterState modifyDataStream(
     ClusterState updateDataLifecycle(ClusterState currentState, List<String> dataStreamNames, @Nullable DataStreamLifecycle lifecycle) {
         Metadata metadata = currentState.metadata();
         Metadata.Builder builder = Metadata.builder(metadata);
-        boolean atLeastOneDataStreamIsNotSystem = false;
+        boolean onlyInternalDataStreams = true;
         for (var dataStreamName : dataStreamNames) {
             var dataStream = validateDataStream(metadata, dataStreamName);
             builder.put(dataStream.copy().setLifecycle(lifecycle).build());
-            atLeastOneDataStreamIsNotSystem = atLeastOneDataStreamIsNotSystem || dataStream.isSystem() == false;
+            onlyInternalDataStreams = onlyInternalDataStreams && dataStream.isInternal();
         }
         if (lifecycle != null) {
-            if (atLeastOneDataStreamIsNotSystem) {
-                // We don't issue any warnings if all data streams are system data streams
-                lifecycle.addWarningHeaderIfDataRetentionNotEffective(globalRetentionSettings.get());
-            }
+            // We don't issue any warnings if all data streams are internal data streams
+            lifecycle.addWarningHeaderIfDataRetentionNotEffective(globalRetentionSettings.get(), onlyInternalDataStreams);
         }
         return ClusterState.builder(currentState).metadata(builder.build()).build();
     }