Composing data lifecycle in index templates (#96027)

gmarouli · web-flow · commit 815b02192ee0 · 2023-05-15T17:58:48.000+02:00
In this PR we introduce the composing of lifecycle templates. It follows the pattern of the other templates, for example, in the following case we have two lifecycle both of which have the data_retention defined.
```
     [
       {
         "lifecycle": {
           "data_retention" : "10d"
         }
       },
       {
         "lifecycle": {
           "data_retention" : "20d"
         }
       }
     ]
```
The result will be { "lifecycle": { "data_retention" : "20d"}}.

However, if we have the following two lifecycles:
```
     [
       {
         "lifecycle": {
           "data_retention" : "10d"
         }
       },
       {
       "lifecycle": { }
       }
     ]

The result will be { "lifecycle": { "data_retention" : "10d"} } because the latest lifecycle does not the data_retention defined.
```
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/DataLifecycle.java b/server/src/main/java/org/elasticsearch/cluster/metadata/DataLifecycle.java
@@ -26,6 +26,7 @@
 import org.elasticsearch.xcontent.XContentParser;
 
 import java.io.IOException;
+import java.util.List;
 import java.util.Objects;
 
 /**
@@ -83,6 +84,54 @@ public DataLifecycle(long timeInMills) {
         this(TimeValue.timeValueMillis(timeInMills));
     }
 
+    /**
+     * This method composes a series of lifecycles to a final one. The lifecycles are getting composed one level deep,
+     * meaning that the keys present on the latest lifecycle will override the ones of the others. If a key is missing
+     * then it keeps the value of the previous lifecycles. For example, if we have the following two lifecycles:
+     * [
+     *   {
+     *     "lifecycle": {
+     *       "data_retention" : "10d"
+     *     }
+     *   },
+     *   {
+     *     "lifecycle": {
+     *       "data_retention" : "20d"
+     *     }
+     *   }
+     * ]
+     * The result will be { "lifecycle": { "data_retention" : "20d"}} because the second data retention overrides the first.
+     * However, if we have the following two lifecycles:
+     * [
+     *   {
+     *     "lifecycle": {
+     *       "data_retention" : "10d"
+     *     }
+     *   },
+     *   {
+     *   "lifecycle": { }
+     *   }
+     * ]
+     * The result will be { "lifecycle": { "data_retention" : "10d"} } because the latest lifecycle does not have any
+     * information on retention.
+     * @param lifecycles a sorted list of lifecycles in the order that they will be composed
+     * @return the final lifecycle
+     */
+    @Nullable
+    public static DataLifecycle compose(List<DataLifecycle> lifecycles) {
+        DataLifecycle.Builder builder = null;
+        for (DataLifecycle current : lifecycles) {
+            if (builder == null) {
+                builder = Builder.newBuilder(current);
+            } else {
+                if (current.dataRetention != null) {
+                    builder.dataRetention(current.getDataRetention());
+                }
+            }
+        }
+        return builder == null ? null : builder.build();
+    }
+
     @Nullable
     public TimeValue getDataRetention() {
         return dataRetention;
@@ -145,4 +194,25 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params, @Nulla
     public static DataLifecycle fromXContent(XContentParser parser) throws IOException {
         return PARSER.parse(parser, null);
     }
+
+    /**
+     * This builder helps during the composition of the data lifecycle templates.
+     */
+    static class Builder {
+        @Nullable
+        private TimeValue dataRetention = null;
+
+        Builder dataRetention(@Nullable TimeValue value) {
+            dataRetention = value;
+            return this;
+        }
+
+        DataLifecycle build() {
+            return new DataLifecycle(dataRetention);
+        }
+
+        static Builder newBuilder(DataLifecycle dataLifecycle) {
+            return new Builder().dataRetention(dataLifecycle.getDataRetention());
+        }
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java
@@ -1423,19 +1423,22 @@ public static DataLifecycle resolveLifecycle(final Metadata metadata, final Stri
     public static DataLifecycle resolveLifecycle(ComposableIndexTemplate template, Map<String, ComponentTemplate> componentTemplates) {
         Objects.requireNonNull(template, "attempted to resolve lifecycle for a null template");
         Objects.requireNonNull(componentTemplates, "attempted to resolve lifecycle with null component templates");
-        // The actual index template's lifecycle takes the highest precedence.
+
+        List<DataLifecycle> lifecycles = new ArrayList<>();
+        for (String componentTemplateName : template.composedOf()) {
+            if (componentTemplates.containsKey(componentTemplateName) == false) {
+                continue;
+            }
+            DataLifecycle dataLifecycle = componentTemplates.get(componentTemplateName).template().lifecycle();
+            if (dataLifecycle != null) {
+                lifecycles.add(dataLifecycle);
+            }
+        }
+        // The actual index template's lifecycle has the highest precedence.
         if (template.template() != null && template.template().lifecycle() != null) {
-            return template.template().lifecycle();
+            lifecycles.add(template.template().lifecycle());
         }
-        List<DataLifecycle> componentLifecycle = template.composedOf()
-            .stream()
-            .map(componentTemplates::get)
-            .filter(Objects::nonNull)
-            .map(ComponentTemplate::template)
-            .map(Template::lifecycle)
-            .filter(Objects::nonNull)
-            .toList();
-        return componentLifecycle.isEmpty() ? null : componentLifecycle.get(componentLifecycle.size() - 1);
+        return DataLifecycle.compose(lifecycles);
     }
 
     /**
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/DataLifecycleTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/DataLifecycleTests.java
@@ -23,6 +23,7 @@
 import org.elasticsearch.xcontent.XContentType;
 
 import java.io.IOException;
+import java.util.List;
 import java.util.Set;
 
 import static org.hamcrest.Matchers.containsString;
@@ -78,6 +79,34 @@ public void testXContentSerializationWithRollover() throws IOException {
         }
     }
 
+    public void testLifecycleComposition() {
+        // No lifecycles result to null
+        {
+            List<DataLifecycle> lifecycles = List.of();
+            assertThat(DataLifecycle.compose(lifecycles), nullValue());
+        }
+        // One lifecycle results to this lifecycle as the final
+        {
+            DataLifecycle lifecycle = createTestInstance();
+            List<DataLifecycle> lifecycles = List.of(lifecycle);
+            assertThat(DataLifecycle.compose(lifecycles), equalTo(lifecycle));
+        }
+        // If the last lifecycle is missing a property we keep the latest from the previous ones
+        {
+            DataLifecycle lifecycleWithRetention = new DataLifecycle(randomMillisUpToYear9999());
+            List<DataLifecycle> lifecycles = List.of(lifecycleWithRetention, new DataLifecycle());
+            assertThat(DataLifecycle.compose(lifecycles).getDataRetention(), equalTo(lifecycleWithRetention.getDataRetention()));
+        }
+        // If both lifecycle have all properties, then the latest one overwrites all the others
+        {
+            DataLifecycle lifecycle1 = new DataLifecycle(randomMillisUpToYear9999());
+            DataLifecycle lifecycle2 = new DataLifecycle(randomMillisUpToYear9999());
+            List<DataLifecycle> lifecycles = List.of(lifecycle1, lifecycle2);
+            assertThat(DataLifecycle.compose(lifecycles), equalTo(lifecycle2));
+        }
+
+    }
+
     public void testDefaultClusterSetting() {
         ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
         RolloverConfiguration rolloverConfiguration = clusterSettings.get(DataLifecycle.CLUSTER_DLM_DEFAULT_ROLLOVER_SETTING);
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java
@@ -1505,15 +1505,15 @@ public void testResolveLifecycle() throws Exception {
         ComponentTemplate ct2 = new ComponentTemplate(new Template(null, null, null, dataLifecycle2), null, null);
         ComponentTemplate ctNoLifecycle = new ComponentTemplate(new Template(null, null, null, null), null, null);
 
-        state = service.addComponentTemplate(state, true, "ct_high", ct1);
-        state = service.addComponentTemplate(state, true, "ct_low", ct2);
+        state = service.addComponentTemplate(state, true, "ct_1", ct1);
+        state = service.addComponentTemplate(state, true, "ct_2", ct2);
         state = service.addComponentTemplate(state, true, "ct_no_lifecycle", ctNoLifecycle);
         {
             // Respect the order the templates are defined
             ComposableIndexTemplate it = new ComposableIndexTemplate(
                 List.of("i1*"),
                 new Template(null, null, null),
-                List.of("ct_low", "ct_high"),
+                List.of("ct_2", "ct_1"),
                 0L,
                 1L,
                 null,
@@ -1530,7 +1530,7 @@ public void testResolveLifecycle() throws Exception {
             ComposableIndexTemplate it = new ComposableIndexTemplate(
                 List.of("i2*"),
                 new Template(null, null, null),
-                List.of("ct_high", "ct_no_lifecycle"),
+                List.of("ct_1", "ct_no_lifecycle"),
                 0L,
                 1L,
                 null,
@@ -1548,7 +1548,7 @@ public void testResolveLifecycle() throws Exception {
             ComposableIndexTemplate it = new ComposableIndexTemplate(
                 List.of("i3*"),
                 new Template(null, null, null, dataLifecycle2),
-                List.of("ct_no_lifecycle", "ct_high"),
+                List.of("ct_no_lifecycle", "ct_1"),
                 0L,
                 1L,
                 null,
@@ -1559,9 +1559,27 @@ public void testResolveLifecycle() throws Exception {
 
             DataLifecycle resolvedLifecycle = MetadataIndexTemplateService.resolveLifecycle(state.metadata(), "my-template-3");
 
-            // Based on precedence only the latest
+            // The index template has higher precedence and overwrites the retention of the others.
             assertThat(resolvedLifecycle, equalTo(dataLifecycle2));
         }
+        {
+            ComposableIndexTemplate it = new ComposableIndexTemplate(
+                List.of("i4*"),
+                new Template(null, null, null, new DataLifecycle()),
+                List.of("ct_no_lifecycle", "ct_1"),
+                0L,
+                1L,
+                null,
+                new ComposableIndexTemplate.DataStreamTemplate(),
+                null
+            );
+            state = service.addIndexTemplateV2(state, true, "my-template-4", it);
+
+            DataLifecycle resolvedLifecycle = MetadataIndexTemplateService.resolveLifecycle(state.metadata(), "my-template-4");
+
+            // The index template lifecycle does not have a retention so the one from ct_1 remains unchanged.
+            assertThat(resolvedLifecycle, equalTo(dataLifecycle1));
+        }
     }
 
     public void testAddInvalidTemplate() throws Exception {
