Introduce the index.downsample.origin.name and index.downsample.origin.uuid settings (#99061) (#99182)

This proposes introducing two new settings we configure when
downsampling: `index.downsample.origin.name`
`index.downsample.origin.uuid`

These settings will carry on the name and uuid of the first source index
we downsample (like the source.name and source.uuid settings behaved
before this PR).

This also changes the behaviour of `index.downsample.source.name` and
`index.downsample.source.uuid` to always reflect the source of the
current downsampling round.

If an index is downsampled once, both the source and origin settings
will have the same value.

However, for subsequent downsampling operations, a later downsampling
round will have the actual index name and uuid that were used as the
source of the downsampling operation configured  e.g.
`downsample-350s-.ds-metrics-foo-2023.08.17-000001` will have the
`index.downsample.source.name` configured to
`downsample-1s-.ds-metrics-foo-2023.08.17-000001` and
`index.downsample.origin.name` configured to
`.ds-metrics-foo-2023.08.17-000001`.

Note that this will help us simplify things in data stream lifecycle as
with subsequent  downsampling configuration we currently have to store
the true source of the downsampling operation in the downsample index
metadata (
https://github.com/elastic/elasticsearch/blob/main/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/downsampling/ReplaceSourceWithDownsampleIndexTask.java#L173
)

If you agree with this proposal we'll have to manually backport to 8.10
as well to make sure the `generateDownsampleIndexName` method is
consistent across releases
https://github.com/elastic/elasticsearch/blob/main/server/src/main/java/org/elasticsearch/action/downsample/DownsampleConfig.java#L249

Note that the `index.provided_name` setting which is set on indices when
rolling over cannot act like a  source (initially I thought we can use
that instead of introducing a new setting, the origin one) because  it
contains date-math patterns.

(cherry picked from commit 6c3a464)
Signed-off-by: Andrei Dan <[email protected]>

Author: andreidan

server/src/main/java/org/elasticsearch/action/downsample/DownsampleConfig.java

@@ -242,7 +242,7 @@ public static Rounding.Prepared createRounding(final String expr, final String t
      * prefix-fixedInterval-baseIndexName
      *
      * Note that this looks for the base index name of the provided index metadata via the
-     * {@link IndexMetadata#INDEX_DOWNSAMPLE_SOURCE_NAME_KEY} setting. This means that in case
+     * {@link IndexMetadata#INDEX_DOWNSAMPLE_ORIGIN_NAME_KEY} setting. This means that in case
      * the provided index was already downsampled, we'll use the original source index (of the
      * current provided downsample index) as the base index name.
      */
@@ -251,10 +251,13 @@ public static String generateDownsampleIndexName(
         IndexMetadata sourceIndexMetadata,
         DateHistogramInterval fixedInterval
     ) {
-        String downsampleSourceName = sourceIndexMetadata.getSettings().get(IndexMetadata.INDEX_DOWNSAMPLE_SOURCE_NAME_KEY);
+        String downsampleOriginName = IndexMetadata.INDEX_DOWNSAMPLE_ORIGIN_NAME.get(sourceIndexMetadata.getSettings());
         String sourceIndexName;
-        if (downsampleSourceName != null) {
-            sourceIndexName = downsampleSourceName;
+        if (Strings.hasText(downsampleOriginName)) {
+            sourceIndexName = downsampleOriginName;
+        } else if (Strings.hasText(IndexMetadata.INDEX_DOWNSAMPLE_SOURCE_NAME.get(sourceIndexMetadata.getSettings()))) {
+            // bwc for downsample indices created pre 8.10 which didn't configure the origin
+            sourceIndexName = IndexMetadata.INDEX_DOWNSAMPLE_SOURCE_NAME.get(sourceIndexMetadata.getSettings());
         } else {
             sourceIndexName = sourceIndexMetadata.getIndex().getName();
         }

server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java

@@ -1227,6 +1227,8 @@ public Index getResizeSourceIndex() {
 
     public static final String INDEX_DOWNSAMPLE_SOURCE_UUID_KEY = "index.downsample.source.uuid";
     public static final String INDEX_DOWNSAMPLE_SOURCE_NAME_KEY = "index.downsample.source.name";
+    public static final String INDEX_DOWNSAMPLE_ORIGIN_NAME_KEY = "index.downsample.origin.name";
+    public static final String INDEX_DOWNSAMPLE_ORIGIN_UUID_KEY = "index.downsample.origin.uuid";
 
     public static final String INDEX_DOWNSAMPLE_STATUS_KEY = "index.downsample.status";
     public static final Setting<String> INDEX_DOWNSAMPLE_SOURCE_UUID = Setting.simpleString(
@@ -1240,6 +1242,18 @@ public Index getResizeSourceIndex() {
         Property.PrivateIndex
     );
 
+    public static final Setting<String> INDEX_DOWNSAMPLE_ORIGIN_NAME = Setting.simpleString(
+        INDEX_DOWNSAMPLE_ORIGIN_NAME_KEY,
+        Property.IndexScope,
+        Property.PrivateIndex
+    );
+
+    public static final Setting<String> INDEX_DOWNSAMPLE_ORIGIN_UUID = Setting.simpleString(
+        INDEX_DOWNSAMPLE_ORIGIN_UUID_KEY,
+        Property.IndexScope,
+        Property.PrivateIndex
+    );
+
     public enum DownsampleTaskStatus {
         UNKNOWN,
         STARTED,

server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java

@@ -72,6 +72,8 @@ public final class IndexScopedSettings extends AbstractScopedSettings {
         IndexMetadata.INDEX_FORMAT_SETTING,
         IndexMetadata.INDEX_DOWNSAMPLE_SOURCE_NAME,
         IndexMetadata.INDEX_DOWNSAMPLE_SOURCE_UUID,
+        IndexMetadata.INDEX_DOWNSAMPLE_ORIGIN_NAME,
+        IndexMetadata.INDEX_DOWNSAMPLE_ORIGIN_UUID,
         IndexMetadata.INDEX_DOWNSAMPLE_STATUS,
         SearchSlowLog.INDEX_SEARCH_SLOWLOG_THRESHOLD_FETCH_DEBUG_SETTING,
         SearchSlowLog.INDEX_SEARCH_SLOWLOG_THRESHOLD_FETCH_WARN_SETTING,

server/src/test/java/org/elasticsearch/action/downsample/DonwsampleConfigTests.java

@@ -28,7 +28,19 @@ public void testGenerateDownsampleIndexName() {
         {
             String downsampledIndex = "downsample-1h-test";
             IndexMetadata indexMeta = IndexMetadata.builder(downsampledIndex)
-                .settings(indexSettings(IndexVersion.current(), 1, 0).put(IndexMetadata.INDEX_DOWNSAMPLE_SOURCE_NAME_KEY, "test"))
+                .settings(indexSettings(IndexVersion.current(), 1, 0).put(IndexMetadata.INDEX_DOWNSAMPLE_ORIGIN_NAME_KEY, "test"))
+                .build();
+            assertThat(generateDownsampleIndexName("downsample-", indexMeta, new DateHistogramInterval("8h")), is("downsample-8h-test"));
+        }
+
+        {
+            // test origin takes higher precedence than the configured source setting
+            String downsampledIndex = "downsample-1h-test";
+            IndexMetadata indexMeta = IndexMetadata.builder(downsampledIndex)
+                .settings(
+                    indexSettings(IndexVersion.current(), 1, 0).put(IndexMetadata.INDEX_DOWNSAMPLE_ORIGIN_NAME_KEY, "test")
+                        .put(IndexMetadata.INDEX_DOWNSAMPLE_SOURCE_NAME_KEY, "downsample-1s-test")
+                )
                 .build();
             assertThat(generateDownsampleIndexName("downsample-", indexMeta, new DateHistogramInterval("8h")), is("downsample-8h-test"));
         }

x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/TransportDownsampleAction.java

@@ -680,17 +680,19 @@ static IndexMetadata.Builder copyIndexMetadata(
         }
 
         /*
-         * Add the source index name and UUID to the downsample index metadata.
-         * If the source index is a downsample index, we will add the name and UUID
+         * Add the origin index name and UUID to the downsample index metadata.
+         * If the origin index is a downsample index, we will add the name and UUID
          * of the first index that we initially rolled up.
          */
-        if (IndexMetadata.INDEX_DOWNSAMPLE_SOURCE_UUID.exists(sourceIndexMetadata.getSettings()) == false
-            || IndexMetadata.INDEX_DOWNSAMPLE_SOURCE_NAME.exists(sourceIndexMetadata.getSettings()) == false) {
-            Index sourceIndex = sourceIndexMetadata.getIndex();
-            targetSettings.put(IndexMetadata.INDEX_DOWNSAMPLE_SOURCE_NAME.getKey(), sourceIndex.getName())
-                .put(IndexMetadata.INDEX_DOWNSAMPLE_SOURCE_UUID.getKey(), sourceIndex.getUUID());
+        Index sourceIndex = sourceIndexMetadata.getIndex();
+        if (IndexMetadata.INDEX_DOWNSAMPLE_ORIGIN_UUID.exists(sourceIndexMetadata.getSettings()) == false
+            || IndexMetadata.INDEX_DOWNSAMPLE_ORIGIN_NAME.exists(sourceIndexMetadata.getSettings()) == false) {
+            targetSettings.put(IndexMetadata.INDEX_DOWNSAMPLE_ORIGIN_NAME.getKey(), sourceIndex.getName())
+                .put(IndexMetadata.INDEX_DOWNSAMPLE_ORIGIN_UUID.getKey(), sourceIndex.getUUID());
         }
 
+        targetSettings.put(IndexMetadata.INDEX_DOWNSAMPLE_SOURCE_NAME_KEY, sourceIndex.getName());
+        targetSettings.put(IndexMetadata.INDEX_DOWNSAMPLE_SOURCE_UUID_KEY, sourceIndex.getUUID());
         return IndexMetadata.builder(downsampleIndexMetadata).settings(targetSettings);
     }
 

x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/DownsampleActionSingleNodeTests.java

@@ -33,6 +33,7 @@
 import org.elasticsearch.cluster.metadata.ComposableIndexTemplate;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.Template;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.document.DocumentField;
 import org.elasticsearch.common.network.NetworkAddress;
@@ -339,7 +340,7 @@ public void testDownsampleOfDownsample() throws IOException {
         String downsampleIndex2 = downsampleIndex + "-2";
         DownsampleConfig config2 = new DownsampleConfig(DateHistogramInterval.minutes(intervalMinutes * randomIntBetween(2, 50)));
         downsample(downsampleIndex, downsampleIndex2, config2);
-        assertDownsampleIndex(sourceIndex, downsampleIndex2, config2);
+        assertDownsampleIndex(downsampleIndex, downsampleIndex2, config2);
     }
 
     private Date randomDate() {
@@ -1278,6 +1279,18 @@ private void assertDownsampleIndexSettings(String sourceIndex, String downsample
         assertEquals(sourceIndex, downsampleSettings.get(IndexMetadata.INDEX_DOWNSAMPLE_SOURCE_NAME_KEY));
         assertEquals(sourceSettings.get(IndexSettings.MODE.getKey()), downsampleSettings.get(IndexSettings.MODE.getKey()));
 
+        if (Strings.hasText(IndexMetadata.INDEX_DOWNSAMPLE_SOURCE_NAME.get(sourceSettings))) {
+            // if the source is a downsample index itself, we're in the "downsample of downsample" test case and both indices should have
+            // the same ORIGIN configured
+            assertEquals(
+                IndexMetadata.INDEX_DOWNSAMPLE_ORIGIN_NAME.get(sourceSettings),
+                IndexMetadata.INDEX_DOWNSAMPLE_ORIGIN_NAME.get(downsampleSettings)
+            );
+            assertEquals(
+                IndexMetadata.INDEX_DOWNSAMPLE_ORIGIN_UUID.get(sourceSettings),
+                IndexMetadata.INDEX_DOWNSAMPLE_ORIGIN_UUID.get(downsampleSettings)
+            );
+        }
         assertNotNull(sourceSettings.get(IndexSettings.TIME_SERIES_START_TIME.getKey()));
         assertNotNull(downsampleSettings.get(IndexSettings.TIME_SERIES_START_TIME.getKey()));
         assertEquals(

x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/DownsampleActionIT.java

@@ -373,7 +373,8 @@ public void testDownsampleTwice() throws Exception {
                 assertThat(indexExists(downsampleIndexName), is(false));
 
                 Map<String, Object> settings = getOnlyIndexSettings(client(), downsampleOfDownsampleIndexName);
-                assertEquals(firstBackingIndex, settings.get(IndexMetadata.INDEX_DOWNSAMPLE_SOURCE_NAME.getKey()));
+                assertEquals(firstBackingIndex, settings.get(IndexMetadata.INDEX_DOWNSAMPLE_ORIGIN_NAME.getKey()));
+                assertEquals(downsampleIndexName, settings.get(IndexMetadata.INDEX_DOWNSAMPLE_SOURCE_NAME.getKey()));
                 assertEquals(DownsampleTaskStatus.SUCCESS.toString(), settings.get(IndexMetadata.INDEX_DOWNSAMPLE_STATUS.getKey()));
                 assertEquals(policy, settings.get(LifecycleSettings.LIFECYCLE_NAME_SETTING.getKey()));
             }, 60, TimeUnit.SECONDS);