Data tiers: migrate the coldest node attribute (#81940) (#82247)

This fixes the migrate to data tiers routing API to take into account
the scenario where the node attribute configuration for an index is more
accurate than the existing `_tier_preference` configuration.

Previously we would simply remove the node attributes routing if there
was a `_tier_preference` configured for the index.

With this commit, we'll look if either the `require.data` or
`include.data` custom routings are colder than the existing `_tier_preference`
configuration (ie. `cold` vs `data_warm,data_hot`) and update the tier
routing accordingly.

eg.
{
  index.routing.allocation.require.data: "warm",
  index.routing.allocation.include.data: "cold",
  index.routing.allocation.include._tier_preference: "data_hot"
}
will be migrated to:
{
  index.routing.allocation.include._tier_preference: "data_cold,data_warm,data_hot"
}

This also removes the existing invariant that had the `require.data`
configuration take precedence over a possible `include.data`
configuration, and will now migrate the coldest configuration to the
corresponding `_tier_preference`.

eg.
{
  index.routing.allocation.require.data: "warm",
  index.routing.allocation.include.data: "cold"
}
will be migrated to:
{
  index.routing.allocation.include._tier_preference: "data_cold,data_warm,data_hot"
}

Author: andreidan

docs/reference/data-management/migrate-index-allocation-filters.asciidoc

@@ -194,3 +194,34 @@ PUT my-index/_settings
 // TEST[continued]
 
 If an index is already in the cold phase, include the cold, warm, and hot tiers.
+
+For indices that have both the `_tier_preference` and `require.data` configured
+but the `_tier_preference` is outdated (ie. the node attribute configuration
+is "colder" than the configured `_tier_preference`), the migration needs to
+remove the `require.data` attribute and update the `_tier_preference` to reflect
+the correct tiering.
+
+eg. For an index with the following routing configuration:
+[source,JSON]
+----
+{
+  "index.routing.allocation.require.data": "warm",
+  "index.routing.allocation.include._tier_preference": "data_hot"
+}
+----
+
+The routing configuration should be fixed like so:
+[source,console]
+----
+PUT my-index/_settings
+{
+  "index.routing.allocation.require.data": null,
+  "index.routing.allocation.include._tier_preference": "data_warm,data_hot"
+}
+----
+// TEST[continued]
+
+This situation can occur in a system that defaults to data tiers when, e.g.,
+an ILM policy that uses node attributes is restored and transitions the managed
+indices from the hot phase into the warm phase. In this case the node attribute
+configuration indicates the correct tier where the index should be allocated.

server/src/main/java/org/elasticsearch/cluster/routing/allocation/DataTier.java

@@ -188,6 +188,37 @@ public static List<String> parseTierList(String tiers) {
         }
     }
 
+    /**
+     * Compares the provided tiers for coldness order (eg. warm is colder than hot).
+     *
+     * Similar to {@link java.util.Comparator#compare(Object, Object)} returns
+     *   -1 if tier1 is colder than tier2 (ie. compare("data_cold", "data_hot"))
+     *   0 if tier1 is as cold as tier2 (ie. tier1.equals(tier2) )
+     *   1 if tier1 is warmer than tier2 (ie. compare("data_hot", "data_cold"))
+     *
+     * The provided tiers parameters must be valid data tiers values (ie. {@link #ALL_DATA_TIERS}.
+     * NOTE: `data_content` is treated as "equal to data_hot" in the tiers hierarchy.
+     * If invalid tier names are passed the result is non-deterministic.
+     */
+    public static int compare(String tier1, String tier2) {
+        if (tier1.equals(DATA_CONTENT)) {
+            tier1 = DATA_HOT;
+        }
+        if (tier2.equals(DATA_CONTENT)) {
+            tier2 = DATA_HOT;
+        }
+        int indexOfTier1 = ORDERED_FROZEN_TO_HOT_TIERS.indexOf(tier1);
+        assert indexOfTier1 >= 0 : "expecting a valid tier to compare but got:" + tier1;
+        int indexOfTier2 = ORDERED_FROZEN_TO_HOT_TIERS.indexOf(tier2);
+        assert indexOfTier2 >= 0 : "expecting a valid tier to compare but got:" + tier2;
+
+        if (indexOfTier1 == indexOfTier2) {
+            return 0;
+        } else {
+            return indexOfTier1 < indexOfTier2 ? -1 : 1;
+        }
+    }
+
     /**
      * This setting provider injects the setting allocating all newly created indices with
      * {@code index.routing.allocation.include._tier_preference: "data_hot"} for a data stream index

server/src/test/java/org/elasticsearch/cluster/routing/allocation/DataTierTests.java

@@ -177,4 +177,16 @@ public void testDataTierSettingValidator() {
         expectThrows(IllegalArgumentException.class, () -> validator.validate(DATA_WARM + ", "));
         expectThrows(IllegalArgumentException.class, () -> validator.validate(DATA_WARM + ", " + DATA_HOT));
     }
+
+    public void testCompareDataTiers() {
+        assertThat(DataTier.compare("data_cold", "data_warm"), is(-1));
+        assertThat(DataTier.compare("data_cold", "data_cold"), is(0));
+        assertThat(DataTier.compare("data_warm", "data_cold"), is(1));
+        // data_content is treated as equal to data_hot
+        assertThat(DataTier.compare("data_warm", "data_content"), is(-1));
+        assertThat(DataTier.compare("data_content", "data_content"), is(0));
+        assertThat(DataTier.compare("data_content", "data_hot"), is(0));
+        assertThat(DataTier.compare("data_content", "data_warm"), is(1));
+
+    }
 }

x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/cluster/metadata/MetadataMigrateToDataTiersRoutingService.java

@@ -91,7 +91,7 @@ private MetadataMigrateToDataTiersRoutingService() {}
      *  index.routing.allocation.include.{nodeAttrName} setting (if present) to the corresponding data tier `_tier_preference` routing.
      *  We are only able to convert the `frozen`, `cold`, `warm`, or `hot` setting values to the `_tier_preference`. If other
      *  configuration values are present eg ("the_warm_nodes") the index will not be migrated.
-     *  If the require or include setting is successfully migrated to _tier_preference, the **other** routing settings for the
+     *  If the require or include setting is successfully migrated to _tier_preference, all the **other** routing settings for the
      *  provided attribute are also removed (if present).
      *  Eg. if we manage to migrate the `index.routing.allocation.require.data` setting, but the index also has configured
      *  `index.routing.allocation.include.data` and `index.routing.allocation.exclude.data`, the
@@ -109,6 +109,19 @@ private MetadataMigrateToDataTiersRoutingService() {}
      *        index.routing.allocation.include._tier_preference: "data_warm,data_hot"
      *    }
      *
+     * If both the `index.routing.allocation.require.data` and `index.routing.allocation.include.data` settings are configured to
+     * recognized values the coldest one will be converted to the corresponding `_tier_preference` configuration.
+     * Eg. the following configuration:
+     *    {
+     *      index.routing.allocation.require.data: "warm",
+     *      index.routing.allocation.include.data: "cold",
+     *      index.routing.allocation.exclude.data: "rack2,rack3"
+     *    }
+     *  will be migrated to:
+     *    {
+     *        index.routing.allocation.include._tier_preference: "data_cold,data_warm,data_hot"
+     *    }
+     *
      * If no @param nodeAttrName is provided "data" will be used.
      * If no @param indexTemplateToDelete is provided, no index templates will be deleted.
      *
@@ -452,18 +465,21 @@ static List<String> migrateIndices(Metadata.Builder mb, ClusterState currentStat
         String nodeAttrIndexExcludeRoutingSetting = INDEX_ROUTING_EXCLUDE_GROUP_SETTING.getKey() + nodeAttrName;
         for (ObjectObjectCursor<String, IndexMetadata> index : currentState.metadata().indices()) {
             IndexMetadata indexMetadata = index.value;
+            String indexName = indexMetadata.getIndex().getName();
             Settings currentSettings = indexMetadata.getSettings();
 
             boolean removeNodeAttrIndexRoutingSettings = true;
 
             // migrate using the `require` setting
-            Settings newSettings = maybeMigrateRoutingSettingToTierPreference(nodeAttrIndexRequireRoutingSetting, indexMetadata);
+            Settings newSettings = maybeMigrateRoutingSettingToTierPreference(
+                nodeAttrIndexRequireRoutingSetting,
+                currentSettings,
+                indexName
+            );
+            // we possibly migrated the `require` setting, but maybe that attribute was not the coldest configured.
+            // let's try to migrate the `include` setting as well
+            newSettings = maybeMigrateRoutingSettingToTierPreference(nodeAttrIndexIncludeRoutingSetting, newSettings, indexName);
 
-            if (newSettings.equals(currentSettings)) {
-                // migrating based on the `require` setting was not successful, so let's check if the index used the `include` routing
-                // setting to configure the allocations and try to migrate it
-                newSettings = maybeMigrateRoutingSettingToTierPreference(nodeAttrIndexIncludeRoutingSetting, indexMetadata);
-            }
             if (newSettings.equals(currentSettings)) {
                 removeNodeAttrIndexRoutingSettings = false;
                 // migrating based on the `include` setting was not successful,
@@ -493,25 +509,51 @@ static List<String> migrateIndices(Metadata.Builder mb, ClusterState currentStat
 
     /**
      * Attempts to migrate the value of the given attribute routing setting to the _tier_preference equivalent. The provided setting
-     * needs to be configured and have one of the supported values (hot, warm, cold, or frozen) in order for the migration to be preformed.
+     * needs to be configured and have one of the supported values (hot, warm, cold, or frozen) in order for the migration to be performed.
      * If the migration is successful the provided setting will be removed.
      *
      * If the migration is **not** executed the current index settings is returned, otherwise the updated settings are returned
      */
     private static Settings maybeMigrateRoutingSettingToTierPreference(
         String attributeBasedRoutingSettingName,
-        IndexMetadata indexMetadata
+        Settings currentIndexSettings,
+        String indexName
     ) {
-        Settings currentIndexSettings = indexMetadata.getSettings();
         if (currentIndexSettings.keySet().contains(attributeBasedRoutingSettingName) == false) {
             return currentIndexSettings;
         }
 
         Settings.Builder newSettingsBuilder = Settings.builder().put(currentIndexSettings);
-        String indexName = indexMetadata.getIndex().getName();
 
         // look at the value, get the correct tiers config and update the settings
         if (currentIndexSettings.keySet().contains(TIER_PREFERENCE)) {
+            String tierPreferenceConfiguration = currentIndexSettings.get(TIER_PREFERENCE);
+            List<String> tiersConfiguration = DataTier.parseTierList(tierPreferenceConfiguration);
+            if (tiersConfiguration.isEmpty() == false) {
+                String coldestConfiguredTier = tiersConfiguration.get(0);
+                String attributeValue = currentIndexSettings.get(attributeBasedRoutingSettingName);
+                String attributeTierEquivalent = "data_" + attributeValue;
+                if (DataTier.validTierName(attributeTierEquivalent)) {
+                    // if the attribute's tier equivalent would be colder than what is currently the coldest tier configured
+                    // in the _tier_preference setting, the configured attribute routing is more accurate so we'll update the
+                    // tier_preference to reflect this before removing the attribute routing setting.
+                    if (DataTier.compare(attributeTierEquivalent, coldestConfiguredTier) < 0) {
+                        String newTierPreferenceConfiguration = convertAttributeValueToTierPreference(attributeValue);
+                        if (newTierPreferenceConfiguration != null) {
+                            logger.debug(
+                                "index [{}]: updated the [{}] setting to [{}] as the attribute based routing setting [{}] had "
+                                    + "the value [{}]",
+                                indexName,
+                                TIER_PREFERENCE,
+                                newTierPreferenceConfiguration,
+                                attributeBasedRoutingSettingName,
+                                attributeValue
+                            );
+                            newSettingsBuilder.put(TIER_PREFERENCE, newTierPreferenceConfiguration);
+                        }
+                    }
+                }
+            }
             newSettingsBuilder.remove(attributeBasedRoutingSettingName);
             logger.debug("index [{}]: removed setting [{}]", indexName, attributeBasedRoutingSettingName);
         } else {