Add code to skip ignored indices in the DS deprecation checks

Author: lukewhiting

x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DataStreamDeprecationChecks.java

@@ -10,6 +10,7 @@
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.DataStream;
 import org.elasticsearch.index.Index;
+import org.elasticsearch.index.IndexVersion;
 import org.elasticsearch.index.IndexVersions;
 import org.elasticsearch.xpack.core.deprecation.DeprecationIssue;
 
@@ -19,13 +20,24 @@
 
 import static java.util.Map.entry;
 import static java.util.Map.ofEntries;
+import static org.elasticsearch.cluster.metadata.IndexMetadata.INDEX_IGNORE_DEPRECATION_WARNING_FOR_VERSION_KEY;
 
 public class DataStreamDeprecationChecks {
+
+    static final IndexVersion MINIMUM_VERSION = IndexVersions.UPGRADE_TO_LUCENE_10_0_0;
+
     static DeprecationIssue oldIndicesCheck(DataStream dataStream, ClusterState clusterState) {
         List<Index> backingIndices = dataStream.getIndices();
 
         Set<String> indicesNeedingUpgrade = backingIndices.stream()
-            .filter(index -> clusterState.metadata().index(index).getCreationVersion().onOrBefore(IndexVersions.UPGRADE_TO_LUCENE_10_0_0))
+            .filter(index -> clusterState.metadata().index(index).getCreationVersion().before(MINIMUM_VERSION))
+            .filter(index -> {
+                IndexVersion ignoreVersion = clusterState.metadata()
+                    .index(index)
+                    .getSettings()
+                    .getAsVersionId(INDEX_IGNORE_DEPRECATION_WARNING_FOR_VERSION_KEY, IndexVersion::fromId);
+                return ignoreVersion == null || MINIMUM_VERSION.after(ignoreVersion);
+            })
             .map(Index::getName)
             .collect(Collectors.toUnmodifiableSet());
 
@@ -38,6 +50,7 @@ static DeprecationIssue oldIndicesCheck(DataStream dataStream, ClusterState clus
                 false,
                 ofEntries(
                     entry("reindex_required", true),
+                    entry("minimum_version_id_required", MINIMUM_VERSION.id()),
                     entry("total_backing_indices", backingIndices.size()),
                     entry("indices_requiring_upgrade_count", indicesNeedingUpgrade.size()),
                     entry("indices_requiring_upgrade", indicesNeedingUpgrade)

x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/IndexDeprecationChecks.java

@@ -28,16 +28,22 @@
 import java.util.function.BiFunction;
 import java.util.function.Function;
 
+import static org.elasticsearch.cluster.metadata.IndexMetadata.INDEX_IGNORE_DEPRECATION_WARNING_FOR_VERSION_KEY;
+
 /**
  * Index-specific deprecation checks
  */
 public class IndexDeprecationChecks {
 
+    private static final IndexVersion MINIMUM_VERSION = IndexVersions.V_8_0_0;
+
     static DeprecationIssue oldIndicesCheck(IndexMetadata indexMetadata, ClusterState clusterState) {
         // TODO: this check needs to be revised. It's trivially true right now.
         IndexVersion currentCompatibilityVersion = indexMetadata.getCompatibilityVersion();
         // We intentionally exclude indices that are in data streams because they will be picked up by DataStreamDeprecationChecks
-        if (currentCompatibilityVersion.before(IndexVersions.V_8_0_0) && isNotDataStreamIndex(indexMetadata, clusterState)) {
+        if (currentCompatibilityVersion.before(MINIMUM_VERSION)
+            && isNotDataStreamIndex(indexMetadata, clusterState)
+            && notIgnored(indexMetadata)) {
             return new DeprecationIssue(
                 DeprecationIssue.Level.CRITICAL,
                 "Old index with a compatibility version < 8.0",
@@ -50,6 +56,12 @@ static DeprecationIssue oldIndicesCheck(IndexMetadata indexMetadata, ClusterStat
         return null;
     }
 
+    private static boolean notIgnored(IndexMetadata indexMetadata) {
+        IndexVersion ignoreVersion = indexMetadata.getSettings()
+            .getAsVersionId(INDEX_IGNORE_DEPRECATION_WARNING_FOR_VERSION_KEY, IndexVersion::fromId);
+        return ignoreVersion == null || IndexVersions.V_8_0_0.after(ignoreVersion);
+    }
+
     private static boolean isNotDataStreamIndex(IndexMetadata indexMetadata, ClusterState clusterState) {
         return clusterState.metadata().findDataStreams(indexMetadata.getIndex().getName()).isEmpty();
     }

x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/DataStreamDeprecationChecksTests.java

@@ -22,13 +22,15 @@
 
 import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
-import java.util.stream.Collectors;
+import java.util.Set;
 
 import static java.util.Collections.singletonList;
 import static java.util.Map.entry;
 import static java.util.Map.ofEntries;
+import static org.elasticsearch.xpack.deprecation.DataStreamDeprecationChecks.MINIMUM_VERSION;
 import static org.elasticsearch.xpack.deprecation.DeprecationChecks.DATA_STREAM_CHECKS;
 import static org.hamcrest.Matchers.equalTo;
 
@@ -41,14 +43,26 @@ public void testOldIndicesCheck() {
         List<Index> allIndices = new ArrayList<>();
         Map<String, IndexMetadata> nameToIndexMetadata = new HashMap<>();
 
+        Set<String> indicesToUpgrade = new HashSet<>();
+
         for (int i = 0; i < oldIndexCount; i++) {
             Settings.Builder settingsBuilder = settings(IndexVersion.fromId(7170099));
+
+            // Ensure at least 1 index does not have the ignore setting, otherwise 1 in 3 should be ignored
+            boolean shouldIgnore = indicesToUpgrade.isEmpty() == false || randomIntBetween(0, 2) == 0;
+            if (shouldIgnore) {
+                settingsBuilder.put(IndexMetadata.INDEX_IGNORE_DEPRECATION_WARNING_FOR_VERSION_KEY, MINIMUM_VERSION.id());
+            }
+
             IndexMetadata oldIndexMetadata = IndexMetadata.builder("old-data-stream-index-" + i)
                 .settings(settingsBuilder)
                 .numberOfShards(1)
                 .numberOfReplicas(0)
                 .build();
             allIndices.add(oldIndexMetadata.getIndex());
+            if (shouldIgnore == false) {
+                indicesToUpgrade.add(oldIndexMetadata.getIndex().getName());
+            }
             nameToIndexMetadata.put(oldIndexMetadata.getIndex().getName(), oldIndexMetadata);
         }
 
@@ -90,16 +104,11 @@ public void testOldIndicesCheck() {
             "This data stream has backing indices that were created before Elasticsearch 8.0.0",
             false,
             ofEntries(
+                entry("minimum_version_id_required", MINIMUM_VERSION.id()),
                 entry("reindex_required", true),
                 entry("total_backing_indices", oldIndexCount + newIndexCount),
-                entry("indices_requiring_upgrade_count", oldIndexCount),
-                entry(
-                    "indices_requiring_upgrade",
-                    nameToIndexMetadata.keySet()
-                        .stream()
-                        .filter(name -> name.startsWith("old-data-stream-index-"))
-                        .collect(Collectors.toUnmodifiableSet())
-                )
+                entry("indices_requiring_upgrade_count", indicesToUpgrade.size()),
+                entry("indices_requiring_upgrade", indicesToUpgrade)
             )
         );