[8.x] Allow archive and searchable snapshots indices in N-2 version (#118923)

This is the backport of #118941 for 8.18.

This change relaxes the index compatibility version checks 
to allow archive and searchable snapshot indices in version 
N-2 to exist on a 9.x cluster. It uses the min. read-only index 
compatible version added in #118884 to accept join requests 
from 9.x nodes that can read indices created in version 7.x
 (N-2) as long as they have a write block and are either archive 
or searchable snapshot indices.

Relates ES-10274

Author: tlrx

server/src/main/java/org/elasticsearch/cluster/coordination/NodeJoinExecutor.java

@@ -49,6 +49,7 @@
 import java.util.function.Function;
 import java.util.stream.Collectors;
 
+import static org.elasticsearch.cluster.metadata.IndexMetadataVerifier.isReadOnlySupportedVersion;
 import static org.elasticsearch.gateway.GatewayService.STATE_NOT_RECOVERED_BLOCK;
 
 public class NodeJoinExecutor implements ClusterStateTaskExecutor<JoinTask> {
@@ -179,8 +180,12 @@ public ClusterState execute(BatchExecutionContext<JoinTask> batchExecutionContex
                         Set<String> newNodeEffectiveFeatures = enforceNodeFeatureBarrier(node, effectiveClusterFeatures, features);
                         // we do this validation quite late to prevent race conditions between nodes joining and importing dangling indices
                         // we have to reject nodes that don't support all indices we have in this cluster
-                        ensureIndexCompatibility(node.getMinIndexVersion(), node.getMaxIndexVersion(), initialState.getMetadata());
-
+                        ensureIndexCompatibility(
+                            node.getMinIndexVersion(),
+                            node.getMinReadOnlyIndexVersion(),
+                            node.getMaxIndexVersion(),
+                            initialState.getMetadata()
+                        );
                         nodesBuilder.add(node);
                         compatibilityVersionsMap.put(node.getId(), compatibilityVersions);
                         // store the actual node features here, not including assumed features, as this is persisted in cluster state
@@ -394,9 +399,15 @@ private Set<String> calculateEffectiveClusterFeatures(DiscoveryNodes nodes, Map<
      * will not be created with a newer version of elasticsearch as well as that all indices are newer or equal to the minimum index
      * compatibility version.
      * @see IndexVersions#MINIMUM_COMPATIBLE
+     * @see IndexVersions#MINIMUM_READONLY_COMPATIBLE
      * @throws IllegalStateException if any index is incompatible with the given version
      */
-    public static void ensureIndexCompatibility(IndexVersion minSupportedVersion, IndexVersion maxSupportedVersion, Metadata metadata) {
+    public static void ensureIndexCompatibility(
+        IndexVersion minSupportedVersion,
+        IndexVersion minReadOnlySupportedVersion,
+        IndexVersion maxSupportedVersion,
+        Metadata metadata
+    ) {
         // we ensure that all indices in the cluster we join are compatible with us no matter if they are
         // closed or not we can't read mappings of these indices so we need to reject the join...
         for (IndexMetadata idxMetadata : metadata) {
@@ -411,14 +422,17 @@ public static void ensureIndexCompatibility(IndexVersion minSupportedVersion, In
                 );
             }
             if (idxMetadata.getCompatibilityVersion().before(minSupportedVersion)) {
-                throw new IllegalStateException(
-                    "index "
-                        + idxMetadata.getIndex()
-                        + " version not supported: "
-                        + idxMetadata.getCompatibilityVersion().toReleaseVersion()
-                        + " minimum compatible index version is: "
-                        + minSupportedVersion.toReleaseVersion()
-                );
+                boolean isReadOnlySupported = isReadOnlySupportedVersion(idxMetadata, minSupportedVersion, minReadOnlySupportedVersion);
+                if (isReadOnlySupported == false) {
+                    throw new IllegalStateException(
+                        "index "
+                            + idxMetadata.getIndex()
+                            + " version not supported: "
+                            + idxMetadata.getCompatibilityVersion().toReleaseVersion()
+                            + " minimum compatible index version is: "
+                            + minSupportedVersion.toReleaseVersion()
+                    );
+                }
             }
         }
     }
@@ -542,7 +556,12 @@ public static Collection<BiConsumer<DiscoveryNode, ClusterState>> addBuiltInJoin
         final Collection<BiConsumer<DiscoveryNode, ClusterState>> validators = new ArrayList<>();
         validators.add((node, state) -> {
             ensureNodesCompatibility(node.getVersion(), state.getNodes());
-            ensureIndexCompatibility(node.getMinIndexVersion(), node.getMaxIndexVersion(), state.getMetadata());
+            ensureIndexCompatibility(
+                node.getMinIndexVersion(),
+                node.getMinReadOnlyIndexVersion(),
+                node.getMaxIndexVersion(),
+                state.getMetadata()
+            );
         });
         validators.addAll(onJoinValidators);
         return Collections.unmodifiableCollection(validators);

server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadataVerifier.java

@@ -88,8 +88,12 @@ public IndexMetadataVerifier(
      * If the index does not need upgrade it returns the index metadata unchanged, otherwise it returns a modified index metadata. If index
      * cannot be updated the method throws an exception.
      */
-    public IndexMetadata verifyIndexMetadata(IndexMetadata indexMetadata, IndexVersion minimumIndexCompatibilityVersion) {
-        checkSupportedVersion(indexMetadata, minimumIndexCompatibilityVersion);
+    public IndexMetadata verifyIndexMetadata(
+        IndexMetadata indexMetadata,
+        IndexVersion minimumIndexCompatibilityVersion,
+        IndexVersion minimumReadOnlyIndexCompatibilityVersion
+    ) {
+        checkSupportedVersion(indexMetadata, minimumIndexCompatibilityVersion, minimumReadOnlyIndexCompatibilityVersion);
 
         // First convert any shared_cache searchable snapshot indices to only use _tier_preference: data_frozen
         IndexMetadata newMetadata = convertSharedCacheTierPreference(indexMetadata);
@@ -105,26 +109,81 @@ public IndexMetadata verifyIndexMetadata(IndexMetadata indexMetadata, IndexVersi
     }
 
     /**
-     * Check that the index version is compatible. Elasticsearch does not support indices created before the
-     * previous major version.
+     * Check that the index version is compatible. Elasticsearch supports reading and writing indices created in the current version ("N")
+     + as well as the previous major version ("N-1"). Elasticsearch only supports reading indices created down to the penultimate version
+     + ("N-2") and does not support reading nor writing any version below that.
      */
-    private static void checkSupportedVersion(IndexMetadata indexMetadata, IndexVersion minimumIndexCompatibilityVersion) {
-        boolean isSupportedVersion = indexMetadata.getCompatibilityVersion().onOrAfter(minimumIndexCompatibilityVersion);
-        if (isSupportedVersion == false) {
-            throw new IllegalStateException(
-                "The index "
-                    + indexMetadata.getIndex()
-                    + " has current compatibility version ["
-                    + indexMetadata.getCompatibilityVersion().toReleaseVersion()
-                    + "] but the minimum compatible version is ["
-                    + minimumIndexCompatibilityVersion.toReleaseVersion()
-                    + "]. It should be re-indexed in Elasticsearch "
-                    + (Version.CURRENT.major - 1)
-                    + ".x before upgrading to "
-                    + Build.current().version()
-                    + "."
-            );
+    private static void checkSupportedVersion(
+        IndexMetadata indexMetadata,
+        IndexVersion minimumIndexCompatibilityVersion,
+        IndexVersion minimumReadOnlyIndexCompatibilityVersion
+    ) {
+        if (isFullySupportedVersion(indexMetadata, minimumIndexCompatibilityVersion)) {
+            return;
+        }
+        if (isReadOnlySupportedVersion(indexMetadata, minimumIndexCompatibilityVersion, minimumReadOnlyIndexCompatibilityVersion)) {
+            return;
+        }
+        throw new IllegalStateException(
+            "The index "
+                + indexMetadata.getIndex()
+                + " has current compatibility version ["
+                + indexMetadata.getCompatibilityVersion().toReleaseVersion()
+                + "] but the minimum compatible version is ["
+                + minimumIndexCompatibilityVersion.toReleaseVersion()
+                + "]. It should be re-indexed in Elasticsearch "
+                + (Version.CURRENT.major - 1)
+                + ".x before upgrading to "
+                + Build.current().version()
+                + "."
+        );
+    }
+
+    private static boolean isFullySupportedVersion(IndexMetadata indexMetadata, IndexVersion minimumIndexCompatibilityVersion) {
+        return indexMetadata.getCompatibilityVersion().onOrAfter(minimumIndexCompatibilityVersion);
+    }
+
+    /**
+     * Returns {@code true} if the index version is compatible in read-only mode. As of today, only searchable snapshots and archive indices
+     * in version N-2 with a write block are read-only compatible. This method throws an {@link IllegalStateException} if the index is
+     * either a searchable snapshot or an archive index with a read-only compatible version but is missing the write block.
+     *
+     * @param indexMetadata                         the index metadata
+     * @param minimumIndexCompatibilityVersion      the min. index compatible version for reading and writing indices (used in assertion)
+     * @param minReadOnlyIndexCompatibilityVersion  the min. index compatible version for only reading indices
+     *
+     * @return {@code true} if the index version is compatible in read-only mode, {@code false} otherwise.
+     * @throws IllegalStateException if the index is read-only compatible but has no write block in place.
+     */
+    public static boolean isReadOnlySupportedVersion(
+        IndexMetadata indexMetadata,
+        IndexVersion minimumIndexCompatibilityVersion,
+        IndexVersion minReadOnlyIndexCompatibilityVersion
+    ) {
+        boolean isReadOnlySupportedVersion = indexMetadata.getCompatibilityVersion().onOrAfter(minReadOnlyIndexCompatibilityVersion);
+        assert isFullySupportedVersion(indexMetadata, minimumIndexCompatibilityVersion) == false;
+
+        if (isReadOnlySupportedVersion
+            && (indexMetadata.isSearchableSnapshot() || indexMetadata.getCreationVersion().isLegacyIndexVersion())) {
+            boolean isReadOnly = IndexMetadata.INDEX_BLOCKS_WRITE_SETTING.get(indexMetadata.getSettings());
+            if (isReadOnly == false) {
+                throw new IllegalStateException(
+                    "The index "
+                        + indexMetadata.getIndex()
+                        + " created in version ["
+                        + indexMetadata.getCreationVersion()
+                        + "] with current compatibility version ["
+                        + indexMetadata.getCompatibilityVersion().toReleaseVersion()
+                        + "] must be marked as read-only using the setting ["
+                        + IndexMetadata.SETTING_BLOCKS_WRITE
+                        + "] set to [true] before upgrading to "
+                        + Build.current().version()
+                        + '.'
+                );
+            }
+            return true;
         }
+        return false;
     }
 
     /**

server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexStateService.java

@@ -1120,6 +1120,7 @@ private ClusterState openIndices(final Index[] indices, final ClusterState curre
             final Metadata.Builder metadata = Metadata.builder(currentState.metadata());
             final ClusterBlocks.Builder blocks = ClusterBlocks.builder(currentState.blocks());
             final IndexVersion minIndexCompatibilityVersion = currentState.getNodes().getMinSupportedIndexVersion();
+            final IndexVersion minReadOnlyIndexCompatibilityVersion = currentState.getNodes().getMinReadOnlySupportedIndexVersion();
 
             for (IndexMetadata indexMetadata : indicesToOpen) {
                 final Index index = indexMetadata.getIndex();
@@ -1137,7 +1138,11 @@ private ClusterState openIndices(final Index[] indices, final ClusterState curre
 
                     // The index might be closed because we couldn't import it due to an old incompatible
                     // version, so we need to verify its compatibility.
-                    newIndexMetadata = indexMetadataVerifier.verifyIndexMetadata(newIndexMetadata, minIndexCompatibilityVersion);
+                    newIndexMetadata = indexMetadataVerifier.verifyIndexMetadata(
+                        newIndexMetadata,
+                        minIndexCompatibilityVersion,
+                        minReadOnlyIndexCompatibilityVersion
+                    );
                     try {
                         indicesService.verifyIndexMetadata(newIndexMetadata, newIndexMetadata);
                     } catch (Exception e) {

server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java

@@ -308,7 +308,11 @@ static Metadata upgradeMetadata(Metadata metadata, IndexMetadataVerifier indexMe
         boolean changed = false;
         final Metadata.Builder upgradedMetadata = Metadata.builder(metadata);
         for (IndexMetadata indexMetadata : metadata) {
-            IndexMetadata newMetadata = indexMetadataVerifier.verifyIndexMetadata(indexMetadata, IndexVersions.MINIMUM_COMPATIBLE);
+            IndexMetadata newMetadata = indexMetadataVerifier.verifyIndexMetadata(
+                indexMetadata,
+                IndexVersions.MINIMUM_COMPATIBLE,
+                IndexVersions.MINIMUM_READONLY_COMPATIBLE
+            );
             changed |= indexMetadata != newMetadata;
             upgradedMetadata.put(newMetadata, false);
         }

server/src/main/java/org/elasticsearch/gateway/LocalAllocateDangledIndices.java

@@ -125,6 +125,7 @@ public ClusterState execute(ClusterState currentState) {
                         currentState.routingTable()
                     );
                     IndexVersion minIndexCompatibilityVersion = currentState.nodes().getMinSupportedIndexVersion();
+                    IndexVersion minReadOnlyIndexCompatibilityVersion = currentState.nodes().getMinReadOnlySupportedIndexVersion();
                     IndexVersion maxIndexCompatibilityVersion = currentState.nodes().getMaxDataNodeCompatibleIndexVersion();
                     boolean importNeeded = false;
                     StringBuilder sb = new StringBuilder();
@@ -176,7 +177,11 @@ public ClusterState execute(ClusterState currentState) {
                         try {
                             // The dangled index might be from an older version, we need to make sure it's compatible
                             // with the current version.
-                            newIndexMetadata = indexMetadataVerifier.verifyIndexMetadata(indexMetadata, minIndexCompatibilityVersion);
+                            newIndexMetadata = indexMetadataVerifier.verifyIndexMetadata(
+                                indexMetadata,
+                                minIndexCompatibilityVersion,
+                                minReadOnlyIndexCompatibilityVersion
+                            );
                             newIndexMetadata = IndexMetadata.builder(newIndexMetadata)
                                 .settings(
                                     Settings.builder()

server/src/main/java/org/elasticsearch/index/IndexVersion.java

@@ -124,7 +124,7 @@ public static IndexVersion current() {
     }
 
     public boolean isLegacyIndexVersion() {
-        return before(IndexVersions.MINIMUM_COMPATIBLE);
+        return before(IndexVersions.MINIMUM_READONLY_COMPATIBLE);
     }
 
     public static IndexVersion getMinimumCompatibleIndexVersion(int versionId) {

server/src/main/java/org/elasticsearch/snapshots/RestoreService.java

@@ -1315,6 +1315,7 @@ public ClusterState execute(ClusterState currentState) {
             final Map<ShardId, ShardRestoreStatus> shards = new HashMap<>();
 
             final IndexVersion minIndexCompatibilityVersion = currentState.getNodes().getMinSupportedIndexVersion();
+            final IndexVersion minReadOnlyIndexCompatibilityVersion = currentState.getNodes().getMinReadOnlySupportedIndexVersion();
             final String localNodeId = clusterService.state().nodes().getLocalNodeId();
             for (Map.Entry<String, IndexId> indexEntry : indicesToRestore.entrySet()) {
                 final IndexId index = indexEntry.getValue();
@@ -1327,12 +1328,16 @@ public ClusterState execute(ClusterState currentState) {
                     request.indexSettings(),
                     request.ignoreIndexSettings()
                 );
-                if (snapshotIndexMetadata.getCompatibilityVersion().before(minIndexCompatibilityVersion)) {
+                if (snapshotIndexMetadata.getCompatibilityVersion().isLegacyIndexVersion()) {
                     // adapt index metadata so that it can be understood by current version
                     snapshotIndexMetadata = convertLegacyIndex(snapshotIndexMetadata, currentState, indicesService);
                 }
                 try {
-                    snapshotIndexMetadata = indexMetadataVerifier.verifyIndexMetadata(snapshotIndexMetadata, minIndexCompatibilityVersion);
+                    snapshotIndexMetadata = indexMetadataVerifier.verifyIndexMetadata(
+                        snapshotIndexMetadata,
+                        minIndexCompatibilityVersion,
+                        minReadOnlyIndexCompatibilityVersion
+                    );
                 } catch (Exception ex) {
                     throw new SnapshotRestoreException(snapshot, "cannot restore index [" + index + "] because it cannot be upgraded", ex);
                 }