[8.x] Support recovery for closed shard in N-2 version (#120595)

Changes in 8.x to allow shard recovery for shards in version N-2 that have been verified before being closed, but not verified as read-only, in 7.x or 8.x.

Reopening such closed indices automatically adds an index.blocks.write.

Author: tlrx

server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadataVerifier.java

@@ -38,6 +38,7 @@
 import java.util.Map;
 import java.util.Set;
 
+import static org.elasticsearch.cluster.metadata.MetadataIndexStateService.isIndexVerifiedBeforeClosed;
 import static org.elasticsearch.core.Strings.format;
 
 /**
@@ -163,7 +164,7 @@ public static boolean isReadOnlySupportedVersion(
     ) {
         if (isReadOnlyCompatible(indexMetadata, minimumCompatible, minimumReadOnlyCompatible)) {
             assert isFullySupportedVersion(indexMetadata, minimumCompatible) == false : indexMetadata;
-            final boolean isReadOnly = hasIndexWritesBlock(indexMetadata);
+            final boolean isReadOnly = hasReadOnlyBlocks(indexMetadata) || isIndexVerifiedBeforeClosed(indexMetadata);
             if (isReadOnly == false) {
                 throw new IllegalStateException(
                     "The index "
@@ -206,7 +207,7 @@ private static boolean isReadOnlyCompatible(
         return false;
     }
 
-    private static boolean hasIndexWritesBlock(IndexMetadata indexMetadata) {
+    static boolean hasReadOnlyBlocks(IndexMetadata indexMetadata) {
         var indexSettings = indexMetadata.getSettings();
         if (IndexMetadata.INDEX_BLOCKS_WRITE_SETTING.get(indexSettings) || IndexMetadata.INDEX_READ_ONLY_SETTING.get(indexSettings)) {
             return indexMetadata.isSearchableSnapshot()

server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexStateService.java

@@ -90,6 +90,7 @@
 import java.util.stream.Collectors;
 
 import static java.util.stream.Collectors.joining;
+import static org.elasticsearch.cluster.metadata.IndexMetadataVerifier.hasReadOnlyBlocks;
 import static org.elasticsearch.core.Strings.format;
 
 /**
@@ -1185,6 +1186,18 @@ private ClusterState openIndices(final Index[] indices, final ClusterState curre
                     final Settings.Builder updatedSettings = Settings.builder().put(indexMetadata.getSettings());
                     updatedSettings.remove(VERIFIED_BEFORE_CLOSE_SETTING.getKey());
 
+                    // Reopening a read-only compatible index that has not been marked as read-only is possible if the index was
+                    // verified-before-close in the first place.
+                    var compatibilityVersion = indexMetadata.getCompatibilityVersion();
+                    if (compatibilityVersion.before(minIndexCompatibilityVersion) && hasReadOnlyBlocks(indexMetadata) == false) {
+                        if (isIndexVerifiedBeforeClosed(indexMetadata)) {
+                            updatedSettings.put(VERIFIED_READ_ONLY_SETTING.getKey(), true);
+                            // at least set a write block if the index was verified-before-close at the time the cluster was upgraded
+                            blocks.addIndexBlock(index.getName(), APIBlock.WRITE.block);
+                            updatedSettings.put(APIBlock.WRITE.settingName(), true);
+                        } // or else, the following indexMetadataVerifier.verifyIndexMetadata() should throw.
+                    }
+
                     IndexMetadata newIndexMetadata = IndexMetadata.builder(indexMetadata)
                         .state(IndexMetadata.State.OPEN)
                         .settingsVersion(indexMetadata.getSettingsVersion() + 1)