Add tests to randomly close and re-open searchable snapshots in N-2 version (#119594)

Relates ES-10433

Author: tlrx

qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/AbstractIndexCompatibilityTestCase.java

@@ -12,6 +12,7 @@
 import org.apache.http.entity.ContentType;
 import org.apache.http.entity.InputStreamEntity;
 import org.elasticsearch.client.Request;
+import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.core.Strings;
 import org.elasticsearch.index.IndexSettings;
@@ -232,4 +233,9 @@ protected static void updateRandomMappings(String indexName) throws IOException
         assertOK(client().performRequest(request));
     }
 
+    protected static boolean isIndexClosed(String indexName) throws Exception {
+        var responseBody = createFromResponse(client().performRequest(new Request("GET", "_cluster/state/metadata/" + indexName)));
+        var state = responseBody.evaluate("metadata.indices." + indexName + ".state");
+        return IndexMetadata.State.fromString((String) state) == IndexMetadata.State.CLOSE;
+    }
 }

qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/FullClusterRestartSearchableSnapshotIndexCompatibilityIT.java

@@ -76,7 +76,6 @@ public void testSearchableSnapshot() throws Exception {
             var mountedIndex = suffix("index-mounted");
             logger.debug("--> mounting index [{}] as [{}]", index, mountedIndex);
             mountIndex(repository, snapshot, index, randomBoolean(), mountedIndex);
-
             ensureGreen(mountedIndex);
 
             assertThat(indexVersion(mountedIndex), equalTo(VERSION_MINUS_2));
@@ -88,6 +87,23 @@ public void testSearchableSnapshot() throws Exception {
             logger.debug("--> adding replica to test peer-recovery");
             updateIndexSettings(mountedIndex, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1));
             ensureGreen(mountedIndex);
+
+            logger.debug("--> closing index [{}]", mountedIndex);
+            closeIndex(mountedIndex);
+            ensureGreen(mountedIndex);
+
+            logger.debug("--> adding replica to test peer-recovery for closed shards");
+            updateIndexSettings(mountedIndex, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 2));
+            ensureGreen(mountedIndex);
+
+            logger.debug("--> re-opening index [{}]", mountedIndex);
+            openIndex(mountedIndex);
+            ensureGreen(mountedIndex);
+
+            assertDocCount(client(), mountedIndex, numDocs);
+
+            logger.debug("--> deleting index [{}]", mountedIndex);
+            deleteIndex(mountedIndex);
         }
     }
 
@@ -138,20 +154,36 @@ public void testSearchableSnapshotUpgrade() throws Exception {
 
             assertThat(indexVersion(mountedIndex), equalTo(VERSION_MINUS_2));
             assertDocCount(client(), mountedIndex, numDocs);
+
+            logger.debug("--> adding replica to test replica upgrade");
+            updateIndexSettings(mountedIndex, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1));
+            ensureGreen(mountedIndex);
+
+            if (randomBoolean()) {
+                logger.debug("--> random closing of index [{}] before upgrade", mountedIndex);
+                closeIndex(mountedIndex);
+                ensureGreen(mountedIndex);
+            }
             return;
         }
 
         if (isFullyUpgradedTo(VERSION_CURRENT)) {
             ensureGreen(mountedIndex);
 
+            if (isIndexClosed(mountedIndex)) {
+                logger.debug("--> re-opening index [{}] after upgrade", mountedIndex);
+                openIndex(mountedIndex);
+                ensureGreen(mountedIndex);
+            }
+
             assertThat(indexVersion(mountedIndex), equalTo(VERSION_MINUS_2));
             assertDocCount(client(), mountedIndex, numDocs);
 
             updateRandomIndexSettings(mountedIndex);
             updateRandomMappings(mountedIndex);
 
             logger.debug("--> adding replica to test peer-recovery");
-            updateIndexSettings(mountedIndex, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1));
+            updateIndexSettings(mountedIndex, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 2));
             ensureGreen(mountedIndex);
         }
     }

qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/RollingUpgradeSearchableSnapshotIndexCompatibilityIT.java

@@ -73,7 +73,6 @@ public void testMountSearchableSnapshot() throws Exception {
         try {
             logger.debug("--> mounting index [{}] as [{}]", index, mountedIndex);
             mountIndex(repository, snapshot, index, randomBoolean(), mountedIndex);
-
             ensureGreen(mountedIndex);
 
             updateRandomIndexSettings(mountedIndex);
@@ -82,6 +81,14 @@ public void testMountSearchableSnapshot() throws Exception {
             assertThat(indexVersion(mountedIndex), equalTo(VERSION_MINUS_2));
             assertDocCount(client(), mountedIndex, numDocs);
 
+            logger.debug("--> closing mounted index [{}]", mountedIndex);
+            closeIndex(mountedIndex);
+            ensureGreen(mountedIndex);
+
+            logger.debug("--> re-opening index [{}]", mountedIndex);
+            openIndex(mountedIndex);
+            ensureGreen(mountedIndex);
+
             logger.debug("--> deleting mounted index [{}]", mountedIndex);
             deleteIndex(mountedIndex);
 
@@ -137,10 +144,22 @@ public void testSearchableSnapshotUpgrade() throws Exception {
 
         ensureGreen(mountedIndex);
 
+        if (isIndexClosed(mountedIndex)) {
+            logger.debug("--> re-opening index [{}] after upgrade", mountedIndex);
+            openIndex(mountedIndex);
+            ensureGreen(mountedIndex);
+        }
+
         updateRandomIndexSettings(mountedIndex);
         updateRandomMappings(mountedIndex);
 
         assertThat(indexVersion(mountedIndex), equalTo(VERSION_MINUS_2));
         assertDocCount(client(), mountedIndex, numDocs);
+
+        if (randomBoolean()) {
+            logger.debug("--> random closing of index [{}] before upgrade", mountedIndex);
+            closeIndex(mountedIndex);
+            ensureGreen(mountedIndex);
+        }
     }
 }