Handle 404s from the _cat/indices api in FullClusterRestartIT (#122537)

Author: joegallo

modules/ingest-geoip/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/ingest/geoip/FullClusterRestartIT.java

@@ -20,6 +20,7 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.core.Nullable;
+import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.test.cluster.ElasticsearchCluster;
 import org.elasticsearch.test.cluster.FeatureFlag;
 import org.elasticsearch.test.cluster.local.distribution.DistributionType;
@@ -170,7 +171,7 @@ public void testGeoIpSystemFeaturesMigration() throws Exception {
     @SuppressWarnings("unchecked")
     private void testDatabasesLoaded() throws IOException {
         Request getTaskState = new Request("GET", "/_cluster/state");
-        ObjectPath state = ObjectPath.createFromResponse(client().performRequest(getTaskState));
+        ObjectPath state = ObjectPath.createFromResponse(assertOK(client().performRequest(getTaskState)));
 
         List<?> tasks = state.evaluate("metadata.persistent_tasks.tasks");
         // Short-circuit to avoid using steams if the list is empty
@@ -196,7 +197,10 @@ private void testDatabasesLoaded() throws IOException {
 
     private void testCatIndices(List<String> indexNames, @Nullable List<String> additionalIndexNames) throws IOException {
         Request catIndices = new Request("GET", "_cat/indices/*?s=index&h=index&expand_wildcards=all");
-        String response = EntityUtils.toString(client().performRequest(catIndices).getEntity());
+        // the cat APIs can sometimes 404, erroneously
+        // see https://github.com/elastic/elasticsearch/issues/104371
+        setIgnoredErrorResponseCodes(catIndices, RestStatus.NOT_FOUND);
+        String response = EntityUtils.toString(assertOK(client().performRequest(catIndices)).getEntity());
         List<String> indices = List.of(response.trim().split("\\s+"));
 
         if (additionalIndexNames != null && additionalIndexNames.isEmpty() == false) {
@@ -215,7 +219,7 @@ private void testIndexGeoDoc() throws IOException {
         assertOK(client().performRequest(putDoc));
 
         Request getDoc = new Request("GET", "/my-index-00001/_doc/my_id");
-        ObjectPath doc = ObjectPath.createFromResponse(client().performRequest(getDoc));
+        ObjectPath doc = ObjectPath.createFromResponse(assertOK(client().performRequest(getDoc)));
         assertNull(doc.evaluate("_source.tags"));
         assertEquals("Sweden", doc.evaluate("_source.geo.country_name"));
     }
@@ -225,8 +229,7 @@ private void testGetStar(List<String> indexNames, @Nullable List<String> additio
         getStar.setOptions(
             RequestOptions.DEFAULT.toBuilder().setWarningsHandler(WarningsHandler.PERMISSIVE) // we don't care about warnings, just errors
         );
-        Response response = client().performRequest(getStar);
-        assertOK(response);
+        Response response = assertOK(client().performRequest(getStar));
 
         if (additionalIndexNames != null && additionalIndexNames.isEmpty() == false) {
             indexNames = new ArrayList<>(indexNames); // recopy into a mutable list
@@ -244,8 +247,7 @@ private void testGetStarAsKibana(List<String> indexNames, @Nullable List<String>
                 .addHeader("X-elastic-product-origin", "kibana")
                 .setWarningsHandler(WarningsHandler.PERMISSIVE) // we don't care about warnings, just errors
         );
-        Response response = client().performRequest(getStar);
-        assertOK(response);
+        Response response = assertOK(client().performRequest(getStar));
 
         if (additionalIndexNames != null && additionalIndexNames.isEmpty() == false) {
             indexNames = new ArrayList<>(indexNames); // recopy into a mutable list

muted-tests.yml

@@ -323,9 +323,6 @@ tests:
 - class: org.elasticsearch.xpack.security.profile.ProfileIntegTests
   method: testSuggestProfileWithData
   issue: https://github.com/elastic/elasticsearch/issues/121258
-- class: org.elasticsearch.ingest.geoip.FullClusterRestartIT
-  method: testGeoIpSystemFeaturesMigration {cluster=UPGRADED}
-  issue: https://github.com/elastic/elasticsearch/issues/121115
 - class: org.elasticsearch.smoketest.DocsClientYamlTestSuiteIT
   method: test {yaml=reference/cat/health/cat-health-no-timestamp-example}
   issue: https://github.com/elastic/elasticsearch/issues/121867

qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/FullClusterRestartIT.java

@@ -31,6 +31,7 @@
 import org.elasticsearch.index.IndexVersion;
 import org.elasticsearch.index.IndexVersions;
 import org.elasticsearch.index.mapper.DateFieldMapper;
+import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.rest.action.admin.indices.RestPutIndexTemplateAction;
 import org.elasticsearch.search.SearchFeatures;
 import org.elasticsearch.test.NotEqualMessageBuilder;
@@ -628,13 +629,14 @@ public void testRollover() throws Exception {
                 )
             );
 
-            // assertBusy to work around https://github.com/elastic/elasticsearch/issues/104371
-            assertBusy(
-                () -> assertThat(
-                    EntityUtils.toString(client().performRequest(new Request("GET", "/_cat/indices?v&error_trace")).getEntity()),
-                    containsString("testrollover-000002")
-                )
-            );
+            assertBusy(() -> {
+                Request catIndices = new Request("GET", "/_cat/indices?v&error_trace");
+                // the cat APIs can sometimes 404, erroneously
+                // see https://github.com/elastic/elasticsearch/issues/104371
+                setIgnoredErrorResponseCodes(catIndices, RestStatus.NOT_FOUND);
+                Response response = assertOK(client().performRequest(catIndices));
+                assertThat(EntityUtils.toString(response.getEntity()), containsString("testrollover-000002"));
+            });
         }
 
         Request countRequest = new Request("POST", "/" + index + "-*/_search");