Handle 404s from the _cat/indices api in FullClusterRestartIT (#122537) (#122564)

Author: joegallo

modules/ingest-geoip/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/ingest/geoip/FullClusterRestartIT.java

@@ -20,6 +20,7 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.core.Nullable;
+import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.test.cluster.ElasticsearchCluster;
 import org.elasticsearch.test.cluster.FeatureFlag;
 import org.elasticsearch.test.cluster.local.distribution.DistributionType;
@@ -168,7 +169,7 @@ public void testGeoIpSystemFeaturesMigration() throws Exception {
     @SuppressWarnings("unchecked")
     private void testDatabasesLoaded() throws IOException {
         Request getTaskState = new Request("GET", "/_cluster/state");
-        ObjectPath state = ObjectPath.createFromResponse(client().performRequest(getTaskState));
+        ObjectPath state = ObjectPath.createFromResponse(assertOK(client().performRequest(getTaskState)));
 
         List<?> tasks = state.evaluate("metadata.persistent_tasks.tasks");
         // Short-circuit to avoid using steams if the list is empty
@@ -194,7 +195,10 @@ private void testDatabasesLoaded() throws IOException {
 
     private void testCatIndices(List<String> indexNames, @Nullable List<String> additionalIndexNames) throws IOException {
         Request catIndices = new Request("GET", "_cat/indices/*?s=index&h=index&expand_wildcards=all");
-        String response = EntityUtils.toString(client().performRequest(catIndices).getEntity());
+        // the cat APIs can sometimes 404, erroneously
+        // see https://github.com/elastic/elasticsearch/issues/104371
+        setIgnoredErrorResponseCodes(catIndices, RestStatus.NOT_FOUND);
+        String response = EntityUtils.toString(assertOK(client().performRequest(catIndices)).getEntity());
         List<String> indices = List.of(response.trim().split("\\s+"));
 
         if (additionalIndexNames != null && additionalIndexNames.isEmpty() == false) {
@@ -213,7 +217,7 @@ private void testIndexGeoDoc() throws IOException {
         assertOK(client().performRequest(putDoc));
 
         Request getDoc = new Request("GET", "/my-index-00001/_doc/my_id");
-        ObjectPath doc = ObjectPath.createFromResponse(client().performRequest(getDoc));
+        ObjectPath doc = ObjectPath.createFromResponse(assertOK(client().performRequest(getDoc)));
         assertNull(doc.evaluate("_source.tags"));
         assertEquals("Sweden", doc.evaluate("_source.geo.country_name"));
     }
@@ -223,8 +227,7 @@ private void testGetStar(List<String> indexNames, @Nullable List<String> additio
         getStar.setOptions(
             RequestOptions.DEFAULT.toBuilder().setWarningsHandler(WarningsHandler.PERMISSIVE) // we don't care about warnings, just errors
         );
-        Response response = client().performRequest(getStar);
-        assertOK(response);
+        Response response = assertOK(client().performRequest(getStar));
 
         if (additionalIndexNames != null && additionalIndexNames.isEmpty() == false) {
             indexNames = new ArrayList<>(indexNames); // recopy into a mutable list
@@ -242,8 +245,7 @@ private void testGetStarAsKibana(List<String> indexNames, @Nullable List<String>
                 .addHeader("X-elastic-product-origin", "kibana")
                 .setWarningsHandler(WarningsHandler.PERMISSIVE) // we don't care about warnings, just errors
         );
-        Response response = client().performRequest(getStar);
-        assertOK(response);
+        Response response = assertOK(client().performRequest(getStar));
 
         if (additionalIndexNames != null && additionalIndexNames.isEmpty() == false) {
             indexNames = new ArrayList<>(indexNames); // recopy into a mutable list

muted-tests.yml

@@ -459,9 +459,6 @@ tests:
 - class: org.elasticsearch.xpack.transform.integration.TransformAuditorIT
   method: testAuditorWritesAudits
   issue: https://github.com/elastic/elasticsearch/issues/121241
-- class: org.elasticsearch.ingest.geoip.FullClusterRestartIT
-  method: testGeoIpSystemFeaturesMigration {cluster=UPGRADED}
-  issue: https://github.com/elastic/elasticsearch/issues/121115
 - class: org.elasticsearch.xpack.security.authc.jwt.JwtRealmSingleNodeTests
   method: testGrantApiKeyForJWT
   issue: https://github.com/elastic/elasticsearch/issues/121039

qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/FullClusterRestartIT.java

@@ -33,6 +33,7 @@
 import org.elasticsearch.index.IndexVersion;
 import org.elasticsearch.index.IndexVersions;
 import org.elasticsearch.index.mapper.DateFieldMapper;
+import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.rest.action.admin.indices.RestPutIndexTemplateAction;
 import org.elasticsearch.test.NotEqualMessageBuilder;
 import org.elasticsearch.test.XContentTestUtils;
@@ -633,13 +634,14 @@ public void testRollover() throws Exception {
                 )
             );
 
-            // assertBusy to work around https://github.com/elastic/elasticsearch/issues/104371
-            assertBusy(
-                () -> assertThat(
-                    EntityUtils.toString(client().performRequest(new Request("GET", "/_cat/indices?v&error_trace")).getEntity()),
-                    containsString("testrollover-000002")
-                )
-            );
+            assertBusy(() -> {
+                Request catIndices = new Request("GET", "/_cat/indices?v&error_trace");
+                // the cat APIs can sometimes 404, erroneously
+                // see https://github.com/elastic/elasticsearch/issues/104371
+                setIgnoredErrorResponseCodes(catIndices, RestStatus.NOT_FOUND);
+                Response response = assertOK(client().performRequest(catIndices));
+                assertThat(EntityUtils.toString(response.getEntity()), containsString("testrollover-000002"));
+            });
         }
 
         Request countRequest = new Request("POST", "/" + index + "-*/_search");