[TEST] Work around _cat/indices bug with security enabled (#47160)

When the ML native multi-node tests use _cat/indices/_all
and the request goes to a non-master node, _all is
translated to a list of concrete indices by the authz layer
on the coordinating node before the request is forwarded
to the master node. Then it is possible for the master
node to return an index_not_found_exception if one of
the concrete indices that was expanded on the
coordinating node has been deleted in the meantime.
(#47159 has been opened to track the underlying problem.)

It has been observed that the index that gets deleted when
the problem affects the ML native multi-node tests is
always the ML notifications index. The tests that fail are
only interested in the presence or absense of ML results
indices. Therefore the workaround is to only _cat indices
that match the ML results index pattern.

Fixes #45652

Author: droberts195

x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/MlJobIT.java

@@ -208,7 +208,9 @@ public void testCreateJobsWithIndexNameOption() throws Exception {
             }
         });
 
-        String responseAsString = EntityUtils.toString(client().performRequest(new Request("GET", "/_cat/indices")).getEntity());
+        // Use _cat/indices/.ml-anomalies-* instead of _cat/indices/_all to workaround https://github.com/elastic/elasticsearch/issues/45652
+        String responseAsString = EntityUtils.toString(client().performRequest(
+            new Request("GET", "/_cat/indices/" + AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + "*")).getEntity());
         assertThat(responseAsString,
                 containsString(AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + "custom-" + indexName));
         assertThat(responseAsString, not(containsString(AnomalyDetectorsIndex.jobResultsAliasedName(jobId1))));
@@ -272,7 +274,8 @@ public void testCreateJobsWithIndexNameOption() throws Exception {
         assertThat(responseAsString, not(containsString(AnomalyDetectorsIndex.jobResultsAliasedName(jobId1))));
         assertThat(responseAsString, containsString(AnomalyDetectorsIndex.jobResultsAliasedName(jobId2))); //job2 still exists
 
-        responseAsString = EntityUtils.toString(client().performRequest(new Request("GET", "/_cat/indices")).getEntity());
+        responseAsString = EntityUtils.toString(client().performRequest(
+            new Request("GET", "/_cat/indices/" + AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + "*")).getEntity());
         assertThat(responseAsString, containsString(AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + "custom-" + indexName));
 
         client().performRequest(new Request("POST", "/_refresh"));
@@ -287,7 +290,8 @@ public void testCreateJobsWithIndexNameOption() throws Exception {
         assertThat(responseAsString, not(containsString(AnomalyDetectorsIndex.jobResultsAliasedName(jobId2))));
 
         client().performRequest(new Request("POST", "/_refresh"));
-        responseAsString = EntityUtils.toString(client().performRequest(new Request("GET", "/_cat/indices")).getEntity());
+        responseAsString = EntityUtils.toString(client().performRequest(
+            new Request("GET", "/_cat/indices/" + AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + "*")).getEntity());
         assertThat(responseAsString, not(containsString(AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + "custom-" + indexName)));
     }
 
@@ -394,19 +398,21 @@ public void testCreateJob_WithClashingFieldMappingsFails() throws Exception {
                         "avoid the clash by assigning a dedicated results index"));
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/45652")
     public void testDeleteJob() throws Exception {
         String jobId = "delete-job-job";
         String indexName = AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + AnomalyDetectorsIndexFields.RESULTS_INDEX_DEFAULT;
         createFarequoteJob(jobId);
 
-        String indicesBeforeDelete = EntityUtils.toString(client().performRequest(new Request("GET", "/_cat/indices")).getEntity());
+        // Use _cat/indices/.ml-anomalies-* instead of _cat/indices/_all to workaround https://github.com/elastic/elasticsearch/issues/45652
+        String indicesBeforeDelete = EntityUtils.toString(client().performRequest(
+            new Request("GET", "/_cat/indices/" + AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + "*")).getEntity());
         assertThat(indicesBeforeDelete, containsString(indexName));
 
         client().performRequest(new Request("DELETE", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId));
 
         // check that the index still exists (it's shared by default)
-        String indicesAfterDelete = EntityUtils.toString(client().performRequest(new Request("GET", "/_cat/indices")).getEntity());
+        String indicesAfterDelete = EntityUtils.toString(client().performRequest(
+            new Request("GET", "/_cat/indices/" + AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + "*")).getEntity());
         assertThat(indicesAfterDelete, containsString(indexName));
 
         waitUntilIndexIsEmpty(indexName);
@@ -465,13 +471,14 @@ public void testDeleteJob_TimingStatsDocumentIsDeleted() throws Exception {
         assertThat(exception.getResponse().getStatusLine().getStatusCode(), equalTo(404));
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/45652")
     public void testDeleteJobAsync() throws Exception {
         String jobId = "delete-job-async-job";
         String indexName = AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + AnomalyDetectorsIndexFields.RESULTS_INDEX_DEFAULT;
         createFarequoteJob(jobId);
 
-        String indicesBeforeDelete = EntityUtils.toString(client().performRequest(new Request("GET", "/_cat/indices")).getEntity());
+        // Use _cat/indices/.ml-anomalies-* instead of _cat/indices/_all to workaround https://github.com/elastic/elasticsearch/issues/45652
+        String indicesBeforeDelete = EntityUtils.toString(client().performRequest(
+            new Request("GET", "/_cat/indices/" + AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + "*")).getEntity());
         assertThat(indicesBeforeDelete, containsString(indexName));
 
         Response response = client().performRequest(new Request("DELETE", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId
@@ -483,7 +490,8 @@ public void testDeleteJobAsync() throws Exception {
         assertThat(EntityUtils.toString(taskResponse.getEntity()), containsString("\"acknowledged\":true"));
 
         // check that the index still exists (it's shared by default)
-        String indicesAfterDelete = EntityUtils.toString(client().performRequest(new Request("GET", "/_cat/indices")).getEntity());
+        String indicesAfterDelete = EntityUtils.toString(client().performRequest(
+            new Request("GET", "/_cat/indices/" + AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + "*")).getEntity());
         assertThat(indicesAfterDelete, containsString(indexName));
 
         waitUntilIndexIsEmpty(indexName);
@@ -518,7 +526,9 @@ public void testDeleteJobAfterMissingIndex() throws Exception {
         String indexName = AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + AnomalyDetectorsIndexFields.RESULTS_INDEX_DEFAULT;
         createFarequoteJob(jobId);
 
-        String indicesBeforeDelete = EntityUtils.toString(client().performRequest(new Request("GET", "/_cat/indices")).getEntity());
+        // Use _cat/indices/.ml-anomalies-* instead of _cat/indices/_all to workaround https://github.com/elastic/elasticsearch/issues/45652
+        String indicesBeforeDelete = EntityUtils.toString(client().performRequest(
+            new Request("GET", "/_cat/indices/" + AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + "*")).getEntity());
         assertThat(indicesBeforeDelete, containsString(indexName));
 
         // Manually delete the index so that we can test that deletion proceeds
@@ -528,7 +538,8 @@ public void testDeleteJobAfterMissingIndex() throws Exception {
         client().performRequest(new Request("DELETE", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId));
 
         // check index was deleted
-        String indicesAfterDelete = EntityUtils.toString(client().performRequest(new Request("GET", "/_cat/indices")).getEntity());
+        String indicesAfterDelete = EntityUtils.toString(client().performRequest(
+            new Request("GET", "/_cat/indices/" + AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + "*")).getEntity());
         assertThat(indicesAfterDelete, not(containsString(aliasName)));
         assertThat(indicesAfterDelete, not(containsString(indexName)));
 
@@ -598,7 +609,9 @@ public void testMultiIndexDelete() throws Exception {
             "}");
         client().performRequest(extraIndex2);
 
-        String indicesBeforeDelete = EntityUtils.toString(client().performRequest(new Request("GET", "/_cat/indices")).getEntity());
+        // Use _cat/indices/.ml-anomalies-* instead of _cat/indices/_all to workaround https://github.com/elastic/elasticsearch/issues/45652
+        String indicesBeforeDelete = EntityUtils.toString(client().performRequest(
+            new Request("GET", "/_cat/indices/" + AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + "*")).getEntity());
         assertThat(indicesBeforeDelete, containsString(indexName));
         assertThat(indicesBeforeDelete, containsString(indexName + "-001"));
         assertThat(indicesBeforeDelete, containsString(indexName + "-002"));
@@ -637,7 +650,8 @@ public void testMultiIndexDelete() throws Exception {
         client().performRequest(new Request("POST", "/_refresh"));
 
         // check that the indices still exist but are empty
-        String indicesAfterDelete = EntityUtils.toString(client().performRequest(new Request("GET", "/_cat/indices")).getEntity());
+        String indicesAfterDelete = EntityUtils.toString(client().performRequest(
+            new Request("GET", "/_cat/indices/" + AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + "*")).getEntity());
         assertThat(indicesAfterDelete, containsString(indexName));
         assertThat(indicesAfterDelete, containsString(indexName + "-001"));
         assertThat(indicesAfterDelete, containsString(indexName + "-002"));