[ML] Avoid the .ml-stats index in post test cleanup (#116476)

Fixes ml yaml rest tests failing in the post clean up with a search_phase_execution_exception
against the .ml-stats index. The fix is to use another method to find reference ingest pipelines
avoid the call to _ml/trained_models/_stats

Author: davidkyle

muted-tests.yml

@@ -17,9 +17,6 @@ tests:
 - class: org.elasticsearch.smoketest.WatcherYamlRestIT
   method: test {p0=watcher/usage/10_basic/Test watcher usage stats output}
   issue: https://github.com/elastic/elasticsearch/issues/112189
-- class: org.elasticsearch.xpack.test.rest.XPackRestIT
-  method: test {p0=ml/inference_processor/Test create processor with missing mandatory fields}
-  issue: https://github.com/elastic/elasticsearch/issues/112191
 - class: org.elasticsearch.xpack.esql.action.ManyShardsIT
   method: testRejection
   issue: https://github.com/elastic/elasticsearch/issues/112406
@@ -142,9 +139,6 @@ tests:
 - class: org.elasticsearch.search.SearchServiceTests
   method: testParseSourceValidation
   issue: https://github.com/elastic/elasticsearch/issues/115936
-- class: org.elasticsearch.xpack.test.rest.XPackRestIT
-  method: test {p0=ml/inference_crud/Test delete given model referenced by pipeline}
-  issue: https://github.com/elastic/elasticsearch/issues/115970
 - class: org.elasticsearch.index.reindex.ReindexNodeShutdownIT
   method: testReindexWithShutdown
   issue: https://github.com/elastic/elasticsearch/issues/115996
@@ -168,48 +162,27 @@ tests:
 - class: org.elasticsearch.xpack.ml.integration.DatafeedJobsRestIT
   method: testLookbackWithIndicesOptions
   issue: https://github.com/elastic/elasticsearch/issues/116127
-- class: org.elasticsearch.xpack.test.rest.XPackRestIT
-  method: test {p0=ml/inference_crud/Test delete given model with alias referenced by pipeline}
-  issue: https://github.com/elastic/elasticsearch/issues/116133
 - class: org.elasticsearch.xpack.esql.qa.multi_node.EsqlSpecIT
   method: test {categorize.Categorize SYNC}
   issue: https://github.com/elastic/elasticsearch/issues/113054
 - class: org.elasticsearch.xpack.esql.qa.multi_node.EsqlSpecIT
   method: test {categorize.Categorize ASYNC}
   issue: https://github.com/elastic/elasticsearch/issues/113055
-- class: org.elasticsearch.xpack.inference.InferenceRestIT
-  method: test {p0=inference/40_semantic_text_query/Query a field that uses the default ELSER 2 endpoint}
-  issue: https://github.com/elastic/elasticsearch/issues/114376
-- class: org.elasticsearch.xpack.test.rest.XPackRestIT
-  method: test {p0=ml/inference_crud/Test force delete given model with alias referenced by pipeline}
-  issue: https://github.com/elastic/elasticsearch/issues/116136
 - class: org.elasticsearch.xpack.test.rest.XPackRestIT
   method: test {p0=transform/transforms_start_stop/Test start already started transform}
   issue: https://github.com/elastic/elasticsearch/issues/98802
 - class: org.elasticsearch.action.search.SearchPhaseControllerTests
   method: testProgressListener
   issue: https://github.com/elastic/elasticsearch/issues/116149
-- class: org.elasticsearch.xpack.test.rest.XPackRestIT
-  method: test {p0=ml/forecast/Test forecast unknown job}
-  issue: https://github.com/elastic/elasticsearch/issues/116150
 - class: org.elasticsearch.xpack.test.rest.XPackRestIT
   method: test {p0=terms_enum/10_basic/Test security}
   issue: https://github.com/elastic/elasticsearch/issues/116178
 - class: org.elasticsearch.search.basic.SearchWithRandomDisconnectsIT
   method: testSearchWithRandomDisconnects
   issue: https://github.com/elastic/elasticsearch/issues/116175
-- class: org.elasticsearch.xpack.test.rest.XPackRestIT
-  method: test {p0=ml/start_stop_datafeed/Test start datafeed given index pattern with no matching indices}
-  issue: https://github.com/elastic/elasticsearch/issues/116220
 - class: org.elasticsearch.search.basic.SearchWhileRelocatingIT
   method: testSearchAndRelocateConcurrentlyRandomReplicas
   issue: https://github.com/elastic/elasticsearch/issues/116145
-- class: org.elasticsearch.xpack.test.rest.XPackRestIT
-  method: test {p0=ml/filter_crud/Test update filter}
-  issue: https://github.com/elastic/elasticsearch/issues/116271
-- class: org.elasticsearch.xpack.test.rest.XPackRestIT
-  method: test {p0=ml/get_datafeeds/Test explicit get all datafeeds}
-  issue: https://github.com/elastic/elasticsearch/issues/116284
 - class: org.elasticsearch.xpack.deprecation.DeprecationHttpIT
   method: testDeprecatedSettingsReturnWarnings
   issue: https://github.com/elastic/elasticsearch/issues/108628
@@ -231,24 +204,9 @@ tests:
 - class: org.elasticsearch.threadpool.SimpleThreadPoolIT
   method: testThreadPoolMetrics
   issue: https://github.com/elastic/elasticsearch/issues/108320
-- class: org.elasticsearch.xpack.test.rest.XPackRestIT
-  method: test {p0=ml/jobs_crud/Test put job deprecated bucket span}
-  issue: https://github.com/elastic/elasticsearch/issues/116419
-- class: org.elasticsearch.xpack.test.rest.XPackRestIT
-  method: test {p0=ml/explain_data_frame_analytics/Test both job id and body}
-  issue: https://github.com/elastic/elasticsearch/issues/116433
-- class: org.elasticsearch.smoketest.MlWithSecurityIT
-  method: test {yaml=ml/inference_crud/Test force delete given model with alias referenced by pipeline}
-  issue: https://github.com/elastic/elasticsearch/issues/116443
 - class: org.elasticsearch.xpack.downsample.ILMDownsampleDisruptionIT
   method: testILMDownsampleRollingRestart
   issue: https://github.com/elastic/elasticsearch/issues/114233
-- class: org.elasticsearch.xpack.test.rest.XPackRestIT
-  method: test {p0=ml/data_frame_analytics_crud/Test put config with unknown field in outlier detection analysis}
-  issue: https://github.com/elastic/elasticsearch/issues/116458
-- class: org.elasticsearch.xpack.test.rest.XPackRestIT
-  method: test {p0=ml/evaluate_data_frame/Test outlier_detection with query}
-  issue: https://github.com/elastic/elasticsearch/issues/116484
 - class: org.elasticsearch.xpack.kql.query.KqlQueryBuilderTests
   issue: https://github.com/elastic/elasticsearch/issues/116487
 - class: org.elasticsearch.reservedstate.service.FileSettingsServiceTests
@@ -263,12 +221,6 @@ tests:
 - class: org.elasticsearch.xpack.logsdb.qa.StandardVersusLogsIndexModeRandomDataDynamicMappingChallengeRestIT
   method: testMatchAllQuery
   issue: https://github.com/elastic/elasticsearch/issues/116536
-- class: org.elasticsearch.xpack.test.rest.XPackRestIT
-  method: test {p0=ml/inference_crud/Test force delete given model referenced by pipeline}
-  issue: https://github.com/elastic/elasticsearch/issues/116555
-- class: org.elasticsearch.smoketest.MlWithSecurityIT
-  method: test {yaml=ml/data_frame_analytics_crud/Test delete given stopped config}
-  issue: https://github.com/elastic/elasticsearch/issues/116608
 - class: org.elasticsearch.xpack.esql.ccq.MultiClusterSpecIT
   method: test {categorize.Categorize}
   issue: https://github.com/elastic/elasticsearch/issues/116434
@@ -284,9 +236,6 @@ tests:
 - class: org.elasticsearch.packaging.test.BootstrapCheckTests
   method: test20RunWithBootstrapChecks
   issue: https://github.com/elastic/elasticsearch/issues/116620
-- class: org.elasticsearch.smoketest.MlWithSecurityIT
-  method: test {yaml=ml/inference_crud/Test force delete given model referenced by pipeline}
-  issue: https://github.com/elastic/elasticsearch/issues/116624
 - class: org.elasticsearch.packaging.test.DockerTests
   method: test011SecurityEnabledStatus
   issue: https://github.com/elastic/elasticsearch/issues/116628

x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/integration/MlRestTestStateCleaner.java

@@ -10,14 +10,15 @@
 import org.elasticsearch.client.Request;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.client.RestClient;
-import org.elasticsearch.common.xcontent.support.XContentMapValues;
 import org.elasticsearch.test.rest.ESRestTestCase;
 
 import java.io.IOException;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
-import java.util.Set;
-import java.util.stream.Collectors;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.hasSize;
 
 public class MlRestTestStateCleaner {
 
@@ -30,37 +31,34 @@ public MlRestTestStateCleaner(Logger logger, RestClient adminClient) {
     }
 
     public void resetFeatures() throws IOException {
-        waitForMlStatsIndexToInitialize();
-        deleteAllTrainedModelIngestPipelines();
+        deletePipelinesWithInferenceProcessors();
         // This resets all features, not just ML, but they should have been getting reset between tests anyway so it shouldn't matter
         adminClient.performRequest(new Request("POST", "/_features/_reset"));
     }
 
     @SuppressWarnings("unchecked")
-    private void deleteAllTrainedModelIngestPipelines() throws IOException {
-        final Request getAllTrainedModelStats = new Request("GET", "/_ml/trained_models/_stats");
-        getAllTrainedModelStats.addParameter("size", "10000");
-        final Response trainedModelsStatsResponse = adminClient.performRequest(getAllTrainedModelStats);
+    private void deletePipelinesWithInferenceProcessors() throws IOException {
+        final Response pipelinesResponse = adminClient.performRequest(new Request("GET", "/_ingest/pipeline"));
+        final Map<String, Object> pipelines = ESRestTestCase.entityAsMap(pipelinesResponse);
+
+        var pipelinesWithInferenceProcessors = new HashSet<String>();
+        for (var entry : pipelines.entrySet()) {
+            var pipelineDef = (Map<String, Object>) entry.getValue(); // each top level object is a separate pipeline
+            var processors = (List<Map<String, Object>>) pipelineDef.get("processors");
+            for (var processor : processors) {
+                assertThat(processor.entrySet(), hasSize(1));
+                if ("inference".equals(processor.keySet().iterator().next())) {
+                    pipelinesWithInferenceProcessors.add(entry.getKey());
+                }
+            }
+        }
 
-        final List<Map<String, Object>> pipelines = (List<Map<String, Object>>) XContentMapValues.extractValue(
-            "trained_model_stats.ingest.pipelines",
-            ESRestTestCase.entityAsMap(trainedModelsStatsResponse)
-        );
-        Set<String> pipelineIds = pipelines.stream().flatMap(m -> m.keySet().stream()).collect(Collectors.toSet());
-        for (String pipelineId : pipelineIds) {
+        for (String pipelineId : pipelinesWithInferenceProcessors) {
             try {
                 adminClient.performRequest(new Request("DELETE", "/_ingest/pipeline/" + pipelineId));
             } catch (Exception ex) {
                 logger.warn(() -> "failed to delete pipeline [" + pipelineId + "]", ex);
             }
         }
     }
-
-    private void waitForMlStatsIndexToInitialize() throws IOException {
-        ESRestTestCase.ensureHealth(adminClient, ".ml-stats-*", (request) -> {
-            request.addParameter("wait_for_no_initializing_shards", "true");
-            request.addParameter("level", "shards");
-            request.addParameter("timeout", "30s");
-        });
-    }
 }