[ML] Unmute MlJobIT tests (#114553)

A large number (almost the entirety) of tests in the `MlJobIT` tests suite have been muted. In all cases the cause of failure of the tests is the same, persistent tasks for `cluster:admin/xpack/ml/job/close` and `cluster:admin/xpack/ml/job/close[n]` have been detected as present after the test case has completed.

Examination of the tests show that the majority of them do not call either `close` directly or indirectly, indicating that the root cause lies with some previous test. As the `close` task inherits the default timeout of half an hour, an instance of it lingering about can cause a lot of damage to subsequent tests.

The approach taken in this PR is to call the `_task/_cancel` endpoint after every test execution in the `MlJobIT` suite as the final operation. This should restrict the impact of the lingering `close` task to the test responsible, and the reduction in noise should permit better identification of the culprit.

Closes #105239, #113581, #113046, #112729, #113528, #112701, #113742, #113370, #112823, #112088, #112212, #112730, #113654, #113655, #112381, #113477, #112382, #113651, #112510

Author: edsavage

muted-tests.yml

@@ -83,9 +83,6 @@ tests:
 - class: org.elasticsearch.xpack.inference.InferenceRestIT
   method: test {p0=inference/80_random_rerank_retriever/Random rerank retriever predictably shuffles results}
   issue: https://github.com/elastic/elasticsearch/issues/111999
-- class: org.elasticsearch.xpack.ml.integration.MlJobIT
-  method: testDeleteJobAfterMissingIndex
-  issue: https://github.com/elastic/elasticsearch/issues/112088
 - class: org.elasticsearch.smoketest.SmokeTestMultiNodeClientYamlTestSuiteIT
   issue: https://github.com/elastic/elasticsearch/issues/112147
 - class: org.elasticsearch.smoketest.WatcherYamlRestIT
@@ -94,15 +91,9 @@ tests:
 - class: org.elasticsearch.xpack.test.rest.XPackRestIT
   method: test {p0=ml/inference_processor/Test create processor with missing mandatory fields}
   issue: https://github.com/elastic/elasticsearch/issues/112191
-- class: org.elasticsearch.xpack.ml.integration.MlJobIT
-  method: testDeleteJobAsync
-  issue: https://github.com/elastic/elasticsearch/issues/112212
 - class: org.elasticsearch.smoketest.DocsClientYamlTestSuiteIT
   method: test {yaml=reference/rest-api/watcher/put-watch/line_120}
   issue: https://github.com/elastic/elasticsearch/issues/99517
-- class: org.elasticsearch.xpack.ml.integration.MlJobIT
-  method: testMultiIndexDelete
-  issue: https://github.com/elastic/elasticsearch/issues/112381
 - class: org.elasticsearch.xpack.esql.expression.function.aggregate.SpatialCentroidTests
   method: "testAggregateIntermediate {TestCase=<geo_point> #2}"
   issue: https://github.com/elastic/elasticsearch/issues/112461
@@ -129,9 +120,6 @@ tests:
 - class: org.elasticsearch.xpack.esql.EsqlAsyncSecurityIT
   method: testIndexPatternErrorMessageComparison_ESQL_SearchDSL
   issue: https://github.com/elastic/elasticsearch/issues/112630
-- class: org.elasticsearch.xpack.ml.integration.MlJobIT
-  method: testPutJob_GivenFarequoteConfig
-  issue: https://github.com/elastic/elasticsearch/issues/112382
 - class: org.elasticsearch.packaging.test.PackagesSecurityAutoConfigurationTests
   method: test20SecurityNotAutoConfiguredOnReInstallation
   issue: https://github.com/elastic/elasticsearch/issues/112635
@@ -147,26 +135,11 @@ tests:
 - class: org.elasticsearch.xpack.sql.qa.single_node.JdbcSqlSpecIT
   method: test {case-functions.testUcaseInline3}
   issue: https://github.com/elastic/elasticsearch/issues/112643
-- class: org.elasticsearch.xpack.ml.integration.MlJobIT
-  method: testDelete_multipleRequest
-  issue: https://github.com/elastic/elasticsearch/issues/112701
-- class: org.elasticsearch.xpack.ml.integration.MlJobIT
-  method: testCreateJobInSharedIndexUpdatesMapping
-  issue: https://github.com/elastic/elasticsearch/issues/112729
-- class: org.elasticsearch.xpack.ml.integration.MlJobIT
-  method: testGetJob_GivenNoSuchJob
-  issue: https://github.com/elastic/elasticsearch/issues/112730
 - class: org.elasticsearch.script.StatsSummaryTests
   method: testEqualsAndHashCode
   issue: https://github.com/elastic/elasticsearch/issues/112439
-- class: org.elasticsearch.xpack.ml.integration.MlJobIT
-  method: testDeleteJobAfterMissingAliases
-  issue: https://github.com/elastic/elasticsearch/issues/112823
 - class: org.elasticsearch.repositories.blobstore.testkit.analyze.HdfsRepositoryAnalysisRestIT
   issue: https://github.com/elastic/elasticsearch/issues/112889
-- class: org.elasticsearch.xpack.ml.integration.MlJobIT
-  method: testCreateJob_WithClashingFieldMappingsFails
-  issue: https://github.com/elastic/elasticsearch/issues/113046
 - class: org.elasticsearch.xpack.sql.qa.security.JdbcSqlSpecIT
   method: test {case-functions.testUcaseInline1}
   issue: https://github.com/elastic/elasticsearch/issues/112641
@@ -218,9 +191,6 @@ tests:
 - class: org.elasticsearch.smoketest.DocsClientYamlTestSuiteIT
   method: test {yaml=reference/ccr/apis/follow/post-resume-follow/line_84}
   issue: https://github.com/elastic/elasticsearch/issues/113343
-- class: org.elasticsearch.xpack.ml.integration.MlJobIT
-  method: testDeleteJob_TimingStatsDocumentIsDeleted
-  issue: https://github.com/elastic/elasticsearch/issues/113370
 - class: org.elasticsearch.backwards.MixedClusterClientYamlTestSuiteIT
   method: test {p0=search/500_date_range/from, to, include_lower, include_upper deprecated}
   issue: https://github.com/elastic/elasticsearch/pull/113286
@@ -233,15 +203,6 @@ tests:
 - class: org.elasticsearch.xpack.inference.InferenceCrudIT
   method: testSupportedStream
   issue: https://github.com/elastic/elasticsearch/issues/113430
-- class: org.elasticsearch.xpack.ml.integration.MlJobIT
-  method: testOutOfOrderData
-  issue: https://github.com/elastic/elasticsearch/issues/113477
-- class: org.elasticsearch.xpack.ml.integration.MlJobIT
-  method: testCreateJobsWithIndexNameOption
-  issue: https://github.com/elastic/elasticsearch/issues/113528
-- class: org.elasticsearch.xpack.ml.integration.MlJobIT
-  method: testCantCreateJobWithSameID
-  issue: https://github.com/elastic/elasticsearch/issues/113581
 - class: org.elasticsearch.integration.KibanaUserRoleIntegTests
   method: testFieldMappings
   issue: https://github.com/elastic/elasticsearch/issues/113592
@@ -254,12 +215,6 @@ tests:
 - class: org.elasticsearch.smoketest.MlWithSecurityIT
   method: test {yaml=ml/3rd_party_deployment/Test start and stop multiple deployments}
   issue: https://github.com/elastic/elasticsearch/issues/101458
-- class: org.elasticsearch.xpack.ml.integration.MlJobIT
-  method: testGetJobs_GivenMultipleJobs
-  issue: https://github.com/elastic/elasticsearch/issues/113654
-- class: org.elasticsearch.xpack.ml.integration.MlJobIT
-  method: testGetJobs_GivenSingleJob
-  issue: https://github.com/elastic/elasticsearch/issues/113655
 - class: org.elasticsearch.xpack.security.authz.interceptor.SearchRequestCacheDisablingInterceptorTests
   method: testHasRemoteIndices
   issue: https://github.com/elastic/elasticsearch/issues/113660

x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlJobIT.java

@@ -747,7 +747,15 @@ public void testMultiIndexDelete() throws Exception {
                 }
               }
             }""", AnomalyDetectorsIndex.jobResultsAliasedName(jobId), Job.ID, jobId));
-        client().performRequest(extraIndex1);
+
+        // Creating an index with a leading dot (".") is now deprecated.
+        // Ensure the ensuing warning exception doesn't cause a test case failure
+        try {
+            client().performRequest(extraIndex1);
+        } catch (org.elasticsearch.client.WarningFailureException e) {
+            logger.warn(e.getMessage());
+        }
+
         Request extraIndex2 = new Request("PUT", indexName + "-002");
         extraIndex2.setJsonEntity(Strings.format("""
             {
@@ -762,7 +770,14 @@ public void testMultiIndexDelete() throws Exception {
                 }
               }
             }""", AnomalyDetectorsIndex.jobResultsAliasedName(jobId), Job.ID, jobId));
-        client().performRequest(extraIndex2);
+
+        // Creating an index with a leading dot (".") is now deprecated.
+        // Ensure the ensuing warning exception doesn't cause a test case failure
+        try {
+            client().performRequest(extraIndex2);
+        } catch (org.elasticsearch.client.WarningFailureException e) {
+            logger.warn(e.getMessage());
+        }
 
         // Use _cat/indices/.ml-anomalies-* instead of _cat/indices/_all to workaround https://github.com/elastic/elasticsearch/issues/45652
         String indicesBeforeDelete = EntityUtils.toString(
@@ -983,10 +998,10 @@ private Response openJob(String jobId, Optional<TimeValue> timeout) throws IOExc
     }
 
     private void closeJob(String jobId) throws IOException {
-        Response openResponse = client().performRequest(
+        Response closeResponse = client().performRequest(
             new Request("POST", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId + "/_close")
         );
-        assertThat(entityAsMap(openResponse), hasEntry("closed", true));
+        assertThat(entityAsMap(closeResponse), hasEntry("closed", true));
     }
 
     private Response putJob(String jobId, String jsonBody) throws IOException {
@@ -1000,5 +1015,8 @@ public void clearMlState() throws Exception {
         new MlRestTestStateCleaner(logger, adminClient()).resetFeatures();
         // Don't check analytics jobs as they are independent of anomaly detection jobs and should not be created by this test.
         waitForPendingTasks(adminClient(), taskName -> taskName.contains(MlTasks.DATA_FRAME_ANALYTICS_TASK_NAME));
+        // Finally, clean up any lingering persistent tasks (such as "_close", "_close[n]" etc.) that may negatively
+        // impact subsequent tests.
+        client().performRequest(new Request("POST", "/_tasks/_cancel"));
     }
 }