[8.x] backport datafeeds test fixes (elastic#120289)

* Fix(?) and unmute DatafeedJobsRestIT. (elastic#119839)

* Fix(?) and unmute DatafeedJobsRestIT.

* [CI] Auto commit changes from spotless

---------

Co-authored-by: elasticsearchmachine <[email protected]>

* Datafeed jobs it wait for task cancel (elastic#120177)

* Fix task cancellation in DatafeedJobsIT.

* fix

* unmute DatafeedJobsIT

* unmute

---------

Co-authored-by: elasticsearchmachine <[email protected]>

Author: jan-elastic

muted-tests.yml

@@ -213,14 +213,9 @@ tests:
   issue: https://github.com/elastic/elasticsearch/issues/115664
 - class: org.elasticsearch.indices.mapping.UpdateMappingIntegrationIT
   issue: https://github.com/elastic/elasticsearch/issues/116126
-- class: org.elasticsearch.xpack.ml.integration.DatafeedJobsRestIT
-  issue: https://github.com/elastic/elasticsearch/issues/111319
 - class: org.elasticsearch.upgrades.FullClusterRestartIT
   method: testSnapshotRestore {cluster=OLD}
   issue: https://github.com/elastic/elasticsearch/issues/111777
-- class: org.elasticsearch.xpack.ml.integration.DatafeedJobsRestIT
-  method: testLookbackWithIndicesOptions
-  issue: https://github.com/elastic/elasticsearch/issues/116127
 - class: org.elasticsearch.xpack.restart.CoreFullClusterRestartIT
   method: testSnapshotRestore {cluster=UPGRADED}
   issue: https://github.com/elastic/elasticsearch/issues/111799
@@ -432,48 +427,9 @@ tests:
 - class: org.elasticsearch.oldrepos.OldRepositoryAccessIT
   method: testOldSourceOnlyRepoAccess
   issue: https://github.com/elastic/elasticsearch/issues/120080
-- class: org.elasticsearch.xpack.ml.integration.DatafeedJobsIT
-  method: testLookbackOnlyDataStream
-  issue: https://github.com/elastic/elasticsearch/issues/120102
-- class: org.elasticsearch.xpack.ml.integration.DatafeedJobsIT
-  method: testCloseJobStopsLookbackOnlyDatafeed
-  issue: https://github.com/elastic/elasticsearch/issues/120103
-- class: org.elasticsearch.xpack.ml.integration.DatafeedJobsIT
-  method: testCloseJobStopsRealtimeDatafeed
-  issue: https://github.com/elastic/elasticsearch/issues/120104
-- class: org.elasticsearch.search.basic.SearchWithRandomIOExceptionsIT
-  method: testRandomDirectoryIOExceptions
-  issue: https://github.com/elastic/elasticsearch/issues/118733
-- class: org.elasticsearch.xpack.ml.integration.DatafeedJobsIT
-  method: testLookbackOnlyRuntimeMapping
-  issue: https://github.com/elastic/elasticsearch/issues/120153
-- class: org.elasticsearch.xpack.ml.integration.DatafeedJobsIT
-  method: testRealtime_multipleStopCalls
-  issue: https://github.com/elastic/elasticsearch/issues/120154
-- class: org.elasticsearch.xpack.ml.integration.DatafeedJobsIT
-  method: testStopAndRestartCompositeDatafeed
-  issue: https://github.com/elastic/elasticsearch/issues/120160
-- class: org.elasticsearch.xpack.ml.integration.DatafeedJobsIT
-  method: testStopLookbackFollowedByProcessKill
-  issue: https://github.com/elastic/elasticsearch/issues/120122
-- class: org.elasticsearch.xpack.ml.integration.DatafeedJobsIT
-  method: testLookbackOnly
-  issue: https://github.com/elastic/elasticsearch/issues/120166
-- class: org.elasticsearch.xpack.ml.integration.DatafeedJobsIT
-  method: testStart_GivenAggregateMetricDoubleWithoutAggs
-  issue: https://github.com/elastic/elasticsearch/issues/120152
-- class: org.elasticsearch.xpack.ml.integration.DatafeedJobsIT
-  method: testRealtime_givenSimultaneousStopAndForceDelete
-  issue: https://github.com/elastic/elasticsearch/issues/120161
 - class: org.elasticsearch.xpack.ml.integration.ClassificationHousePricingIT
   method: testFeatureImportanceValues
   issue: https://github.com/elastic/elasticsearch/issues/116564
-- class: org.elasticsearch.xpack.ml.integration.DatafeedJobsIT
-  method: testRealtime_noDataAndAutoStop
-  issue: https://github.com/elastic/elasticsearch/issues/120167
-- class: org.elasticsearch.xpack.ml.integration.DatafeedJobsIT
-  method: testStartDatafeed_GivenNegativeStartTime_Returns408
-  issue: https://github.com/elastic/elasticsearch/issues/120169
 - class: org.elasticsearch.xpack.searchablesnapshots.minio.MinioSearchableSnapshotsIT
   issue: https://github.com/elastic/elasticsearch/issues/120101
 - class: org.elasticsearch.repositories.s3.RepositoryS3MinioBasicCredentialsRestIT
@@ -486,9 +442,6 @@ tests:
 - class: org.elasticsearch.xpack.inference.InferenceCrudIT
   method: testGetServicesWithCompletionTaskType
   issue: https://github.com/elastic/elasticsearch/issues/119959
-- class: org.elasticsearch.xpack.ml.integration.DatafeedJobsIT
-  method: testDatafeedTimingStats_QueryDelayUpdated_TimingStatsNotReset
-  issue: https://github.com/elastic/elasticsearch/issues/120121
 - class: org.elasticsearch.xpack.esql.qa.mixed.MixedClusterEsqlSpecIT
   method: test {date.Implicit casting strings to dates for IN operator SYNC}
   issue: https://github.com/elastic/elasticsearch/issues/120159

x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/DatafeedJobsIT.java

@@ -83,9 +83,12 @@ public void cleanup() {
         updateClusterSettings(Settings.builder().putNull("logger.org.elasticsearch.xpack.ml.datafeed"));
         cleanUp();
         // Race conditions between closing and killing tasks in these tests,
-        // sometimes result in lingering persistent tasks (such as "_close"),
-        // which cause subsequent tests to fail.
-        client().execute(TransportCancelTasksAction.TYPE, new CancelTasksRequest());
+        // sometimes result in lingering persistent close tasks, which cause
+        // subsequent tests to fail. Therefore, they're explicitly cancelled.
+        CancelTasksRequest cancelTasksRequest = new CancelTasksRequest();
+        cancelTasksRequest.setActions("*close*");
+        cancelTasksRequest.setWaitForCompletion(true);
+        client().execute(TransportCancelTasksAction.TYPE, cancelTasksRequest).actionGet();
     }
 
     public void testLookbackOnly() throws Exception {

x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/DatafeedJobsRestIT.java

@@ -37,7 +37,6 @@
 import static org.hamcrest.Matchers.anyOf;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.not;
 
 public class DatafeedJobsRestIT extends ESRestTestCase {
 
@@ -503,12 +502,12 @@ public void testLookbackWithIndicesOptions() throws Exception {
             }""");
         client().performRequest(createJobRequest);
         String datafeedId = jobId + "-datafeed";
-        new DatafeedBuilder(datafeedId, jobId, "*hidden-*").setIndicesOptions("""
+        new DatafeedBuilder(datafeedId, jobId, "hidden-*").setIndicesOptions("""
             {"expand_wildcards": ["all"],"allow_no_indices": true}""").build();
 
         StringBuilder bulk = new StringBuilder();
 
-        Request createGeoData = new Request("PUT", "/.hidden-index");
+        Request createGeoData = new Request("PUT", "/hidden-index");
         createGeoData.setJsonEntity("""
             {
               "mappings": {
@@ -528,23 +527,23 @@ public void testLookbackWithIndicesOptions() throws Exception {
         client().performRequest(createGeoData);
 
         bulk.append("""
-            {"index": {"_index": ".hidden-index", "_id": 1}}
+            {"index": {"_index": "hidden-index", "_id": 1}}
             {"time":"2016-06-01T00:00:00Z","value": 1000}
-            {"index": {"_index": ".hidden-index", "_id": 2}}
+            {"index": {"_index": "hidden-index", "_id": 2}}
             {"time":"2016-06-01T00:05:00Z","value":1500}
-            {"index": {"_index": ".hidden-index", "_id": 3}}
+            {"index": {"_index": "hidden-index", "_id": 3}}
             {"time":"2016-06-01T00:10:00Z","value":1600}
-            {"index": {"_index": ".hidden-index", "_id": 4}}
+            {"index": {"_index": "hidden-index", "_id": 4}}
             {"time":"2016-06-01T00:15:00Z","value":100}
-            {"index": {"_index": ".hidden-index", "_id": 5}}
+            {"index": {"_index": "hidden-index", "_id": 5}}
             {"time":"2016-06-01T00:20:00Z","value":1}
-            {"index": {"_index": ".hidden-index", "_id": 6}}
+            {"index": {"_index": "hidden-index", "_id": 6}}
             {"time":"2016-06-01T00:25:00Z","value":1500}
-            {"index": {"_index": ".hidden-index", "_id": 7}}
+            {"index": {"_index": "hidden-index", "_id": 7}}
             {"time":"2016-06-01T00:30:00Z","value":1500}
-            {"index": {"_index": ".hidden-index", "_id": 8}}
+            {"index": {"_index": "hidden-index", "_id": 8}}
             {"time":"2016-06-01T00:40:00Z","value":2100}
-            {"index": {"_index": ".hidden-index", "_id": 9}}
+            {"index": {"_index": "hidden-index", "_id": 9}}
             {"time":"2016-06-01T00:41:00Z","value":0}
             """);
         bulkIndex(bulk.toString());
@@ -1802,7 +1801,7 @@ private void bulkIndex(String bulk) throws IOException {
         bulkRequest.addParameter("refresh", "true");
         bulkRequest.addParameter("pretty", null);
         String bulkResponse = EntityUtils.toString(client().performRequest(bulkRequest).getEntity());
-        assertThat(bulkResponse, not(containsString("\"errors\": false")));
+        assertThat(bulkResponse, containsString("\"errors\" : false"));
     }
 
     private Response createJobAndDataFeed(String jobId, String datafeedId) throws IOException {