[8.17] backport datafeeds test fixes (#120301)

* Clean up lingering tasks after DatafeedJobsIT. (#120040)

* [8.x] backport datafeeds test fixes (#120289)

* Fix(?) and unmute DatafeedJobsRestIT. (#119839)

* Fix(?) and unmute DatafeedJobsRestIT.

* [CI] Auto commit changes from spotless

---------

Co-authored-by: elasticsearchmachine <[email protected]>

* Datafeed jobs it wait for task cancel (#120177)

* Fix task cancellation in DatafeedJobsIT.

* fix

* unmute DatafeedJobsIT

* unmute

---------

Co-authored-by: elasticsearchmachine <[email protected]>

---------

Co-authored-by: elasticsearchmachine <[email protected]>

Author: jan-elastic

muted-tests.yml

@@ -237,14 +237,9 @@ tests:
   issue: https://github.com/elastic/elasticsearch/issues/115664
 - class: org.elasticsearch.indices.mapping.UpdateMappingIntegrationIT
   issue: https://github.com/elastic/elasticsearch/issues/116126
-- class: org.elasticsearch.xpack.ml.integration.DatafeedJobsRestIT
-  issue: https://github.com/elastic/elasticsearch/issues/111319
 - class: org.elasticsearch.upgrades.FullClusterRestartIT
   method: testSnapshotRestore {cluster=OLD}
   issue: https://github.com/elastic/elasticsearch/issues/111777
-- class: org.elasticsearch.xpack.ml.integration.DatafeedJobsRestIT
-  method: testLookbackWithIndicesOptions
-  issue: https://github.com/elastic/elasticsearch/issues/116127
 - class: org.elasticsearch.xpack.restart.CoreFullClusterRestartIT
   method: testSnapshotRestore {cluster=UPGRADED}
   issue: https://github.com/elastic/elasticsearch/issues/111799

x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/DatafeedJobsIT.java

@@ -11,6 +11,8 @@
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ElasticsearchStatusException;
 import org.elasticsearch.ResourceNotFoundException;
+import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksRequest;
+import org.elasticsearch.action.admin.cluster.node.tasks.cancel.TransportCancelTasksAction;
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
 import org.elasticsearch.common.ReferenceDocs;
@@ -80,6 +82,13 @@ public class DatafeedJobsIT extends MlNativeAutodetectIntegTestCase {
     public void cleanup() {
         updateClusterSettings(Settings.builder().putNull("logger.org.elasticsearch.xpack.ml.datafeed"));
         cleanUp();
+        // Race conditions between closing and killing tasks in these tests,
+        // sometimes result in lingering persistent close tasks, which cause
+        // subsequent tests to fail. Therefore, they're explicitly cancelled.
+        CancelTasksRequest cancelTasksRequest = new CancelTasksRequest();
+        cancelTasksRequest.setActions("*close*");
+        cancelTasksRequest.setWaitForCompletion(true);
+        client().execute(TransportCancelTasksAction.TYPE, cancelTasksRequest).actionGet();
     }
 
     public void testLookbackOnly() throws Exception {

x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/DatafeedJobsRestIT.java

@@ -37,7 +37,6 @@
 import static org.hamcrest.Matchers.anyOf;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.not;
 
 public class DatafeedJobsRestIT extends ESRestTestCase {
 
@@ -503,12 +502,12 @@ public void testLookbackWithIndicesOptions() throws Exception {
             }""");
         client().performRequest(createJobRequest);
         String datafeedId = jobId + "-datafeed";
-        new DatafeedBuilder(datafeedId, jobId, "*hidden-*").setIndicesOptions("""
+        new DatafeedBuilder(datafeedId, jobId, "hidden-*").setIndicesOptions("""
             {"expand_wildcards": ["all"],"allow_no_indices": true}""").build();
 
         StringBuilder bulk = new StringBuilder();
 
-        Request createGeoData = new Request("PUT", "/.hidden-index");
+        Request createGeoData = new Request("PUT", "/hidden-index");
         createGeoData.setJsonEntity("""
             {
               "mappings": {
@@ -528,23 +527,23 @@ public void testLookbackWithIndicesOptions() throws Exception {
         client().performRequest(createGeoData);
 
         bulk.append("""
-            {"index": {"_index": ".hidden-index", "_id": 1}}
+            {"index": {"_index": "hidden-index", "_id": 1}}
             {"time":"2016-06-01T00:00:00Z","value": 1000}
-            {"index": {"_index": ".hidden-index", "_id": 2}}
+            {"index": {"_index": "hidden-index", "_id": 2}}
             {"time":"2016-06-01T00:05:00Z","value":1500}
-            {"index": {"_index": ".hidden-index", "_id": 3}}
+            {"index": {"_index": "hidden-index", "_id": 3}}
             {"time":"2016-06-01T00:10:00Z","value":1600}
-            {"index": {"_index": ".hidden-index", "_id": 4}}
+            {"index": {"_index": "hidden-index", "_id": 4}}
             {"time":"2016-06-01T00:15:00Z","value":100}
-            {"index": {"_index": ".hidden-index", "_id": 5}}
+            {"index": {"_index": "hidden-index", "_id": 5}}
             {"time":"2016-06-01T00:20:00Z","value":1}
-            {"index": {"_index": ".hidden-index", "_id": 6}}
+            {"index": {"_index": "hidden-index", "_id": 6}}
             {"time":"2016-06-01T00:25:00Z","value":1500}
-            {"index": {"_index": ".hidden-index", "_id": 7}}
+            {"index": {"_index": "hidden-index", "_id": 7}}
             {"time":"2016-06-01T00:30:00Z","value":1500}
-            {"index": {"_index": ".hidden-index", "_id": 8}}
+            {"index": {"_index": "hidden-index", "_id": 8}}
             {"time":"2016-06-01T00:40:00Z","value":2100}
-            {"index": {"_index": ".hidden-index", "_id": 9}}
+            {"index": {"_index": "hidden-index", "_id": 9}}
             {"time":"2016-06-01T00:41:00Z","value":0}
             """);
         bulkIndex(bulk.toString());
@@ -1802,7 +1801,7 @@ private void bulkIndex(String bulk) throws IOException {
         bulkRequest.addParameter("refresh", "true");
         bulkRequest.addParameter("pretty", null);
         String bulkResponse = EntityUtils.toString(client().performRequest(bulkRequest).getEntity());
-        assertThat(bulkResponse, not(containsString("\"errors\": false")));
+        assertThat(bulkResponse, containsString("\"errors\" : false"));
     }
 
     private Response createJobAndDataFeed(String jobId, String datafeedId) throws IOException {