Migrate ILM REST tests to new test framework. (elastic#136744)

I'll follow up consolidating the projects in another PR. This felt
easier to review if migrating in place.

Relates to ES-11813

Author: mosche

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/RestrictedBuildApiService.java

@@ -61,9 +61,6 @@ private static ListMultimap<Class<?>, String> createLegacyRestTestBasePluginUsag
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:eql:qa:correctness");
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:eql:qa:mixed-node");
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:identity-provider:qa:idp-rest-tests");
-        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:ilm:qa:multi-cluster");
-        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:ilm:qa:multi-node");
-        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:ilm:qa:rest");
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:ml:qa:basic-multi-node");
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:ml:qa:disabled");
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:ml:qa:ml-with-security");

test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java

@@ -2594,6 +2594,7 @@ public static Optional<Version> parseLegacyVersion(String version) {
     protected static void waitForActiveLicense(final RestClient restClient) throws Exception {
         assertBusy(() -> {
             final Request request = new Request(HttpGet.METHOD_NAME, "/_xpack");
+            request.addParameter("categories", "license");
             request.setOptions(RequestOptions.DEFAULT.toBuilder());
 
             final Response response = restClient.performRequest(request);

x-pack/plugin/ilm/qa/multi-cluster/build.gradle

@@ -5,64 +5,20 @@
  * 2.0.
  */
 
-import org.elasticsearch.gradle.internal.test.RestIntegTestTask
-import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE
-
-apply plugin: 'elasticsearch.internal-testclusters'
-apply plugin: 'elasticsearch.standalone-rest-test'
+apply plugin: 'elasticsearch.internal-java-rest-test'
 
 dependencies {
-  testImplementation project(':x-pack:plugin:core')
-  testImplementation project(':x-pack:plugin:ilm')
-}
-
-File repoDir = file("$buildDir/testclusters/repo")
-
-tasks.register('leader-cluster', RestIntegTestTask) {
-  mustRunAfter("precommit")
-    systemProperty 'tests.target_cluster', 'leader'
-    /* To support taking index snapshots, we have to set path.repo setting */
-    nonInputProperties.systemProperty 'tests.path.repo', repoDir.absolutePath
-}
-
-testClusters.matching { it.name == 'leader-cluster' }.configureEach {
-  testDistribution = 'DEFAULT'
-  setting 'path.repo', repoDir.absolutePath, IGNORE_VALUE
-  setting 'xpack.ccr.enabled', 'true'
-  setting 'xpack.security.enabled', 'false'
-  setting 'xpack.watcher.enabled', 'false'
-  setting 'xpack.ml.enabled', 'false'
-  setting 'xpack.license.self_generated.type', 'trial'
-  setting 'indices.lifecycle.poll_interval', '1000ms'
-}
-
-tasks.register('follow-cluster', RestIntegTestTask) {
-  dependsOn tasks.findByName('leader-cluster')
-  useCluster testClusters.named('leader-cluster')
-  systemProperty 'tests.target_cluster', 'follow'
-  nonInputProperties.systemProperty 'tests.leader_host',
-    "${-> testClusters."leader-cluster".getAllHttpSocketURI().get(0)}"
-  nonInputProperties.systemProperty 'tests.leader_remote_cluster_seed',
-    "${-> testClusters.'leader-cluster'.getAllTransportPortURI().get(0)}"
-  /* To support taking index snapshots, we have to set path.repo setting */
-  nonInputProperties.systemProperty 'tests.path.repo', repoDir.absolutePath
-}
-
-testClusters.matching{ it.name == 'follow-cluster' }.configureEach {
-  testDistribution = 'DEFAULT'
-  setting 'path.repo', repoDir.absolutePath, IGNORE_VALUE
-  setting 'xpack.ccr.enabled', 'true'
-  setting 'xpack.security.enabled', 'false'
-  setting 'xpack.watcher.enabled', 'false'
-  setting 'xpack.ml.enabled', 'false'
-  setting 'xpack.license.self_generated.type', 'trial'
-  setting 'indices.lifecycle.poll_interval', '1000ms'
-  setting 'cluster.remote.leader_cluster.seeds',
-    { "\"${testClusters.'leader-cluster'.getAllTransportPortURI().get(0)}\"" }, IGNORE_VALUE
+  javaRestTestImplementation(testArtifact(project(xpackModule('core'))))
+  javaRestTestImplementation(testArtifact(project(xpackModule('ccr')), 'javaRestTest'))
+  javaRestTestImplementation project(xpackModule('ilm'))
+
+  clusterModules project(xpackModule('ilm'))
+  clusterModules project(xpackModule('ccr'))
+  clusterModules project(xpackModule('searchable-snapshots'))
+  clusterModules project(':modules:data-streams')
 }
 
-tasks.named("check").configure { dependsOn 'follow-cluster' }
-// Security is explicitly disabled for follow-cluster and leader-cluster, do not run these in FIPS mode
-tasks.withType(Test).configureEach {
-  enabled = buildParams.inFipsJvm == false
+tasks.named("javaRestTest") {
+  // Security is explicitly disabled for follow-cluster and leader-cluster, do not run these in FIPS mode
+  buildParams.withFipsEnabledOnly(it)
 }

x-pack/plugin/ilm/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/ilm/CCRIndexLifecycleIT.java renamed to x-pack/plugin/ilm/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/ilm/CCRIndexLifecycleIT.java

@@ -6,6 +6,9 @@
  */
 package org.elasticsearch.xpack.ilm;
 
+import com.carrotsearch.randomizedtesting.annotations.Name;
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+
 import org.apache.http.entity.ContentType;
 import org.apache.http.entity.StringEntity;
 import org.apache.http.util.EntityUtils;
@@ -23,16 +26,21 @@
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.test.cluster.ElasticsearchCluster;
 import org.elasticsearch.xcontent.ObjectPath;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentType;
 import org.elasticsearch.xcontent.json.JsonXContent;
+import org.elasticsearch.xpack.ccr.AbstractCCRRestTestCase;
 import org.elasticsearch.xpack.core.ilm.LifecycleAction;
 import org.elasticsearch.xpack.core.ilm.LifecyclePolicy;
 import org.elasticsearch.xpack.core.ilm.Phase;
 import org.elasticsearch.xpack.core.ilm.UnfollowAction;
 import org.elasticsearch.xpack.core.ilm.WaitUntilTimeSeriesEndTimePassesStep;
 import org.elasticsearch.xpack.ilm.action.RestPutLifecycleAction;
+import org.junit.ClassRule;
+import org.junit.rules.RuleChain;
+import org.junit.rules.TemporaryFolder;
 
 import java.io.IOException;
 import java.io.InputStream;
@@ -52,7 +60,7 @@
 import static org.hamcrest.Matchers.notNullValue;
 import static org.hamcrest.Matchers.nullValue;
 
-public class CCRIndexLifecycleIT extends ESCCRRestTestCase {
+public class CCRIndexLifecycleIT extends AbstractCCRRestTestCase {
 
     private static final Logger LOGGER = LogManager.getLogger(CCRIndexLifecycleIT.class);
     private static final String TEST_PHASE = "hot";
@@ -88,18 +96,69 @@ public class CCRIndexLifecycleIT extends ESCCRRestTestCase {
             }
         }""";
 
+    public static TemporaryFolder repoDir = new TemporaryFolder();
+
+    public static ElasticsearchCluster leaderCluster = ElasticsearchCluster.local()
+        .name("leader")
+        .module("x-pack-ilm")
+        .module("x-pack-ccr")
+        .module("searchable-snapshots")
+        .module("data-streams")
+        .setting("path.repo", () -> repoDir.getRoot().getAbsolutePath())
+        .setting("xpack.ccr.enabled", "true")
+        .setting("xpack.security.enabled", "false")
+        .setting("xpack.license.self_generated.type", "trial")
+        .setting("indices.lifecycle.poll_interval", "1000ms")
+        .build();
+
+    public static ElasticsearchCluster followerCluster = ElasticsearchCluster.local()
+        .name("follower")
+        .module("x-pack-ilm")
+        .module("x-pack-ccr")
+        .module("searchable-snapshots")
+        .module("data-streams")
+        .setting("path.repo", () -> repoDir.getRoot().getAbsolutePath())
+        .setting("xpack.ccr.enabled", "true")
+        .setting("xpack.security.enabled", "false")
+        .setting("xpack.license.self_generated.type", "trial")
+        .setting("indices.lifecycle.poll_interval", "1000ms")
+        .setting("cluster.remote.leader_cluster.seeds", () -> "\"" + leaderCluster.getTransportEndpoints() + "\"")
+        .build();
+
+    @ClassRule
+    public static RuleChain ruleChain = RuleChain.outerRule(repoDir).around(leaderCluster).around(followerCluster);
+
+    public CCRIndexLifecycleIT(@Name("targetCluster") TargetCluster targetCluster) {
+        super(targetCluster);
+    }
+
+    @ParametersFactory
+    public static Iterable<Object[]> parameters() throws Exception {
+        return leaderFollower();
+    }
+
+    @Override
+    protected ElasticsearchCluster getFollowerCluster() {
+        return followerCluster;
+    }
+
+    @Override
+    protected ElasticsearchCluster getLeaderCluster() {
+        return leaderCluster;
+    }
+
     public void testBasicCCRAndILMIntegration() throws Exception {
         String indexName = "logs-1";
 
         String policyName = "basic-test";
-        if ("leader".equals(targetCluster)) {
+        if (targetCluster == TargetCluster.LEADER) {
             putILMPolicy(policyName, "50GB", null, TimeValue.timeValueHours(7 * 24));
             Settings indexSettings = indexSettings(1, 0).put("index.lifecycle.name", policyName)
                 .put("index.lifecycle.rollover_alias", "logs")
                 .build();
             createIndex(indexName, indexSettings, "", "\"logs\": { }");
             ensureGreen(indexName);
-        } else if ("follow".equals(targetCluster)) {
+        } else if (targetCluster == TargetCluster.FOLLOWER) {
             // Policy with the same name must exist in follower cluster too:
             putILMPolicy(policyName, "50GB", null, TimeValue.timeValueHours(7 * 24));
             followIndex(indexName, indexName);
@@ -136,7 +195,7 @@ public void testBasicCCRAndILMIntegration() throws Exception {
                     // ILM should have unfollowed the follower index, so the following_index setting should have been removed:
                     // (this controls whether the follow engine is used)
                     assertThat(getIndexSetting(client(), indexName, "index.xpack.ccr.following_index"), nullValue());
-                });
+                }, 30, TimeUnit.SECONDS);
             }
         } else {
             fail("unexpected target cluster [" + targetCluster + "]");
@@ -145,10 +204,10 @@ public void testBasicCCRAndILMIntegration() throws Exception {
 
     public void testCCRUnfollowDuringSnapshot() throws Exception {
         String indexName = "unfollow-test-index";
-        if ("leader".equals(targetCluster)) {
+        if (targetCluster == TargetCluster.LEADER) {
             createIndex(adminClient(), indexName, indexSettings(2, 0).build());
             ensureGreen(indexName);
-        } else if ("follow".equals(targetCluster)) {
+        } else if (targetCluster == TargetCluster.FOLLOWER) {
             createNewSingletonPolicy("unfollow-only", "hot", UnfollowAction.INSTANCE, TimeValue.ZERO);
             followIndex(indexName, indexName);
             ensureGreen(indexName);
@@ -162,7 +221,7 @@ public void testCCRUnfollowDuringSnapshot() throws Exception {
                         .field("type", "fs")
                         .startObject("settings")
                         .field("compress", randomBoolean())
-                        .field("location", System.getProperty("tests.path.repo"))
+                        .field("location", repoDir.getRoot().getAbsolutePath())
                         .field("max_snapshot_bytes_per_sec", "256b")
                         .endObject()
                         .endObject()
@@ -215,7 +274,7 @@ public void testCcrAndIlmWithRollover() throws Exception {
         String nextIndexName = "mymetrics-000002";
         String policyName = "rollover-test";
 
-        if ("leader".equals(targetCluster)) {
+        if (targetCluster == TargetCluster.LEADER) {
             // Create a policy on the leader
             putILMPolicy(policyName, null, 1, null);
             Request templateRequest = new Request("PUT", "/_index_template/my_template");
@@ -226,7 +285,7 @@ public void testCcrAndIlmWithRollover() throws Exception {
                 "{\"index_patterns\":  [\"mymetrics-*\"], \"template\":{\"settings\":  " + Strings.toString(indexSettings) + "}}"
             );
             assertOK(client().performRequest(templateRequest));
-        } else if ("follow".equals(targetCluster)) {
+        } else if (targetCluster == TargetCluster.FOLLOWER) {
             // Policy with the same name must exist in follower cluster too:
             putILMPolicy(policyName, null, 1, null);
 
@@ -330,7 +389,7 @@ public void testAliasReplicatedOnShrink() throws Exception {
         final String policyName = "shrink-test-policy";
         final int numberOfAliases = randomIntBetween(0, 4);
 
-        if ("leader".equals(targetCluster)) {
+        if (targetCluster == TargetCluster.LEADER) {
             // this policy won't exist on the leader, that's fine
             Settings indexSettings = indexSettings(3, 0).put("index.lifecycle.name", policyName).build();
             final StringBuilder aliases = new StringBuilder();
@@ -349,7 +408,7 @@ public void testAliasReplicatedOnShrink() throws Exception {
             }
             createIndex(indexName, indexSettings, "", aliases.toString());
             ensureGreen(indexName);
-        } else if ("follow".equals(targetCluster)) {
+        } else if (targetCluster == TargetCluster.FOLLOWER) {
             // Create a policy with just a Shrink action on the follower
             putShrinkOnlyPolicy(client(), policyName);
 
@@ -391,12 +450,12 @@ public void testUnfollowInjectedBeforeShrink() throws Exception {
         final String indexName = "shrink-test";
         final String policyName = "shrink-test-policy";
 
-        if ("leader".equals(targetCluster)) {
+        if (targetCluster == TargetCluster.LEADER) {
             // this policy won't exist on the leader, that's fine
             Settings indexSettings = indexSettings(3, 0).put("index.lifecycle.name", policyName).build();
             createIndex(indexName, indexSettings, "", "");
             ensureGreen(indexName);
-        } else if ("follow".equals(targetCluster)) {
+        } else if (targetCluster == TargetCluster.FOLLOWER) {
             // Create a policy with just a Shrink action on the follower
             putShrinkOnlyPolicy(client(), policyName);
 
@@ -433,14 +492,14 @@ public void testUnfollowInjectedBeforeShrink() throws Exception {
     public void testCannotShrinkLeaderIndex() throws Exception {
         String indexName = "shrink-leader-test";
         String policyName = "shrink-leader-test-policy";
-        if ("leader".equals(targetCluster)) {
+        if (targetCluster == TargetCluster.LEADER) {
             // Set up the policy and index, but don't attach the policy yet,
             // otherwise it'll proceed through shrink before we can set up the
             // follower
             putShrinkOnlyPolicy(client(), policyName);
             createIndex(indexName, indexSettings(2, 0).build(), "", "");
             ensureGreen(indexName);
-        } else if ("follow".equals(targetCluster)) {
+        } else if (targetCluster == TargetCluster.FOLLOWER) {
 
             try (RestClient leaderClient = buildLeaderClient()) {
                 // Policy with the same name must exist in follower cluster too:
@@ -505,16 +564,15 @@ public void testILMUnfollowFailsToRemoveRetentionLeases() throws Exception {
         final String followerIndex = "follower";
         final String policyName = "unfollow_only_policy";
 
-        if ("leader".equals(targetCluster)) {
+        if (targetCluster == TargetCluster.LEADER) {
             Settings indexSettings = indexSettings(1, 0).put("index.lifecycle.name", policyName) // this policy won't exist on the leader,
                                                                                                  // that's fine
                 .build();
             createIndex(leaderIndex, indexSettings, "", "");
             ensureGreen(leaderIndex);
-        } else if ("follow".equals(targetCluster)) {
+        } else if (targetCluster == TargetCluster.FOLLOWER) {
             try (RestClient leaderClient = buildLeaderClient()) {
-                String leaderRemoteClusterSeed = System.getProperty("tests.leader_remote_cluster_seed");
-                configureRemoteClusters("other_remote", leaderRemoteClusterSeed);
+                configureRemoteClusters("other_remote", leaderCluster.getTransportEndpoints());
                 assertBusy(() -> {
                     Map<?, ?> localConnection = (Map<?, ?>) toMap(client().performRequest(new Request("GET", "/_remote/info"))).get(
                         "other_remote"
@@ -579,12 +637,12 @@ public void testTsdbLeaderIndexRolloverAndSyncAfterWaitUntilEndTime() throws Exc
         String dataStream = "tsdb-index-cpu";
         String policyName = "tsdb-policy";
 
-        if ("leader".equals(targetCluster)) {
+        if (targetCluster == TargetCluster.LEADER) {
             putILMPolicy(policyName, null, 1, null);
             Request templateRequest = new Request("PUT", "/_index_template/tsdb_template");
             templateRequest.setJsonEntity(Strings.format(TSDB_INDEX_TEMPLATE, indexPattern, policyName));
             assertOK(client().performRequest(templateRequest));
-        } else if ("follow".equals(targetCluster)) {
+        } else if (targetCluster == TargetCluster.FOLLOWER) {
             // Use unfollow-only policy for follower cluster instead of regular ILM policy
             // Follower clusters should not have their own rollover actions as they are meant
             // to follow the rollover behavior of the leader index, not initiate their own rollovers