Allow REST tests to run in MP mode

test/external-modules/multi-project/build.gradle

@@ -20,4 +20,5 @@ tasks.withType(StandaloneRestIntegTestTask).configureEach {
 
 tasks.named("javaRestTest").configure {
   enabled = buildParams.snapshotBuild
+  systemProperty "tests.multi_project.enabled", true
 }

test/external-modules/multi-project/src/javaRestTest/java/org/elasticsearch/action/admin/indices/IndexMultiProjectCRUDIT.java

@@ -19,7 +19,6 @@
 import org.elasticsearch.multiproject.MultiProjectRestTestCase;
 import org.elasticsearch.test.cluster.ElasticsearchCluster;
 import org.elasticsearch.test.cluster.local.LocalClusterSpecBuilder;
-import org.elasticsearch.test.cluster.local.distribution.DistributionType;
 import org.elasticsearch.xcontent.ObjectPath;
 import org.elasticsearch.xcontent.json.JsonXContent;
 import org.junit.ClassRule;
@@ -46,7 +45,6 @@ public class IndexMultiProjectCRUDIT extends MultiProjectRestTestCase {
     private static ElasticsearchCluster createCluster() {
         LocalClusterSpecBuilder<ElasticsearchCluster> clusterBuilder = ElasticsearchCluster.local()
             .nodes(NODE_NUM)
-            .distribution(DistributionType.INTEG_TEST) // TODO multi-project: make this test suite work under the default distrib
             .module("test-multi-project")
             .setting("test.multi_project.enabled", "true")
             .setting("xpack.security.enabled", "false") // TODO multi-project: make this test suite work with Security enabled

test/external-modules/multi-project/src/javaRestTest/java/org/elasticsearch/action/index/IndexDocumentMultiProjectIT.java

@@ -17,7 +17,6 @@
 import org.elasticsearch.multiproject.MultiProjectRestTestCase;
 import org.elasticsearch.test.cluster.ElasticsearchCluster;
 import org.elasticsearch.test.cluster.local.LocalClusterSpecBuilder;
-import org.elasticsearch.test.cluster.local.distribution.DistributionType;
 import org.junit.ClassRule;
 import org.junit.Rule;
 import org.junit.rules.TestName;
@@ -42,7 +41,6 @@ public class IndexDocumentMultiProjectIT extends MultiProjectRestTestCase {
     private static ElasticsearchCluster createCluster() {
         LocalClusterSpecBuilder<ElasticsearchCluster> clusterBuilder = ElasticsearchCluster.local()
             .nodes(NODE_NUM)
-            .distribution(DistributionType.INTEG_TEST) // TODO multi-project: make this test suite work under the default distrib
             .module("test-multi-project")
             .setting("test.multi_project.enabled", "true")
             .setting("xpack.security.enabled", "false") // TODO multi-project: make this test suite work with Security enabled

test/external-modules/multi-project/src/javaRestTest/java/org/elasticsearch/multiproject/MultiProjectRestTestCase.java

@@ -11,77 +11,32 @@
 
 import org.elasticsearch.client.Request;
 import org.elasticsearch.client.RequestOptions;
-import org.elasticsearch.client.Response;
-import org.elasticsearch.client.ResponseException;
-import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.test.rest.ESRestTestCase;
-import org.elasticsearch.xcontent.ObjectPath;
 import org.junit.After;
 
 import java.io.IOException;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.stream.Collectors;
 
 public abstract class MultiProjectRestTestCase extends ESRestTestCase {
-    protected static Request setRequestProjectId(Request request, String projectId) {
-        RequestOptions.Builder options = request.getOptions().toBuilder();
-        options.removeHeader(Task.X_ELASTIC_PROJECT_ID_HTTP_HEADER);
-        options.addHeader(Task.X_ELASTIC_PROJECT_ID_HTTP_HEADER, projectId);
-        request.setOptions(options);
-        return request;
-    }
-
-    protected static void clearRequestProjectId(Request request) {
-        RequestOptions options = request.getOptions();
-        if (options.containsHeader(Task.X_ELASTIC_PROJECT_ID_HTTP_HEADER)) {
-            request.setOptions(options.toBuilder().removeHeader(Task.X_ELASTIC_PROJECT_ID_HTTP_HEADER).build());
-        }
-    }
-
-    protected void createProject(String projectId) throws IOException {
-        Request request = new Request("PUT", "/_project/" + projectId);
-        try {
-            Response response = adminClient().performRequest(request);
-            assertOK(response);
-            logger.info("Created project {} : {}", projectId, response.getStatusLine());
-        } catch (ResponseException e) {
-            logger.error("Failed to create project: {}", projectId);
-            throw e;
-        }
-    }
-
-    protected void deleteProject(String projectId) throws IOException {
-        final Request request = new Request("DELETE", "/_project/" + projectId);
-        try {
-            final Response response = adminClient().performRequest(request);
-            logger.info("Deleted project {} : {}", projectId, response.getStatusLine());
-        } catch (ResponseException e) {
-            logger.error("Failed to delete project: {}", projectId);
-            throw e;
-        }
-    }
 
-    protected Set<String> listProjects() throws IOException {
-        final Request request = new Request("GET", "/_cluster/state/metadata?multi_project");
-        final Response response = adminClient().performRequest(request);
-        final List<Map<String, ?>> projects = ObjectPath.eval("metadata.projects", entityAsMap(response));
-        return projects.stream().map(m -> String.valueOf(m.get("id"))).collect(Collectors.toSet());
+    @Override
+    protected boolean shouldConfigureProjects() {
+        return false;
     }
 
     @After
     public void removeNonDefaultProjects() throws IOException {
         if (preserveClusterUponCompletion() == false) {
-            final Set<String> projects = listProjects();
-            logger.info("Removing non-default projects from {}", projects);
-            for (String projectId : projects) {
-                if (projectId.equals(Metadata.DEFAULT_PROJECT_ID.id()) == false) {
-                    deleteProject(projectId);
-                }
-            }
+            cleanUpProjects();
         }
     }
 
+    protected static Request setRequestProjectId(Request request, String projectId) {
+        RequestOptions.Builder options = request.getOptions().toBuilder();
+        options.removeHeader(Task.X_ELASTIC_PROJECT_ID_HTTP_HEADER);
+        options.addHeader(Task.X_ELASTIC_PROJECT_ID_HTTP_HEADER, projectId);
+        request.setOptions(options);
+        return request;
+    }
+
 }

test/external-modules/multi-project/src/javaRestTest/java/org/elasticsearch/multiproject/action/ProjectCrudActionIT.java

@@ -12,9 +12,9 @@
 import org.elasticsearch.client.Request;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.client.ResponseException;
+import org.elasticsearch.multiproject.MultiProjectRestTestCase;
 import org.elasticsearch.test.cluster.ElasticsearchCluster;
 import org.elasticsearch.test.cluster.local.distribution.DistributionType;
-import org.elasticsearch.test.rest.ESRestTestCase;
 import org.elasticsearch.test.rest.ObjectPath;
 import org.junit.ClassRule;
 
@@ -31,7 +31,7 @@
 import static org.hamcrest.CoreMatchers.hasItem;
 import static org.hamcrest.CoreMatchers.not;
 
-public class ProjectCrudActionIT extends ESRestTestCase {
+public class ProjectCrudActionIT extends MultiProjectRestTestCase {
 
     @ClassRule
     public static ElasticsearchCluster CLUSTER = ElasticsearchCluster.local()

test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java

@@ -49,12 +49,14 @@
 import org.elasticsearch.client.RestClientBuilder;
 import org.elasticsearch.client.WarningsHandler;
 import org.elasticsearch.cluster.metadata.Metadata;
+import org.elasticsearch.cluster.metadata.ProjectId;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.ssl.PemUtils;
+import org.elasticsearch.common.util.CollectionUtils;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.common.xcontent.XContentHelper;
@@ -92,6 +94,7 @@
 import org.junit.After;
 import org.junit.AfterClass;
 import org.junit.Before;
+import org.junit.BeforeClass;
 
 import java.io.BufferedReader;
 import java.io.IOException;
@@ -118,6 +121,7 @@
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Optional;
@@ -266,6 +270,9 @@ public static boolean hasXPack() {
     private static RestClient cleanupClient;
 
     private static boolean multiProjectEnabled;
+    private static String activeProject;
+    private static Set<String> extraProjects;
+    private static boolean projectsConfigured = false;
 
     public enum ProductFeature {
         XPACK,
@@ -357,6 +364,14 @@ protected static boolean testFeatureServiceInitialized() {
         return testFeatureService != ALL_FEATURES;
     }
 
+    @BeforeClass
+    public static void initializeProjectIds() {
+        // The active project-id is slightly longer, and has a fixed prefix so that it's easier to pick in error messages etc.
+        activeProject = "active00" + randomAlphaOfLength(8).toLowerCase(Locale.ROOT);
+        extraProjects = randomSet(1, 3, () -> randomAlphaOfLength(12).toLowerCase(Locale.ROOT));
+        multiProjectEnabled = Boolean.parseBoolean(System.getProperty("tests.multi_project.enabled"));
+    }
+
     @Before
     public void initClient() throws IOException {
         if (client == null) {
@@ -367,17 +382,19 @@ public void initClient() throws IOException {
             assert testFeatureServiceInitialized() == false;
             clusterHosts = parseClusterHosts(getTestRestCluster());
             logger.info("initializing REST clients against {}", clusterHosts);
-            var clientSettings = restClientSettings();
+            // We add the project ID to the client settings afterward because a lot of subclasses don't call super.restClientSettings(),
+            // meaning the project ID would be removed from the settings.
+            var clientSettings = addProjectIdToSettings(restClientSettings());
             var adminSettings = restAdminSettings();
+            var cleanupSettings = cleanupClientSettings();
             var hosts = clusterHosts.toArray(new HttpHost[0]);
             client = buildClient(clientSettings, hosts);
             adminClient = clientSettings.equals(adminSettings) ? client : buildClient(adminSettings, hosts);
-            cleanupClient = getCleanupClient();
+            cleanupClient = adminSettings.equals(cleanupSettings) ? adminClient : buildClient(cleanupSettings, hosts);
 
             availableFeatures = EnumSet.of(ProductFeature.LEGACY_TEMPLATES);
             Set<String> versions = new HashSet<>();
             boolean serverless = false;
-            String multiProjectPluginVariant = null;
 
             for (Map<?, ?> nodeInfo : getNodesInfo(adminClient).values()) {
                 var nodeVersion = nodeInfo.get("version").toString();
@@ -407,11 +424,6 @@ public void initClient() throws IOException {
                     if (moduleName.startsWith("serverless-")) {
                         serverless = true;
                     }
-                    if (moduleName.contains("test-multi-project")) {
-                        multiProjectPluginVariant = "test";
-                    } else if (moduleName.contains("serverless-multi-project")) {
-                        multiProjectPluginVariant = "serverless";
-                    }
                 }
                 if (serverless) {
                     availableFeatures.removeAll(
@@ -432,20 +444,9 @@ public void initClient() throws IOException {
                 .flatMap(Optional::stream)
                 .collect(Collectors.toSet());
             assert semanticNodeVersions.isEmpty() == false || serverless;
-
-            if (multiProjectPluginVariant != null) {
-                final Request settingRequest = new Request(
-                    "GET",
-                    "/_cluster/settings?include_defaults&filter_path=*." + multiProjectPluginVariant + ".multi_project.enabled"
-                );
-                settingRequest.setOptions(RequestOptions.DEFAULT.toBuilder().setWarningsHandler(WarningsHandler.PERMISSIVE));
-                final var response = entityAsMap(adminClient.performRequest(settingRequest));
-                multiProjectEnabled = Boolean.parseBoolean(
-                    ObjectPath.evaluate(response, "defaults." + multiProjectPluginVariant + ".multi_project.enabled")
-                );
-            }
-
             testFeatureService = createTestFeatureService(getClusterStateFeatures(adminClient), semanticNodeVersions);
+
+            configureProjects();
         }
 
         assert testFeatureServiceInitialized();
@@ -1621,9 +1622,21 @@ protected Settings restAdminSettings() {
     /**
      * Returns the REST client used for cleaning up the cluster.
      */
-    protected RestClient getCleanupClient() {
-        assert adminClient != null;
-        return adminClient;
+    protected Settings cleanupClientSettings() {
+        if (multiProjectEnabled == false || shouldConfigureProjects() == false) {
+            return restAdminSettings();
+        }
+        return addProjectIdToSettings(restAdminSettings());
+    }
+
+    private Settings addProjectIdToSettings(Settings settings) {
+        if (multiProjectEnabled == false || shouldConfigureProjects() == false) {
+            return settings;
+        }
+        return Settings.builder()
+            .put(settings)
+            .put(ThreadContext.PREFIX + "." + Task.X_ELASTIC_PROJECT_ID_HTTP_HEADER, activeProject)
+            .build();
     }
 
     /**
@@ -2741,12 +2754,50 @@ protected static void assertResultMap(
         assertMap(result, mapMatcher.entry("columns", columnMatcher).entry("values", valuesMatcher));
     }
 
+    /**
+     * Whether the test framework should configure an active projects and some extra projects. This is true by default (when multi-project
+     * is enabled). Subclasses can override this method to avoid configuring projects - e.g. when they configure projects themselves.
+     */
+    protected boolean shouldConfigureProjects() {
+        assert multiProjectEnabled;
+        return true;
+    }
+
+    private void configureProjects() throws IOException {
+        if (projectsConfigured || multiProjectEnabled == false || shouldConfigureProjects() == false) {
+            return;
+        }
+        projectsConfigured = true;
+        createProject(activeProject);
+        for (var project : extraProjects) {
+            createProject(project);
+        }
+
+        // The admin client does not set a project id, and can see all projects
+        assertProjectIds(
+            adminClient(),
+            CollectionUtils.concatLists(List.of(Metadata.DEFAULT_PROJECT_ID.id(), activeProject), extraProjects)
+        );
+        // The test client can only see the project it targets
+        assertProjectIds(client(), List.of(activeProject));
+    }
+
+    @After
+    public final void assertEmptyProjects() throws Exception {
+        if (projectsConfigured == false) {
+            return;
+        }
+        assertEmptyProject(Metadata.DEFAULT_PROJECT_ID.id());
+        for (var project : extraProjects) {
+            assertEmptyProject(project);
+        }
+    }
+
     protected void createProject(String project) throws IOException {
         assert multiProjectEnabled;
-        RestClient client = adminClient();
         final Request request = new Request("PUT", "/_project/" + project);
         try {
-            final Response response = client.performRequest(request);
+            final Response response = adminClient().performRequest(request);
             logger.info("Created project {} : {}", project, response.getStatusLine());
         } catch (ResponseException e) {
             logger.error("Failed to create project: {}", project);
@@ -2777,6 +2828,23 @@ private Collection<String> getProjectIds(RestClient client) throws IOException {
         }
     }
 
+    protected void cleanUpProjects() throws IOException {
+        assert multiProjectEnabled;
+        final var projectIds = getProjectIds(adminClient());
+        for (String projectId : projectIds) {
+            if (projectId.equals(ProjectId.DEFAULT.id())) {
+                continue;
+            }
+            deleteProject(projectId);
+        }
+    }
+
+    private void deleteProject(String project) throws IOException {
+        assert multiProjectEnabled;
+        final Request request = new Request("DELETE", "/_project/" + project);
+        cleanupClient().performRequest(request);
+    }
+
     protected void assertEmptyProject(String projectId) throws IOException {
         assert multiProjectEnabled;
         final Request request = new Request("GET", "_cluster/state/metadata,routing_table,customs");

test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/DefaultLocalClusterSpecBuilder.java

@@ -20,6 +20,7 @@ public final class DefaultLocalClusterSpecBuilder extends AbstractLocalClusterSp
     public DefaultLocalClusterSpecBuilder() {
         super();
         this.apply(new FipsEnabledClusterConfigProvider());
+        this.apply(new MultiProjectEnabledClusterConfigProvider());
         this.systemProperties(new DefaultSystemPropertyProvider());
         this.settings(new DefaultSettingsProvider());
         this.environment(new DefaultEnvironmentProvider());

test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/MultiProjectEnabledClusterConfigProvider.java

@@ -0,0 +1,26 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.test.cluster.local;
+
+public class MultiProjectEnabledClusterConfigProvider implements LocalClusterConfigProvider {
+
+    @Override
+    public void apply(LocalClusterSpecBuilder<?> builder) {
+        if (isMultiProjectEnabled()) {
+            builder.setting("test.multi_project.enabled", "true").module("test-multi-project");
+        }
+    }
+
+    private static boolean isMultiProjectEnabled() {
+        // TODO: we need to use `tests` instead of `test` here to make gradle passes the system property,
+        // but we need `test` in the setting.
+        return Boolean.getBoolean("tests.multi_project.enabled");
+    }
+}