Skip to content

Commit 34b2907

Browse files
nielsbaumannielsbauman
committed
Allow internal cluster tests to run in MP mode
1 parent a2e580f commit 34b2907

File tree

2 files changed

+179
-5
lines changed

2 files changed

+179
-5
lines changed

‎test/framework/build.gradle‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@ dependencies {
3838
testImplementation project(':x-pack:plugin:mapper-counted-keyword')
3939
testImplementation project(':x-pack:plugin:mapper-constant-keyword')
4040
testImplementation project(':x-pack:plugin:wildcard')
41+
implementation project(':test:external-modules:test-multi-project')
4142
}
4243

4344
sourceSets {

‎test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java‎

Lines changed: 178 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,9 @@
1616
import com.carrotsearch.randomizedtesting.generators.RandomNumbers;
1717
import com.carrotsearch.randomizedtesting.generators.RandomPicks;
1818

19+
import org.apache.http.Header;
1920
import org.apache.http.HttpHost;
21+
import org.apache.http.message.BasicHeader;
2022
import org.apache.logging.log4j.Logger;
2123
import org.apache.lucene.search.Sort;
2224
import org.apache.lucene.search.TotalHits;
@@ -69,11 +71,15 @@
6971
import org.elasticsearch.action.support.RefCountingListener;
7072
import org.elasticsearch.action.support.SubscribableListener;
7173
import org.elasticsearch.action.support.broadcast.BroadcastResponse;
74+
import org.elasticsearch.client.Request;
75+
import org.elasticsearch.client.Response;
76+
import org.elasticsearch.client.ResponseException;
7277
import org.elasticsearch.client.RestClient;
7378
import org.elasticsearch.client.RestClientBuilder;
7479
import org.elasticsearch.client.internal.AdminClient;
7580
import org.elasticsearch.client.internal.Client;
7681
import org.elasticsearch.client.internal.ClusterAdminClient;
82+
import org.elasticsearch.client.internal.FilterClient;
7783
import org.elasticsearch.client.internal.IndicesAdminClient;
7884
import org.elasticsearch.cluster.ClusterInfoService;
7985
import org.elasticsearch.cluster.ClusterInfoServiceUtils;
@@ -85,6 +91,7 @@
8591
import org.elasticsearch.cluster.metadata.DataStream;
8692
import org.elasticsearch.cluster.metadata.IndexMetadata;
8793
import org.elasticsearch.cluster.metadata.Metadata;
94+
import org.elasticsearch.cluster.metadata.ProjectId;
8895
import org.elasticsearch.cluster.metadata.ProjectMetadata;
8996
import org.elasticsearch.cluster.node.DiscoveryNode;
9097
import org.elasticsearch.cluster.routing.IndexRoutingTable;
@@ -143,6 +150,7 @@
143150
import org.elasticsearch.indices.store.IndicesStore;
144151
import org.elasticsearch.ingest.IngestPipelineTestUtils;
145152
import org.elasticsearch.monitor.jvm.HotThreads;
153+
import org.elasticsearch.multiproject.TestOnlyMultiProjectPlugin;
146154
import org.elasticsearch.node.NodeMocksPlugin;
147155
import org.elasticsearch.persistent.PersistentTasks;
148156
import org.elasticsearch.persistent.PersistentTasksCustomMetadata;
@@ -156,9 +164,11 @@
156164
import org.elasticsearch.search.SearchHit;
157165
import org.elasticsearch.search.SearchResponseUtils;
158166
import org.elasticsearch.search.SearchService;
167+
import org.elasticsearch.tasks.Task;
159168
import org.elasticsearch.test.client.RandomizingClient;
160169
import org.elasticsearch.test.disruption.NetworkDisruption;
161170
import org.elasticsearch.test.disruption.ServiceDisruptionScheme;
171+
import org.elasticsearch.test.rest.ObjectPath;
162172
import org.elasticsearch.test.store.MockFSIndexStore;
163173
import org.elasticsearch.test.transport.MockTransportService;
164174
import org.elasticsearch.transport.TransportInterceptor;
@@ -171,6 +181,7 @@
171181
import org.elasticsearch.xcontent.XContentBuilder;
172182
import org.elasticsearch.xcontent.XContentParser;
173183
import org.elasticsearch.xcontent.XContentType;
184+
import org.elasticsearch.xcontent.json.JsonXContent;
174185
import org.elasticsearch.xcontent.smile.SmileXContent;
175186
import org.hamcrest.Matchers;
176187
import org.junit.After;
@@ -364,10 +375,20 @@ public abstract class ESIntegTestCase extends ESTestCase {
364375
private static ESIntegTestCase INSTANCE = null; // see @SuiteScope
365376
private static Long SUITE_SEED = null;
366377

378+
private static boolean multiProjectEnabled;
379+
private static ProjectId activeProject;
380+
private static Set<ProjectId> extraProjects;
381+
private static boolean projectsConfigured = false;
382+
367383
@BeforeClass
368384
public static void beforeClass() throws Exception {
369385
SUITE_SEED = randomLong();
370386
initializeSuiteScope();
387+
388+
// The active project-id is slightly longer, and has a fixed prefix so that it's easier to pick in error messages etc.
389+
activeProject = ProjectId.fromId("active00" + randomAlphaOfLength(8).toLowerCase(Locale.ROOT));
390+
extraProjects = randomSet(1, 3, () -> ProjectId.fromId(randomAlphaOfLength(12).toLowerCase(Locale.ROOT)));
391+
multiProjectEnabled = Boolean.parseBoolean(System.getProperty("tests.multi_project.enabled"));
371392
}
372393

373394
@Override
@@ -377,12 +398,68 @@ protected final boolean enableWarningsCheck() {
377398
return false;
378399
}
379400

401+
private void configureProjects() throws Exception {
402+
if (projectsConfigured || multiProjectEnabled == false) {
403+
return;
404+
}
405+
projectsConfigured = true;
406+
createProject(activeProject);
407+
for (var project : extraProjects) {
408+
createProject(project);
409+
}
410+
}
411+
412+
private void createProject(ProjectId project) throws IOException {
413+
assert multiProjectEnabled;
414+
final Request request = new Request("PUT", "/_project/" + project);
415+
try {
416+
final Response response = getRestClient().performRequest(request);
417+
logger.info("Created project {} : {}", project, response.getStatusLine());
418+
} catch (ResponseException e) {
419+
logger.error("Failed to create project: {}", project);
420+
throw e;
421+
}
422+
}
423+
424+
protected ProjectId activeProject() {
425+
if (multiProjectEnabled == false) {
426+
return ProjectId.DEFAULT;
427+
}
428+
return activeProject;
429+
}
430+
431+
private void checkSecurityIndex() throws IOException {
432+
final Request request = new Request("GET", "/_security/_query/role");
433+
request.setJsonEntity("""
434+
{
435+
"query": {
436+
"bool": {
437+
"must_not": {
438+
"term": {
439+
"metadata._reserved": true
440+
}
441+
}
442+
}
443+
}
444+
}""");
445+
request.setOptions(
446+
request.getOptions().toBuilder().addHeader(Task.X_ELASTIC_PROJECT_ID_HTTP_HEADER, Metadata.DEFAULT_PROJECT_ID.id()).build()
447+
);
448+
final var response = XContentHelper.convertToMap(
449+
JsonXContent.jsonXContent,
450+
getRestClient().performRequest(request).getEntity().getContent(),
451+
false
452+
);
453+
assertThat("Security index should not contain any non-reserved roles", (Collection<?>) response.get("roles"), empty());
454+
}
455+
380456
protected final void beforeInternal() throws Exception {
381457
final Scope currentClusterScope = getCurrentClusterScope();
382458
Callable<Void> setup = () -> {
383459
cluster().beforeTest(random());
384460
cluster().wipe(excludeTemplates());
385461
randomIndexTemplate();
462+
configureProjects();
386463
return null;
387464
};
388465
switch (currentClusterScope) {
@@ -2292,6 +2369,9 @@ private NodeConfigurationSource getNodeConfigSource() {
22922369
} else {
22932370
initialNodeSettings.put(SearchService.QUERY_PHASE_PARALLEL_COLLECTION_ENABLED.getKey(), false);
22942371
}
2372+
if (multiProjectEnabled) {
2373+
initialNodeSettings.put("test.multi_project.enabled", true);
2374+
}
22952375
return new NodeConfigurationSource() {
22962376
@Override
22972377
public Settings nodeSettings(int nodeOrdinal, Settings otherSettings) {
@@ -2308,12 +2388,14 @@ public Path nodeConfigPath(int nodeOrdinal) {
23082388

23092389
@Override
23102390
public Collection<Class<? extends Plugin>> nodePlugins() {
2391+
List<Class<? extends Plugin>> plugins = new ArrayList<>(ESIntegTestCase.this.nodePlugins());
23112392
if (enableConcurrentSearch) {
2312-
List<Class<? extends Plugin>> plugins = new ArrayList<>(ESIntegTestCase.this.nodePlugins());
23132393
plugins.add(ConcurrentSearchTestPlugin.class);
2314-
return plugins;
23152394
}
2316-
return ESIntegTestCase.this.nodePlugins();
2395+
if (multiProjectEnabled) {
2396+
plugins.add(TestOnlyMultiProjectPlugin.class);
2397+
}
2398+
return plugins;
23172399
}
23182400
};
23192401
}
@@ -2337,7 +2419,7 @@ protected boolean enableConcurrentSearch() {
23372419

23382420
/** Returns {@code true} iff this test cluster should use a dummy http transport */
23392421
protected boolean addMockHttpTransport() {
2340-
return true;
2422+
return false;
23412423
}
23422424

23432425
/**
@@ -2360,7 +2442,24 @@ protected boolean addMockFSIndexStore() {
23602442
* framework. By default this method returns an identity function {@link Function#identity()}.
23612443
*/
23622444
protected Function<Client, Client> getClientWrapper() {
2363-
return Function.identity();
2445+
return client -> new FilterClient(client) {
2446+
@Override
2447+
protected <Request extends ActionRequest, Response extends ActionResponse> void doExecute(
2448+
ActionType<Response> action,
2449+
Request request,
2450+
ActionListener<Response> listener
2451+
) {
2452+
if (projectsConfigured == false) {
2453+
super.doExecute(action, request, listener);
2454+
return;
2455+
}
2456+
Map<String, String> headers = Map.of(Task.X_ELASTIC_PROJECT_ID_HTTP_HEADER, activeProject.id());
2457+
ThreadContext threadContext = threadPool().getThreadContext();
2458+
try (ThreadContext.StoredContext ctx = threadContext.stashAndMergeHeaders(headers)) {
2459+
super.doExecute(action, request, listener);
2460+
}
2461+
}
2462+
};
23642463
}
23652464

23662465
/** Return the mock plugins the cluster should use */
@@ -2535,13 +2634,83 @@ public final void cleanUpCluster() throws Exception {
25352634
internalCluster().setBootstrapMasterNodeIndex(InternalTestCluster.BOOTSTRAP_MASTER_NODE_INDEX_AUTO);
25362635
}
25372636
super.ensureAllSearchContextsReleased();
2637+
assertEmptyProjects();
25382638
if (runTestScopeLifecycle()) {
25392639
printTestMessage("cleaning up after");
25402640
afterInternal(false);
25412641
printTestMessage("cleaned up after");
25422642
}
25432643
}
25442644

2645+
private void assertEmptyProjects() throws Exception {
2646+
if (projectsConfigured == false) {
2647+
return;
2648+
}
2649+
assertEmptyProject(Metadata.DEFAULT_PROJECT_ID);
2650+
for (var project : extraProjects) {
2651+
assertEmptyProject(project);
2652+
}
2653+
}
2654+
2655+
protected void assertEmptyProject(ProjectId projectId) throws IOException {
2656+
assert multiProjectEnabled;
2657+
final Request request = new Request("GET", "_cluster/state/metadata,routing_table,customs");
2658+
request.setOptions(request.getOptions().toBuilder().addHeader(Task.X_ELASTIC_PROJECT_ID_HTTP_HEADER, projectId.id()).build());
2659+
2660+
var response = XContentHelper.convertToMap(
2661+
JsonXContent.jsonXContent,
2662+
getRestClient().performRequest(request).getEntity().getContent(),
2663+
false
2664+
);
2665+
ObjectPath state = new ObjectPath(response);
2666+
2667+
final var indexNames = ((Map<?, ?>) state.evaluate("metadata.indices")).keySet();
2668+
final var routingTableEntries = ((Map<?, ?>) state.evaluate("routing_table.indices")).keySet();
2669+
if (indexNames.isEmpty() == false || routingTableEntries.isEmpty() == false) {
2670+
// Only the default project is allowed to have the security index after tests complete.
2671+
// The security index could show up in the indices, routing table, or both.
2672+
// If that happens, we need to check that it hasn't been modified by any leaking API calls.
2673+
if (projectId.equals(Metadata.DEFAULT_PROJECT_ID)
2674+
&& (indexNames.isEmpty() || (indexNames.size() == 1 && indexNames.contains(".security-7")))
2675+
&& (routingTableEntries.isEmpty() || (routingTableEntries.size() == 1 && routingTableEntries.contains(".security-7")))) {
2676+
checkSecurityIndex();
2677+
} else {
2678+
// If there are any other indices or if this is for a non-default project, we fail the test.
2679+
assertThat("Project [" + projectId + "] should not have indices", indexNames, empty());
2680+
assertThat("Project [" + projectId + "] should not have routing entries", routingTableEntries, empty());
2681+
}
2682+
}
2683+
assertThat(
2684+
"Project [" + projectId + "] should not have graveyard entries",
2685+
state.evaluate("metadata.index-graveyard.tombstones"),
2686+
empty()
2687+
);
2688+
2689+
final Map<String, ?> legacyTemplates = state.evaluate("metadata.templates");
2690+
if (legacyTemplates != null) {
2691+
var templateNames = legacyTemplates.keySet().stream().filter(name -> "random_index_template".equals(name) == false).toList();
2692+
assertThat("Project [" + projectId + "] should not have legacy templates", templateNames, empty());
2693+
}
2694+
2695+
final Map<String, Object> indexTemplates = state.evaluate("metadata.index_template.index_template");
2696+
if (indexTemplates != null) {
2697+
var templateNames = indexTemplates.keySet();
2698+
assertThat("Project [" + projectId + "] should not have index templates", templateNames, empty());
2699+
}
2700+
2701+
final Map<String, Object> componentTemplates = state.evaluate("metadata.component_template.component_template");
2702+
if (componentTemplates != null) {
2703+
var templateNames = componentTemplates.keySet();
2704+
assertThat("Project [" + projectId + "] should not have component templates", templateNames, empty());
2705+
}
2706+
2707+
final List<Map<String, ?>> pipelines = state.evaluate("metadata.ingest.pipeline");
2708+
if (pipelines != null) {
2709+
var pipelineNames = pipelines.stream().map(pipeline -> String.valueOf(pipeline.get("id"))).toList();
2710+
assertThat("Project [" + projectId + "] should not have ingest pipelines", pipelineNames, empty());
2711+
}
2712+
}
2713+
25452714
@Override
25462715
protected boolean enableBigArraysReleasedCheck() {
25472716
// checking that all big arrays have been released makes little sense for a still-running cluster, see comments in
@@ -2684,6 +2853,10 @@ protected static RestClient createRestClient(
26842853
if (httpClientConfigCallback != null) {
26852854
builder.setHttpClientConfigCallback(httpClientConfigCallback);
26862855
}
2856+
if (multiProjectEnabled) {
2857+
final var defaultHeaders = new Header[] { new BasicHeader(Task.X_ELASTIC_PROJECT_ID_HTTP_HEADER, activeProject.id()) };
2858+
builder.setDefaultHeaders(defaultHeaders);
2859+
}
26872860
return builder.build();
26882861
}
26892862

0 commit comments

Comments
 (0)