From beb2c4851804e0fae695db2c25cb0bf2b4fa80ca Mon Sep 17 00:00:00 2001 From: Pete Gillin Date: Thu, 8 May 2025 16:56:46 +0100 Subject: [PATCH 1/3] Add multi-project IT for indices stats API This API already works correctly. This change just adds a test. --- .../stats/IndicesStatsMultiProjectIT.java | 165 ++++++++++++++++++ 1 file changed, 165 insertions(+) create mode 100644 test/external-modules/multi-project/src/javaRestTest/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsMultiProjectIT.java diff --git a/test/external-modules/multi-project/src/javaRestTest/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsMultiProjectIT.java b/test/external-modules/multi-project/src/javaRestTest/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsMultiProjectIT.java new file mode 100644 index 0000000000000..6351517517144 --- /dev/null +++ b/test/external-modules/multi-project/src/javaRestTest/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsMultiProjectIT.java @@ -0,0 +1,165 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.action.admin.indices.stats; + +import org.elasticsearch.client.Request; +import org.elasticsearch.client.ResponseException; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.settings.SecureString; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.multiproject.MultiProjectRestTestCase; +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.elasticsearch.test.cluster.local.distribution.DistributionType; +import org.elasticsearch.test.rest.ObjectPath; +import org.junit.ClassRule; + +import java.io.IOException; +import java.util.Map; + +import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.equalTo; + +public class IndicesStatsMultiProjectIT extends MultiProjectRestTestCase { + + private static final String PASSWORD = "hunter2"; + + @ClassRule + public static ElasticsearchCluster cluster = ElasticsearchCluster.local() + .nodes(1) + .distribution(DistributionType.INTEG_TEST) + .module("test-multi-project") + .setting("test.multi_project.enabled", "true") + .setting("xpack.security.enabled", "true") + .user("admin", PASSWORD) + .build(); + + @Override + protected String getTestRestCluster() { + return cluster.getHttpAddresses(); + } + + @Override + protected Settings restClientSettings() { + final String token = basicAuthHeaderValue("admin", new SecureString(PASSWORD.toCharArray())); + return Settings.builder().put(ThreadContext.PREFIX + ".Authorization", token).build(); + } + + public void testIndicesStats() throws IOException { + // Create two projects. We will use the default project as the third project in this test. + createProject("project-1"); + createProject("project-2"); + + // Create and write data into a number of indices. + // Some of the index names are used only in one project, some in two projects, some in all three. + int numDocs1Only = createPopulatedIndex("project-1", "my-index-project-1-only"); + int numDocs2Only = createPopulatedIndex("project-2", "my-index-project-2-only"); + int numDocsDefaultOnly = createPopulatedIndex("default", "my-index-default-project-only"); + int numDocs1Of1And2 = createPopulatedIndex("project-1", "my-index-projects-1-and-2"); + int numDocs2Of1And2 = createPopulatedIndex("project-2", "my-index-projects-1-and-2"); + int numDocs2Of2AndDefault = createPopulatedIndex("project-2", "my-index-projects-2-and-default"); + int numDocsDefaultOf2AndDefault = createPopulatedIndex("default", "my-index-projects-2-and-default"); + int numDocs1All = createPopulatedIndex("project-1", "my-index-all-projects"); + int numDocs2All = createPopulatedIndex("project-2", "my-index-all-projects"); + int numDocsDefaultAll = createPopulatedIndex("default", "my-index-all-projects"); + + // Check indices stats for project 1. + Map statsForProject1 = getAsOrderedMapInProject("/_stats", "project-1"); + assertThat(ObjectPath.evaluate(statsForProject1, "_all.total.docs.count"), equalTo(numDocs1Only + numDocs1Of1And2 + numDocs1All)); + assertThat( + ObjectPath.>evaluate(statsForProject1, "indices").keySet(), + containsInAnyOrder("my-index-project-1-only", "my-index-projects-1-and-2", "my-index-all-projects") + ); + assertThat(ObjectPath.evaluate(statsForProject1, "indices.my-index-project-1-only.total.docs.count"), equalTo(numDocs1Only)); + assertThat(ObjectPath.evaluate(statsForProject1, "indices.my-index-projects-1-and-2.total.docs.count"), equalTo(numDocs1Of1And2)); + assertThat(ObjectPath.evaluate(statsForProject1, "indices.my-index-all-projects.total.docs.count"), equalTo(numDocs1All)); + + // Check indices stats for project 2. + Map statsForProject2 = getAsOrderedMapInProject("/_stats", "project-2"); + assertThat( + ObjectPath.evaluate(statsForProject2, "_all.total.docs.count"), + equalTo(numDocs2Only + numDocs2Of1And2 + numDocs2Of2AndDefault + numDocs2All) + ); + assertThat( + ObjectPath.>evaluate(statsForProject2, "indices").keySet(), + containsInAnyOrder( + "my-index-project-2-only", + "my-index-projects-1-and-2", + "my-index-projects-2-and-default", + "my-index-all-projects" + ) + ); + assertThat(ObjectPath.evaluate(statsForProject2, "indices.my-index-all-projects.total.docs.count"), equalTo(numDocs2All)); + + // Check indices stats for default project. + Map statsForDefaultProject = getAsOrderedMap("/_stats"); + assertThat( + ObjectPath.evaluate(statsForDefaultProject, "_all.total.docs.count"), + equalTo(numDocsDefaultOnly + numDocsDefaultOf2AndDefault + numDocsDefaultAll) + ); + assertThat( + ObjectPath.>evaluate(statsForDefaultProject, "indices").keySet(), + containsInAnyOrder("my-index-default-project-only", "my-index-projects-2-and-default", "my-index-all-projects") + ); + assertThat( + ObjectPath.evaluate(statsForDefaultProject, "indices.my-index-all-projects.total.docs.count"), + equalTo(numDocsDefaultAll) + ); + + // Check single-index stats for each project. + assertThat( + ObjectPath.evaluate(getAsOrderedMapInProject("/my-index-all-projects/_stats", "project-1"), "_all.total.docs.count"), + equalTo(numDocs1All) + ); + assertThat( + ObjectPath.evaluate(getAsOrderedMapInProject("/my-index-all-projects/_stats", "project-2"), "_all.total.docs.count"), + equalTo(numDocs2All) + ); + assertThat( + ObjectPath.evaluate(getAsOrderedMap("/my-index-all-projects/_stats"), "_all.total.docs.count"), + equalTo(numDocsDefaultAll) + ); + + // Check that getting single-index stats for an index that does not exist in a project returns a 404. + ResponseException exception = assertThrows( + ResponseException.class, + () -> client().performRequest(setRequestProjectId(new Request("GET", "/my-index-project-1-only/_stats"), "project-2")) + ); + assertThat(exception.getResponse().getStatusLine().getStatusCode(), equalTo(404)); + + // Check using a wildcard gets the matching indices for that project. + Map statsWithWildcardForProject1 = getAsOrderedMapInProject("/my-index-project*/_stats", "project-1"); + assertThat(ObjectPath.evaluate(statsWithWildcardForProject1, "_all.total.docs.count"), equalTo(numDocs1Only + numDocs1Of1And2)); + assertThat( + ObjectPath.>evaluate(statsWithWildcardForProject1, "indices").keySet(), + containsInAnyOrder("my-index-project-1-only", "my-index-projects-1-and-2") + ); + } + + private int createPopulatedIndex(String projectId, String indexName) throws IOException { + createIndex(req -> { + setRequestProjectId(req, projectId); + return client().performRequest(req); + }, indexName, null, null, null); + int numDocs = randomIntBetween(5, 10); + for (int i = 0; i < numDocs; i++) { + Request request = new Request("POST", "/" + indexName + "/_doc"); + request.setJsonEntity(Strings.format("{ \"num\": %d, \"str\": \"%s\" }", randomInt(), randomAlphaOfLengthBetween(5, 10))); + setRequestProjectId(request, projectId); + client().performRequest(request); + } + client().performRequest(setRequestProjectId(new Request("POST", "/" + indexName + "/_refresh"), projectId)); + return numDocs; + } + + private static Map getAsOrderedMapInProject(String endpoint, String projectId) throws IOException { + return responseAsOrderedMap(client().performRequest(setRequestProjectId(new Request("GET", endpoint), projectId))); + } +} From eee1f34f5cddf18e26fa4b0c3506f6984770a0bd Mon Sep 17 00:00:00 2001 From: Pete Gillin Date: Fri, 9 May 2025 12:10:56 +0100 Subject: [PATCH 2/3] Update test/external-modules/multi-project/src/javaRestTest/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsMultiProjectIT.java Co-authored-by: Niels Bauman <33722607+nielsbauman@users.noreply.github.com> --- .../admin/indices/stats/IndicesStatsMultiProjectIT.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/external-modules/multi-project/src/javaRestTest/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsMultiProjectIT.java b/test/external-modules/multi-project/src/javaRestTest/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsMultiProjectIT.java index 6351517517144..bc4d2477e3490 100644 --- a/test/external-modules/multi-project/src/javaRestTest/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsMultiProjectIT.java +++ b/test/external-modules/multi-project/src/javaRestTest/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsMultiProjectIT.java @@ -71,8 +71,8 @@ public void testIndicesStats() throws IOException { int numDocsDefaultAll = createPopulatedIndex("default", "my-index-all-projects"); // Check indices stats for project 1. - Map statsForProject1 = getAsOrderedMapInProject("/_stats", "project-1"); - assertThat(ObjectPath.evaluate(statsForProject1, "_all.total.docs.count"), equalTo(numDocs1Only + numDocs1Of1And2 + numDocs1All)); + ObjectPath statsForProject1 = getAsObjectPathInProject("/_stats", "project-1"); + assertThat(statsForProject1.evaluate("_all.total.docs.count"), equalTo(numDocs1Only + numDocs1Of1And2 + numDocs1All)); assertThat( ObjectPath.>evaluate(statsForProject1, "indices").keySet(), containsInAnyOrder("my-index-project-1-only", "my-index-projects-1-and-2", "my-index-all-projects") From 0e14e19e59170d59b9b0f49aaf8a8a37bfd65b1a Mon Sep 17 00:00:00 2001 From: Pete Gillin Date: Fri, 9 May 2025 12:22:04 +0100 Subject: [PATCH 3/3] Finish up work from Niels review --- .../stats/IndicesStatsMultiProjectIT.java | 47 ++++++++++--------- 1 file changed, 24 insertions(+), 23 deletions(-) diff --git a/test/external-modules/multi-project/src/javaRestTest/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsMultiProjectIT.java b/test/external-modules/multi-project/src/javaRestTest/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsMultiProjectIT.java index bc4d2477e3490..cea82313b89cd 100644 --- a/test/external-modules/multi-project/src/javaRestTest/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsMultiProjectIT.java +++ b/test/external-modules/multi-project/src/javaRestTest/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsMultiProjectIT.java @@ -74,21 +74,21 @@ public void testIndicesStats() throws IOException { ObjectPath statsForProject1 = getAsObjectPathInProject("/_stats", "project-1"); assertThat(statsForProject1.evaluate("_all.total.docs.count"), equalTo(numDocs1Only + numDocs1Of1And2 + numDocs1All)); assertThat( - ObjectPath.>evaluate(statsForProject1, "indices").keySet(), + statsForProject1.>evaluate("indices").keySet(), containsInAnyOrder("my-index-project-1-only", "my-index-projects-1-and-2", "my-index-all-projects") ); - assertThat(ObjectPath.evaluate(statsForProject1, "indices.my-index-project-1-only.total.docs.count"), equalTo(numDocs1Only)); - assertThat(ObjectPath.evaluate(statsForProject1, "indices.my-index-projects-1-and-2.total.docs.count"), equalTo(numDocs1Of1And2)); - assertThat(ObjectPath.evaluate(statsForProject1, "indices.my-index-all-projects.total.docs.count"), equalTo(numDocs1All)); + assertThat(statsForProject1.evaluate("indices.my-index-project-1-only.total.docs.count"), equalTo(numDocs1Only)); + assertThat(statsForProject1.evaluate("indices.my-index-projects-1-and-2.total.docs.count"), equalTo(numDocs1Of1And2)); + assertThat(statsForProject1.evaluate("indices.my-index-all-projects.total.docs.count"), equalTo(numDocs1All)); // Check indices stats for project 2. - Map statsForProject2 = getAsOrderedMapInProject("/_stats", "project-2"); + ObjectPath statsForProject2 = getAsObjectPathInProject("/_stats", "project-2"); assertThat( - ObjectPath.evaluate(statsForProject2, "_all.total.docs.count"), + statsForProject2.evaluate("_all.total.docs.count"), equalTo(numDocs2Only + numDocs2Of1And2 + numDocs2Of2AndDefault + numDocs2All) ); assertThat( - ObjectPath.>evaluate(statsForProject2, "indices").keySet(), + statsForProject2.>evaluate("indices").keySet(), containsInAnyOrder( "my-index-project-2-only", "my-index-projects-1-and-2", @@ -96,34 +96,31 @@ public void testIndicesStats() throws IOException { "my-index-all-projects" ) ); - assertThat(ObjectPath.evaluate(statsForProject2, "indices.my-index-all-projects.total.docs.count"), equalTo(numDocs2All)); + assertThat(statsForProject2.evaluate("indices.my-index-all-projects.total.docs.count"), equalTo(numDocs2All)); // Check indices stats for default project. - Map statsForDefaultProject = getAsOrderedMap("/_stats"); + ObjectPath statsForDefaultProject = getAsObjectPathInDefaultProject("/_stats"); assertThat( - ObjectPath.evaluate(statsForDefaultProject, "_all.total.docs.count"), + statsForDefaultProject.evaluate("_all.total.docs.count"), equalTo(numDocsDefaultOnly + numDocsDefaultOf2AndDefault + numDocsDefaultAll) ); assertThat( - ObjectPath.>evaluate(statsForDefaultProject, "indices").keySet(), + statsForDefaultProject.>evaluate("indices").keySet(), containsInAnyOrder("my-index-default-project-only", "my-index-projects-2-and-default", "my-index-all-projects") ); - assertThat( - ObjectPath.evaluate(statsForDefaultProject, "indices.my-index-all-projects.total.docs.count"), - equalTo(numDocsDefaultAll) - ); + assertThat(statsForDefaultProject.evaluate("indices.my-index-all-projects.total.docs.count"), equalTo(numDocsDefaultAll)); // Check single-index stats for each project. assertThat( - ObjectPath.evaluate(getAsOrderedMapInProject("/my-index-all-projects/_stats", "project-1"), "_all.total.docs.count"), + getAsObjectPathInProject("/my-index-all-projects/_stats", "project-1").evaluate("_all.total.docs.count"), equalTo(numDocs1All) ); assertThat( - ObjectPath.evaluate(getAsOrderedMapInProject("/my-index-all-projects/_stats", "project-2"), "_all.total.docs.count"), + getAsObjectPathInProject("/my-index-all-projects/_stats", "project-2").evaluate("_all.total.docs.count"), equalTo(numDocs2All) ); assertThat( - ObjectPath.evaluate(getAsOrderedMap("/my-index-all-projects/_stats"), "_all.total.docs.count"), + getAsObjectPathInDefaultProject("/my-index-all-projects/_stats").evaluate("_all.total.docs.count"), equalTo(numDocsDefaultAll) ); @@ -135,10 +132,10 @@ public void testIndicesStats() throws IOException { assertThat(exception.getResponse().getStatusLine().getStatusCode(), equalTo(404)); // Check using a wildcard gets the matching indices for that project. - Map statsWithWildcardForProject1 = getAsOrderedMapInProject("/my-index-project*/_stats", "project-1"); - assertThat(ObjectPath.evaluate(statsWithWildcardForProject1, "_all.total.docs.count"), equalTo(numDocs1Only + numDocs1Of1And2)); + ObjectPath statsWithWildcardForProject1 = getAsObjectPathInProject("/my-index-project*/_stats", "project-1"); + assertThat(statsWithWildcardForProject1.evaluate("_all.total.docs.count"), equalTo(numDocs1Only + numDocs1Of1And2)); assertThat( - ObjectPath.>evaluate(statsWithWildcardForProject1, "indices").keySet(), + statsWithWildcardForProject1.>evaluate("indices").keySet(), containsInAnyOrder("my-index-project-1-only", "my-index-projects-1-and-2") ); } @@ -159,7 +156,11 @@ private int createPopulatedIndex(String projectId, String indexName) throws IOEx return numDocs; } - private static Map getAsOrderedMapInProject(String endpoint, String projectId) throws IOException { - return responseAsOrderedMap(client().performRequest(setRequestProjectId(new Request("GET", endpoint), projectId))); + private static ObjectPath getAsObjectPathInProject(String endpoint, String projectId) throws IOException { + return new ObjectPath(responseAsOrderedMap(client().performRequest(setRequestProjectId(new Request("GET", endpoint), projectId)))); + } + + private static ObjectPath getAsObjectPathInDefaultProject(String endpoint) throws IOException { + return new ObjectPath(getAsMap(endpoint)); } }