Fix single project GetClusterState API response for multiproject (#133936)

As it was noted in #132332 we have a general principle that API output for a single project should be the same regardless of whether the cluster is non-MP or MP. (However, the reserved metadata merge logic was never properly implemented to support this)
As a result of changes in that PR, response formats diverged. This PR makes the single project GetClusterState API response for MP clusters match the non-MP format.

Author: alexey-ivanov-es

server/src/main/java/org/elasticsearch/action/admin/cluster/state/ClusterStateRequest.java

@@ -39,6 +39,7 @@ public class ClusterStateRequest extends LocalClusterStateRequest implements Ind
     private TimeValue waitForTimeout = DEFAULT_WAIT_FOR_NODE_TIMEOUT;
     private String[] indices = Strings.EMPTY_ARRAY;
     private IndicesOptions indicesOptions = IndicesOptions.lenientExpandOpen();
+    private boolean multiproject = false;
 
     public ClusterStateRequest(TimeValue masterNodeTimeout) {
         super(masterNodeTimeout);
@@ -140,6 +141,15 @@ public boolean customs() {
         return customs;
     }
 
+    public ClusterStateRequest multiproject(boolean multiproject) {
+        this.multiproject = multiproject;
+        return this;
+    }
+
+    public boolean multiproject() {
+        return multiproject;
+    }
+
     public TimeValue waitForTimeout() {
         return waitForTimeout;
     }
@@ -200,5 +210,4 @@ public String getDescription() {
         stringBuilder.append("master timeout [").append(masterTimeout()).append("]]");
         return stringBuilder.toString();
     }
-
 }

server/src/main/java/org/elasticsearch/action/admin/cluster/state/ClusterStateRequestBuilder.java

@@ -108,4 +108,12 @@ public ClusterStateRequestBuilder setWaitForTimeOut(TimeValue waitForTimeout) {
         request.waitForTimeout(waitForTimeout);
         return this;
     }
+
+    /**
+     * When set then the response will be in multi-project format
+     */
+    public ClusterStateRequestBuilder setMultiproject(boolean multiproject) {
+        request.multiproject(multiproject);
+        return this;
+    }
 }

server/src/main/java/org/elasticsearch/action/admin/cluster/state/TransportClusterStateAction.java

@@ -28,6 +28,8 @@
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.metadata.ProjectId;
 import org.elasticsearch.cluster.metadata.ProjectMetadata;
+import org.elasticsearch.cluster.metadata.ReservedStateHandlerMetadata;
+import org.elasticsearch.cluster.metadata.ReservedStateMetadata;
 import org.elasticsearch.cluster.project.ProjectResolver;
 import org.elasticsearch.cluster.project.ProjectStateRegistry;
 import org.elasticsearch.cluster.routing.GlobalRoutingTable;
@@ -47,7 +49,9 @@
 
 import java.io.IOException;
 import java.util.Collection;
+import java.util.HashMap;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Set;
 import java.util.function.BiPredicate;
 import java.util.function.Predicate;
@@ -189,12 +193,12 @@ private static Map<String, Set<String>> getClusterFeatures(ClusterState clusterS
     }
 
     private ClusterStateResponse buildResponse(final ClusterStateRequest request, final ClusterState rawState) {
-        final ClusterState currentState = filterClusterState(rawState);
+        final ClusterState filteredState = filterClusterState(rawState);
 
         ThreadPool.assertCurrentThreadPool(ThreadPool.Names.MANAGEMENT); // too heavy to construct & serialize cluster state without forking
 
         if (request.blocks() == false) {
-            final var blockException = currentState.blocks().globalBlockedException(ClusterBlockLevel.METADATA_READ);
+            final var blockException = filteredState.blocks().globalBlockedException(ClusterBlockLevel.METADATA_READ);
             if (blockException != null) {
                 // There's a METADATA_READ block in place, but we aren't returning it to the caller, and yet the caller needs to know that
                 // this block exists (e.g. it's the STATE_NOT_RECOVERED_BLOCK, so the rest of the state is known to be incomplete). Thus we
@@ -203,22 +207,22 @@ private ClusterStateResponse buildResponse(final ClusterStateRequest request, fi
             }
         }
 
-        logger.trace("Serving cluster state request using version {}", currentState.version());
-        ClusterState.Builder builder = ClusterState.builder(currentState.getClusterName());
-        builder.version(currentState.version());
-        builder.stateUUID(currentState.stateUUID());
+        logger.trace("Serving cluster state request using version {}", filteredState.version());
+        ClusterState.Builder builder = ClusterState.builder(filteredState.getClusterName());
+        builder.version(filteredState.version());
+        builder.stateUUID(filteredState.stateUUID());
 
         if (request.nodes()) {
-            builder.nodes(currentState.nodes());
-            builder.nodeIdsToCompatibilityVersions(getCompatibilityVersions(currentState));
-            builder.nodeFeatures(getClusterFeatures(currentState));
+            builder.nodes(filteredState.nodes());
+            builder.nodeIdsToCompatibilityVersions(getCompatibilityVersions(filteredState));
+            builder.nodeFeatures(getClusterFeatures(filteredState));
         }
         if (request.routingTable()) {
             if (request.indices().length > 0) {
-                final GlobalRoutingTable.Builder globalRoutingTableBuilder = GlobalRoutingTable.builder(currentState.globalRoutingTable())
+                final GlobalRoutingTable.Builder globalRoutingTableBuilder = GlobalRoutingTable.builder(filteredState.globalRoutingTable())
                     .clear();
-                for (ProjectMetadata project : currentState.metadata().projects().values()) {
-                    RoutingTable projectRouting = currentState.routingTable(project.id());
+                for (ProjectMetadata project : filteredState.metadata().projects().values()) {
+                    RoutingTable projectRouting = filteredState.routingTable(project.id());
                     RoutingTable.Builder routingTableBuilder = RoutingTable.builder();
                     String[] indices = indexNameExpressionResolver.concreteIndexNames(project, request);
                     for (String filteredIndex : indices) {
@@ -230,18 +234,18 @@ private ClusterStateResponse buildResponse(final ClusterStateRequest request, fi
                 }
                 builder.routingTable(globalRoutingTableBuilder.build());
             } else {
-                builder.routingTable(currentState.globalRoutingTable());
+                builder.routingTable(filteredState.globalRoutingTable());
             }
         } else {
             builder.routingTable(GlobalRoutingTable.builder().build());
         }
         if (request.blocks()) {
-            builder.blocks(currentState.blocks());
+            builder.blocks(filteredState.blocks());
         }
 
         Metadata.Builder mdBuilder = Metadata.builder();
-        mdBuilder.clusterUUID(currentState.metadata().clusterUUID());
-        mdBuilder.coordinationMetadata(currentState.coordinationMetadata());
+        mdBuilder.clusterUUID(filteredState.metadata().clusterUUID());
+        mdBuilder.coordinationMetadata(filteredState.coordinationMetadata());
 
         if (request.metadata()) {
             // filter out metadata that shouldn't be returned by the API
@@ -250,14 +254,30 @@ private ClusterStateResponse buildResponse(final ClusterStateRequest request, fi
             if (request.indices().length > 0) {
                 // if the request specified index names, then we don't want the whole metadata, just the version and projects (which will
                 // be filtered (below) to only include the relevant indices)
-                mdBuilder.version(currentState.metadata().version());
+                mdBuilder.version(filteredState.metadata().version());
             } else {
                 // If there are no requested indices, then we want all the metadata, except for customs that aren't exposed via the API
-                mdBuilder = Metadata.builder(currentState.metadata());
+                mdBuilder = Metadata.builder(filteredState.metadata());
                 mdBuilder.removeCustomIf(notApi);
+
+                if (projectResolver.supportsMultipleProjects() && request.multiproject() == false) {
+                    ProjectStateRegistry projectStateRegistry = ProjectStateRegistry.get(filteredState);
+                    if (projectStateRegistry.size() > 1) {
+                        throw new Metadata.MultiProjectPendingException(
+                            "There are multiple projects " + projectStateRegistry.knownProjects()
+                        );
+                    }
+                    var reservedStateMetadata = new HashMap<>(filteredState.metadata().reservedStateMetadata());
+                    var singleProjectReservedStateMetadata = projectStateRegistry.reservedStateMetadata(projectResolver.getProjectId());
+                    singleProjectReservedStateMetadata.forEach(
+                        (key, value) -> reservedStateMetadata.merge(key, value, this::mergeReservedStateMetadata)
+                    );
+
+                    mdBuilder.put(reservedStateMetadata);
+                }
             }
 
-            for (ProjectMetadata project : currentState.metadata().projects().values()) {
+            for (ProjectMetadata project : filteredState.metadata().projects().values()) {
                 ProjectMetadata.Builder pBuilder;
                 if (request.indices().length > 0) {
                     // if the request specified index names, then only include the project-id and indices
@@ -289,7 +309,7 @@ private ClusterStateResponse buildResponse(final ClusterStateRequest request, fi
                 mdBuilder.put(pBuilder);
             }
         } else {
-            for (ProjectId project : currentState.metadata().projects().keySet()) {
+            for (ProjectId project : filteredState.metadata().projects().keySet()) {
                 // Request doesn't want to retrieve metadata, so we just fill in empty projects
                 // (because we can't have a truly empty Metadata)
                 mdBuilder.put(ProjectMetadata.builder(project));
@@ -298,14 +318,45 @@ private ClusterStateResponse buildResponse(final ClusterStateRequest request, fi
         builder.metadata(mdBuilder);
 
         if (request.customs()) {
-            for (Map.Entry<String, ClusterState.Custom> custom : currentState.customs().entrySet()) {
+            for (Map.Entry<String, ClusterState.Custom> custom : filteredState.customs().entrySet()) {
                 if (custom.getValue().isPrivate() == false) {
                     builder.putCustom(custom.getKey(), custom.getValue());
                 }
             }
         }
 
-        return new ClusterStateResponse(currentState.getClusterName(), builder.build(), false);
+        return new ClusterStateResponse(filteredState.getClusterName(), builder.build(), false);
     }
 
+    private ReservedStateMetadata mergeReservedStateMetadata(
+        ReservedStateMetadata clusterReservedMetadata,
+        ReservedStateMetadata projectReservedMetadata
+    ) {
+        if (Objects.equals(clusterReservedMetadata.version(), projectReservedMetadata.version()) == false) {
+            logger.info(
+                "Reserved state metadata version is different for Metadata ({}) and the requested project ({})",
+                clusterReservedMetadata.version(),
+                projectReservedMetadata.version()
+            );
+        }
+        ReservedStateMetadata.Builder builder = ReservedStateMetadata.builder(clusterReservedMetadata.namespace())
+            .version(Math.max(clusterReservedMetadata.version(), projectReservedMetadata.version()));
+
+        for (ReservedStateHandlerMetadata handler : clusterReservedMetadata.handlers().values()) {
+            builder.putHandler(handler);
+        }
+        for (Map.Entry<String, ReservedStateHandlerMetadata> handlerEntry : projectReservedMetadata.handlers().entrySet()) {
+            assert clusterReservedMetadata.handlers().containsKey(handlerEntry.getKey()) == false
+                : "Duplicate of handler: " + handlerEntry.getKey();
+            builder.putHandler(handlerEntry.getValue());
+        }
+
+        if (projectReservedMetadata.errorMetadata() != null) {
+            builder.errorMetadata(projectReservedMetadata.errorMetadata());
+        } else if (clusterReservedMetadata.errorMetadata() != null) {
+            builder.errorMetadata(clusterReservedMetadata.errorMetadata());
+        }
+
+        return builder.build();
+    }
 }

server/src/main/java/org/elasticsearch/cluster/project/ProjectStateRegistry.java

@@ -204,8 +204,7 @@ public long getProjectsMarkedForDeletionGeneration() {
         return projectsMarkedForDeletionGeneration;
     }
 
-    // visible for testing
-    Set<ProjectId> knownProjects() {
+    public Set<ProjectId> knownProjects() {
         return projectsEntries.keySet();
     }
 

server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterStateAction.java

@@ -112,6 +112,7 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC
         final Map<String, String> params;
         if (request.paramAsBoolean("multi_project", false)) {
             params = Map.of(Metadata.CONTEXT_MODE_PARAM, Metadata.CONTEXT_MODE_API, "multi-project", "true");
+            clusterStateRequest.multiproject(true);
         } else {
             params = Map.of(Metadata.CONTEXT_MODE_PARAM, Metadata.CONTEXT_MODE_API);
         }

server/src/test/java/org/elasticsearch/action/admin/cluster/state/TransportClusterStateActionTests.java

@@ -19,6 +19,8 @@
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.metadata.ProjectId;
 import org.elasticsearch.cluster.metadata.ProjectMetadata;
+import org.elasticsearch.cluster.metadata.ReservedStateHandlerMetadata;
+import org.elasticsearch.cluster.metadata.ReservedStateMetadata;
 import org.elasticsearch.cluster.node.VersionInformation;
 import org.elasticsearch.cluster.project.DefaultProjectResolver;
 import org.elasticsearch.cluster.project.ProjectResolver;
@@ -94,7 +96,7 @@ public void testGetClusterStateWithDefaultProjectOnly() throws Exception {
         final ProjectResolver projectResolver = DefaultProjectResolver.INSTANCE;
 
         final Set<String> indexNames = randomSet(1, 8, () -> randomAlphaOfLengthBetween(4, 12));
-        final ClusterStateRequest request = buildRandomRequest(indexNames);
+        final ClusterStateRequest request = buildRandomRequest(indexNames, false);
         final String[] expectedIndices = getExpectedIndices(request, indexNames);
 
         final ProjectId projectId = Metadata.DEFAULT_PROJECT_ID;
@@ -112,7 +114,7 @@ public void testGetClusterStateForOneProjectOfMany() throws Exception {
         final ProjectId projectId = randomUniqueProjectId();
 
         final ProjectResolver projectResolver = TestProjectResolvers.singleProject(projectId);
-        final ClusterStateRequest request = buildRandomRequest(indexNames);
+        final ClusterStateRequest request = buildRandomRequest(indexNames, false);
         final String[] expectedIndices = getExpectedIndices(request, indexNames);
 
         final int numberOfProjects = randomIntBetween(2, 5);
@@ -141,7 +143,7 @@ public void testGetClusterStateForManyProjects() throws Exception {
         final ClusterState state = buildClusterState(projects);
 
         final ProjectResolver projectResolver = TestProjectResolvers.allProjects();
-        final ClusterStateRequest request = buildRandomRequest(indexNames);
+        final ClusterStateRequest request = buildRandomRequest(indexNames, true);
         final Set<String> requestedIndices = Set.of(getExpectedIndices(request, indexNames));
 
         final ClusterStateResponse response = executeAction(projectResolver, request, state);
@@ -190,6 +192,22 @@ private static void assertSingleProjectResponse(
         assertThat(metadata.projects().keySet(), contains(projectId));
         if (request.metadata()) {
             assertThat(metadata.getProject(projectId).indices().keySet(), containsInAnyOrder(expectedIndices));
+
+            if (request.indices().length == 0) {
+                Map<String, ReservedStateMetadata> reservedStateMetadataMap = metadata.reservedStateMetadata();
+                assertThat(reservedStateMetadataMap, aMapWithSize(1));
+                ReservedStateMetadata fileSettings = reservedStateMetadataMap.get("file_settings");
+                assertNotNull(fileSettings);
+                assertThat(fileSettings.version(), equalTo(43L));
+                Map<String, ReservedStateHandlerMetadata> handlers = fileSettings.handlers();
+                assertThat(handlers, aMapWithSize(2));
+                ReservedStateHandlerMetadata clusterSettingsHandler = handlers.get("cluster_settings");
+                assertNotNull(clusterSettingsHandler);
+                assertThat(clusterSettingsHandler.keys(), containsInAnyOrder("setting_1", "setting_2"));
+                ReservedStateHandlerMetadata projectSettingsHandler = handlers.get("project_settings");
+                assertNotNull(projectSettingsHandler);
+                assertThat(projectSettingsHandler.keys(), containsInAnyOrder("setting_1"));
+            }
         } else {
             assertThat(metadata.getProject(projectId).indices(), anEmptyMap());
         }
@@ -235,7 +253,7 @@ private static String[] getExpectedIndices(ClusterStateRequest request, Set<Stri
         }
     }
 
-    private static ClusterStateRequest buildRandomRequest(Set<String> indexNames) {
+    private static ClusterStateRequest buildRandomRequest(Set<String> indexNames, boolean multipleProjects) {
         final ClusterStateRequest request = new ClusterStateRequest(TEST_REQUEST_TIMEOUT);
         if (randomBoolean()) {
             final int numberSelectedIndices = randomIntBetween(1, indexNames.size());
@@ -248,18 +266,31 @@ private static ClusterStateRequest buildRandomRequest(Set<String> indexNames) {
         request.routingTable(randomBoolean());
         request.blocks(randomBoolean());
         request.customs(true);
+        request.multiproject(multipleProjects);
         return request;
     }
 
     private static ClusterState buildClusterState(ProjectMetadata.Builder... projects) {
         final Metadata.Builder metadataBuilder = Metadata.builder();
+        metadataBuilder.put(
+            ReservedStateMetadata.builder("file_settings")
+                .version(43L)
+                .putHandler(new ReservedStateHandlerMetadata("cluster_settings", Set.of("setting_1", "setting_2")))
+                .build()
+        );
         Arrays.stream(projects).forEach(metadataBuilder::put);
         final var metadata = metadataBuilder.build();
 
         ClusterState.Builder csBuilder = ClusterState.builder(new ClusterName(randomAlphaOfLengthBetween(4, 12)));
         ProjectStateRegistry.Builder psBuilder = ProjectStateRegistry.builder();
         for (ProjectMetadata.Builder project : projects) {
-            psBuilder.putProjectSettings(project.getId(), Settings.builder().put("setting_1", randomIdentifier()).build());
+            psBuilder.putReservedStateMetadata(
+                project.getId(),
+                ReservedStateMetadata.builder("file_settings")
+                    .version(43L)
+                    .putHandler(new ReservedStateHandlerMetadata("project_settings", Set.of("setting_1")))
+                    .build()
+            ).putProjectSettings(project.getId(), Settings.builder().put("setting_1", randomIdentifier()).build());
         }
         return csBuilder.metadata(metadata)
             .routingTable(GlobalRoutingTableTestHelper.buildRoutingTable(metadata, RoutingTable.Builder::addAsNew))