Fail PutProject request for existing project (#127490)

Recreating an existing project breaks cluster state consistency. This PR
prevents it from happening.

Author: ywangd

test/external-modules/multi-project/src/javaRestTest/java/org/elasticsearch/multiproject/action/DeleteProjectActionIT.java

@@ -0,0 +1,102 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.multiproject.action;
+
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.Response;
+import org.elasticsearch.client.ResponseException;
+import org.elasticsearch.test.cluster.ElasticsearchCluster;
+import org.elasticsearch.test.cluster.local.distribution.DistributionType;
+import org.elasticsearch.test.rest.ESRestTestCase;
+import org.elasticsearch.test.rest.ObjectPath;
+import org.junit.ClassRule;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+
+import static org.hamcrest.CoreMatchers.containsString;
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.CoreMatchers.hasItem;
+import static org.hamcrest.CoreMatchers.not;
+
+public class ProjectCrudActionIT extends ESRestTestCase {
+
+    @ClassRule
+    public static ElasticsearchCluster CLUSTER = ElasticsearchCluster.local()
+        .distribution(DistributionType.DEFAULT)
+        .setting("test.multi_project.enabled", "true")
+        .setting("xpack.security.enabled", "false")
+        .build();
+
+    @Override
+    protected String getTestRestCluster() {
+        return CLUSTER.getHttpAddresses();
+    }
+
+    public void testCreateAndDeleteProject() throws IOException {
+        final var projectId = randomUniqueProjectId();
+        var request = new Request("PUT", "/_project/" + projectId);
+
+        final int numberOfRequests = between(1, 8);
+        final var successCount = new AtomicInteger();
+        final var errorCount = new AtomicInteger();
+
+        runInParallel(numberOfRequests, ignore -> {
+            try {
+                var response = client().performRequest(request);
+                assertAcknowledged(response);
+                successCount.incrementAndGet();
+            } catch (IOException e) {
+                if (e instanceof ResponseException responseException) {
+                    assertThat(responseException.getMessage(), containsString("project [" + projectId + "] already exists"));
+                    errorCount.incrementAndGet();
+                    return;
+                }
+                fail(e, "unexpected exception");
+            }
+        });
+
+        assertThat(successCount.get(), equalTo(1));
+        assertThat(errorCount.get(), equalTo(numberOfRequests - 1));
+        assertThat(getProjectIdsFromClusterState(), hasItem(projectId.id()));
+
+        final Response response = client().performRequest(new Request("DELETE", "/_project/" + projectId));
+        assertAcknowledged(response);
+        assertThat(getProjectIdsFromClusterState(), not(hasItem(projectId.id())));
+    }
+
+    private Set<String> getProjectIdsFromClusterState() throws IOException {
+        final Response response = client().performRequest(new Request("GET", "/_cluster/state?multi_project=true"));
+        final ObjectPath clusterState = assertOKAndCreateObjectPath(response);
+
+        final Set<String> projectIdsFromMetadata = extractProjectIds(clusterState, "metadata.projects");
+        final Set<String> projectIdsFromRoutingTable = extractProjectIds(clusterState, "routing_table.projects");
+
+        assertThat(projectIdsFromMetadata, equalTo(projectIdsFromRoutingTable));
+        return projectIdsFromMetadata;
+    }
+
+    @SuppressWarnings("unchecked")
+    private Set<String> extractProjectIds(ObjectPath clusterState, String path) throws IOException {
+        final int numberProjects = ((List<Object>) clusterState.evaluate(path)).size();
+        return IntStream.range(0, numberProjects).mapToObj(i -> {
+            try {
+                return (String) clusterState.evaluate(path + "." + i + ".id");
+            } catch (IOException e) {
+                throw new UncheckedIOException(e);
+            }
+        }).collect(Collectors.toUnmodifiableSet());
+    }
+}

test/external-modules/multi-project/src/javaRestTest/java/org/elasticsearch/multiproject/action/ProjectCrudActionIT.java

@@ -9,6 +9,7 @@
 
 package org.elasticsearch.multiproject.action;
 
+import org.elasticsearch.ResourceAlreadyExistsException;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.ActionType;
@@ -37,6 +38,8 @@
 import org.elasticsearch.transport.TransportService;
 
 import java.io.IOException;
+import java.util.HashSet;
+import java.util.Set;
 import java.util.regex.Pattern;
 
 public class PutProjectAction extends ActionType<AcknowledgedResponse> {
@@ -105,11 +108,17 @@ static class PutProjectExecutor implements ClusterStateTaskExecutor<PutProjectTa
 
         @Override
         public ClusterState execute(BatchExecutionContext<PutProjectTask> batchExecutionContext) throws Exception {
-            var stateBuilder = ClusterState.builder(batchExecutionContext.initialState());
+            final ClusterState initialState = batchExecutionContext.initialState();
+            final Set<ProjectId> knownProjectIds = new HashSet<>(initialState.metadata().projects().keySet());
+            var stateBuilder = ClusterState.builder(initialState);
             for (TaskContext<PutProjectTask> taskContext : batchExecutionContext.taskContexts()) {
                 try {
                     Request request = taskContext.getTask().request();
+                    if (knownProjectIds.contains(request.projectId)) {
+                        throw new ResourceAlreadyExistsException("project [{}] already exists", request.projectId);
+                    }
                     stateBuilder.putProjectMetadata(ProjectMetadata.builder(request.projectId));
+                    knownProjectIds.add(request.projectId);
                     taskContext.success(() -> taskContext.getTask().listener.onResponse(AcknowledgedResponse.TRUE));
                 } catch (Exception e) {
                     taskContext.onFailure(e);