Fix deleting unassigned indices in MP (#127750)

In some cases, deleting unassigned indices in multi-project mode would
result in an error log due to a `MultiProjectPendingException` which was
caused by the use of `Metadata#getProject()`. We fix that by passing the
correct (potentially null) project metadata instead.

Author: nielsbauman

server/src/main/java/org/elasticsearch/indices/IndicesService.java

@@ -1064,19 +1064,25 @@ public void afterIndexShardClosed(ShardId shardId, IndexShard indexShard, Settin
     /**
      * Deletes an index that is not assigned to this node. This method cleans up all disk folders relating to the index
      * but does not deal with in-memory structures. For those call {@link #removeIndex}
+     *
+     * @param reason the reason why this index should be deleted
+     * @param oldIndexMetadata the index metadata of the index that should be deleted
+     * @param currentProject the <i>current</i> project metadata which is used to verify that the index does not exist in the project
+     *                       anymore - can be null in case the whole project got deleted while there were still indices in it
      */
     @Override
-    public void deleteUnassignedIndex(String reason, IndexMetadata oldIndexMetadata, ClusterState clusterState) {
+    public void deleteUnassignedIndex(String reason, IndexMetadata oldIndexMetadata, @Nullable ProjectMetadata currentProject) {
         if (nodeEnv.hasNodeFile()) {
             Index index = oldIndexMetadata.getIndex();
             try {
-                if (clusterState.metadata().getProject().hasIndex(index)) {
-                    final IndexMetadata currentMetadata = clusterState.metadata().getProject().index(index);
+                if (currentProject != null && currentProject.hasIndex(index)) {
+                    final IndexMetadata currentMetadata = currentProject.index(index);
                     throw new IllegalStateException(
                         "Can't delete unassigned index store for ["
                             + index.getName()
-                            + "] - it's still part "
-                            + "of the cluster state ["
+                            + "] - it's still part of project ["
+                            + currentProject.id()
+                            + "] with UUIDs ["
                             + currentMetadata.getIndexUUID()
                             + "] ["
                             + oldIndexMetadata.getIndexUUID()

server/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java

@@ -400,7 +400,12 @@ private void deleteIndices(final ClusterChangedEvent event) {
                 indexServiceClosedListener = SubscribableListener.nullSuccess();
                 final IndexMetadata metadata = project.get().index(index);
                 indexSettings = new IndexSettings(metadata, settings);
-                indicesService.deleteUnassignedIndex("deleted index was not assigned to local node", metadata, state);
+                final var projectId = project.get().id();
+                indicesService.deleteUnassignedIndex(
+                    "deleted index in project [" + projectId + "] was not assigned to local node",
+                    metadata,
+                    state.metadata().projects().get(projectId)
+                );
             } else {
                 // The previous cluster state's metadata also does not contain the index,
                 // which is what happens on node startup when an index was deleted while the
@@ -1257,8 +1262,13 @@ U createIndex(IndexMetadata indexMetadata, List<IndexEventListener> builtInIndex
         /**
          * Deletes an index that is not assigned to this node. This method cleans up all disk folders relating to the index
          * but does not deal with in-memory structures. For those call {@link #removeIndex}
+         *
+         * @param reason the reason why this index should be deleted
+         * @param oldIndexMetadata the index metadata of the index that should be deleted
+         * @param currentProject the <i>current</i> project metadata which is used to verify that the index does not exist in the project
+         *                       anymore - can be null in case the whole project got deleted while there were still indices in it
          */
-        void deleteUnassignedIndex(String reason, IndexMetadata metadata, ClusterState clusterState);
+        void deleteUnassignedIndex(String reason, IndexMetadata oldIndexMetadata, @Nullable ProjectMetadata currentProject);
 
         /**
          * Removes the given index from this service and releases all associated resources. Persistent parts of the index

test/framework/src/main/java/org/elasticsearch/indices/cluster/AbstractIndicesClusterStateServiceTestCase.java

@@ -12,6 +12,7 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
+import org.elasticsearch.cluster.metadata.ProjectMetadata;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.routing.IndexShardRoutingTable;
 import org.elasticsearch.cluster.routing.RoutingNode;
@@ -213,7 +214,7 @@ public IndexMetadata verifyIndexIsDeleted(Index index, ClusterState state) {
         }
 
         @Override
-        public void deleteUnassignedIndex(String reason, IndexMetadata metadata, ClusterState clusterState) {
+        public void deleteUnassignedIndex(String reason, IndexMetadata oldIndexMetadata, ProjectMetadata currentProject) {
 
         }