Add multi-project support for health indicator shards_availability (elastic#125512)

Author: samxbr

benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/ShardsAvailabilityHealthIndicatorBenchmark.java

@@ -14,6 +14,7 @@
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.cluster.project.DefaultProjectResolver;
 import org.elasticsearch.cluster.routing.IndexRoutingTable;
 import org.elasticsearch.cluster.routing.IndexShardRoutingTable;
 import org.elasticsearch.cluster.routing.RecoverySource;
@@ -178,7 +179,12 @@ public void setUp() throws Exception {
             new TaskManager(Settings.EMPTY, threadPool, Collections.emptySet())
         );
         clusterService.getClusterApplierService().setInitialState(initialClusterState);
-        indicatorService = new ShardsAvailabilityHealthIndicatorService(clusterService, allocationService, new SystemIndices(List.of()));
+        indicatorService = new ShardsAvailabilityHealthIndicatorService(
+            clusterService,
+            allocationService,
+            new SystemIndices(List.of()),
+            DefaultProjectResolver.INSTANCE
+        );
     }
 
     private int toInt(String v) {

modules/health-shards-availability/src/main/java/org/elasticsearch/health/plugin/ShardsAvailabilityPlugin.java

@@ -27,7 +27,12 @@ public ShardsAvailabilityPlugin() {}
     @Override
     public Collection<?> createComponents(PluginServices services) {
         this.shardHealthService.set(
-            new ShardsAvailabilityHealthIndicatorService(services.clusterService(), services.allocationService(), services.systemIndices())
+            new ShardsAvailabilityHealthIndicatorService(
+                services.clusterService(),
+                services.allocationService(),
+                services.systemIndices(),
+                services.projectResolver()
+            )
         );
         return Set.of(this.shardHealthService.get());
     }

server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/allocation/shards/ShardsAvailabilityHealthIndicatorServiceIT.java

@@ -13,6 +13,7 @@
 import org.elasticsearch.cluster.ClusterChangedEvent;
 import org.elasticsearch.cluster.ClusterStateListener;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
+import org.elasticsearch.cluster.project.ProjectResolver;
 import org.elasticsearch.cluster.routing.RoutingNodes;
 import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.cluster.service.ClusterService;
@@ -133,8 +134,9 @@ private void assertHealthDuring(Matcher<HealthStatus> statusMatcher, Runnable ac
         var clusterService = internalCluster().getCurrentMasterNodeInstance(ClusterService.class);
         var allocationService = internalCluster().getCurrentMasterNodeInstance(AllocationService.class);
         var systemIndices = internalCluster().getCurrentMasterNodeInstance(SystemIndices.class);
+        var projectResolver = internalCluster().getCurrentMasterNodeInstance(ProjectResolver.class);
 
-        var service = new ShardsAvailabilityHealthIndicatorService(clusterService, allocationService, systemIndices);
+        var service = new ShardsAvailabilityHealthIndicatorService(clusterService, allocationService, systemIndices, projectResolver);
         var states = new ArrayList<RoutingNodesAndHealth>();
         var listener = new ClusterStateListener() {
             @Override

server/src/main/java/org/elasticsearch/cluster/routing/allocation/shards/ShardsAvailabilityHealthIndicatorService.java

@@ -10,6 +10,7 @@
 package org.elasticsearch.health;
 
 import org.elasticsearch.client.internal.node.NodeClient;
+import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.Scope;
@@ -19,6 +20,7 @@
 
 import java.io.IOException;
 import java.util.List;
+import java.util.Set;
 
 import static org.elasticsearch.rest.RestRequest.Method.GET;
 
@@ -29,6 +31,8 @@ public class RestGetHealthAction extends BaseRestHandler {
 
     private static final String SIZE_PARAM = "size";
 
+    private static final String CAPABILITY_MULTI_PROJECT_SHARDS_AVAILABILITY = "multi_project_shards_availability";
+
     @Override
     public String getName() {
         // TODO: Existing - "cluster_health_action", "cat_health_action"
@@ -57,4 +61,9 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli
     public boolean canTripCircuitBreaker() {
         return false;
     }
+
+    @Override
+    public Set<String> supportedCapabilities() {
+        return Sets.union(Set.of(CAPABILITY_MULTI_PROJECT_SHARDS_AVAILABILITY), super.supportedCapabilities());
+    }
 }

server/src/main/java/org/elasticsearch/health/RestGetHealthAction.java

@@ -11,6 +11,7 @@
 
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
+import org.elasticsearch.cluster.metadata.ProjectMetadata;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 
 import java.util.Collection;
@@ -47,6 +48,7 @@ public static String getNodeName(DiscoveryNode node) {
      * logging or user messages. The indices are sorted by priority and then by name to ensure a
      * deterministic message. If there are more indices than 10, it adds the '...' suffix.
      */
+    @Deprecated
     public static String getTruncatedIndices(Set<String> indices, Metadata clusterMetadata) {
         final int maxIndices = 10;
         String truncatedIndicesString = indices.stream()
@@ -59,6 +61,28 @@ public static String getTruncatedIndices(Set<String> indices, Metadata clusterMe
         return truncatedIndicesString;
     }
 
+    /**
+     * Creates a string that displays max 10 indices from the given set to be used as examples in
+     * logging or user messages. The indices are sorted by priority and then by name to ensure a
+     * deterministic message. If there are more indices than 10, it adds the '...' suffix.
+     */
+    public static String getTruncatedProjectIndices(
+        Set<ProjectIndexName> indices,
+        Metadata clusterMetadata,
+        boolean supportsMultipleProjects
+    ) {
+        final int maxIndices = 10;
+        String truncatedIndicesString = indices.stream()
+            .sorted(indicesComparatorByPriorityAndProjectIndex(clusterMetadata, supportsMultipleProjects))
+            .limit(maxIndices)
+            .map(projectIndexName -> projectIndexName.toString(supportsMultipleProjects))
+            .collect(joining(", "));
+        if (maxIndices < indices.size()) {
+            truncatedIndicesString = truncatedIndicesString + ", ...";
+        }
+        return truncatedIndicesString;
+    }
+
     /**
      * Creates a string that displays all the values that fulfilled the predicate sorted in the natural order.
      * @param values, the values to be displayed
@@ -119,11 +143,31 @@ public static String regularVerb(String verb, int count) {
      * @param clusterMetadata Used to look up index priority.
      * @return Comparator instance
      */
+    @Deprecated
     public static Comparator<String> indicesComparatorByPriorityAndName(Metadata clusterMetadata) {
         // We want to show indices with a numerically higher index.priority first (since lower priority ones might get truncated):
         return Comparator.comparingInt((String indexName) -> {
             IndexMetadata indexMetadata = clusterMetadata.getProject().index(indexName);
             return indexMetadata == null ? -1 : indexMetadata.priority();
         }).reversed().thenComparing(Comparator.naturalOrder());
     }
+
+    /**
+     * Sorts index names by their priority first, then alphabetically by name. If the priority cannot be determined for an index then
+     * a priority of -1 is used to sort it behind other index names.
+     * @param clusterMetadata Used to look up index priority.
+     * @param supportsMultipleProjects Whether cluster supports multi-project
+     * @return Comparator instance
+     */
+    public static Comparator<ProjectIndexName> indicesComparatorByPriorityAndProjectIndex(
+        Metadata clusterMetadata,
+        boolean supportsMultipleProjects
+    ) {
+        // We want to show indices with a numerically higher index.priority first (since lower priority ones might get truncated):
+        return Comparator.comparingInt((ProjectIndexName projectIndexName) -> {
+            ProjectMetadata projectMetadata = clusterMetadata.getProject(projectIndexName.projectId());
+            IndexMetadata indexMetadata = projectMetadata.index(projectIndexName.indexName());
+            return indexMetadata == null ? -1 : indexMetadata.priority();
+        }).reversed().thenComparing(projectIndex -> projectIndex.toString(supportsMultipleProjects));
+    }
 }

server/src/main/java/org/elasticsearch/health/node/HealthIndicatorDisplayValues.java

@@ -0,0 +1,35 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.health.node;
+
+import org.elasticsearch.cluster.metadata.ProjectId;
+
+public record ProjectIndexName(ProjectId projectId, String indexName) implements Comparable<ProjectIndexName> {
+    // VisibleForTesting
+    public static final String DELIMITER = "/";
+
+    @Override
+    public String toString() {
+        return toString(true);
+    }
+
+    public String toString(boolean withProjectId) {
+        if (withProjectId) {
+            return projectId.id() + DELIMITER + indexName;
+        } else {
+            return indexName;
+        }
+    }
+
+    @Override
+    public int compareTo(ProjectIndexName other) {
+        return this.toString().compareTo(other.toString());
+    }
+}

server/src/main/java/org/elasticsearch/health/node/ProjectIndexName.java

@@ -13,6 +13,7 @@
 import org.elasticsearch.cluster.metadata.ComposableIndexTemplate;
 import org.elasticsearch.cluster.metadata.DataStream;
 import org.elasticsearch.cluster.metadata.Metadata;
+import org.elasticsearch.cluster.metadata.ProjectMetadata;
 import org.elasticsearch.index.Index;
 
 import java.util.Collections;
@@ -106,8 +107,18 @@ public String getDataStreamName() {
      * @param metadata Metadata in which to look for indices
      * @return List of names of backing indices
      */
+    @Deprecated
     public List<String> getBackingIndexNames(Metadata metadata) {
-        DataStream dataStream = metadata.getProject().dataStreams().get(dataStreamName);
+        return getBackingIndexNames(metadata.getProject());
+    }
+
+    /**
+     * Retrieve backing indices for this system data stream
+     * @param projectMetadata Project metadata in which to look for indices
+     * @return List of names of backing indices
+     */
+    public List<String> getBackingIndexNames(ProjectMetadata projectMetadata) {
+        DataStream dataStream = projectMetadata.dataStreams().get(dataStreamName);
         if (dataStream == null) {
             return Collections.emptyList();
         }

server/src/main/java/org/elasticsearch/indices/SystemDataStreamDescriptor.java

@@ -9,6 +9,7 @@
 
 package org.elasticsearch.cluster.routing.allocation;
 
+import org.elasticsearch.cluster.project.ProjectResolver;
 import org.elasticsearch.cluster.routing.allocation.shards.ShardsAvailabilityHealthIndicatorService;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.settings.ClusterSettings;
@@ -46,7 +47,12 @@ public ShardsAvailabilityActionGuideTests() {
         ClusterService clusterService = mock(ClusterService.class);
         when(clusterService.getClusterSettings()).thenReturn(ClusterSettings.createBuiltInClusterSettings());
         when(clusterService.getSettings()).thenReturn(Settings.EMPTY);
-        service = new ShardsAvailabilityHealthIndicatorService(clusterService, mock(AllocationService.class), mock(SystemIndices.class));
+        service = new ShardsAvailabilityHealthIndicatorService(
+            clusterService,
+            mock(AllocationService.class),
+            mock(SystemIndices.class),
+            mock(ProjectResolver.class)
+        );
     }
 
     public void testRestoreFromSnapshotAction() {