diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/ShardsAvailabilityHealthIndicatorBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/ShardsAvailabilityHealthIndicatorBenchmark.java index 4f2914716b58f..8a6d17d1f47f1 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/ShardsAvailabilityHealthIndicatorBenchmark.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/ShardsAvailabilityHealthIndicatorBenchmark.java @@ -14,6 +14,7 @@ import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.node.DiscoveryNodes; +import org.elasticsearch.cluster.project.DefaultProjectResolver; import org.elasticsearch.cluster.routing.IndexRoutingTable; import org.elasticsearch.cluster.routing.IndexShardRoutingTable; import org.elasticsearch.cluster.routing.RecoverySource; @@ -178,7 +179,12 @@ public void setUp() throws Exception { new TaskManager(Settings.EMPTY, threadPool, Collections.emptySet()) ); clusterService.getClusterApplierService().setInitialState(initialClusterState); - indicatorService = new ShardsAvailabilityHealthIndicatorService(clusterService, allocationService, new SystemIndices(List.of())); + indicatorService = new ShardsAvailabilityHealthIndicatorService( + clusterService, + allocationService, + new SystemIndices(List.of()), + DefaultProjectResolver.INSTANCE + ); } private int toInt(String v) { diff --git a/modules/health-shards-availability/src/main/java/org/elasticsearch/health/plugin/ShardsAvailabilityPlugin.java b/modules/health-shards-availability/src/main/java/org/elasticsearch/health/plugin/ShardsAvailabilityPlugin.java index 33a31d48b3d32..66afd6465c178 100644 --- a/modules/health-shards-availability/src/main/java/org/elasticsearch/health/plugin/ShardsAvailabilityPlugin.java +++ b/modules/health-shards-availability/src/main/java/org/elasticsearch/health/plugin/ShardsAvailabilityPlugin.java @@ -27,7 +27,12 @@ public ShardsAvailabilityPlugin() {} @Override public Collection createComponents(PluginServices services) { this.shardHealthService.set( - new ShardsAvailabilityHealthIndicatorService(services.clusterService(), services.allocationService(), services.systemIndices()) + new ShardsAvailabilityHealthIndicatorService( + services.clusterService(), + services.allocationService(), + services.systemIndices(), + services.projectResolver() + ) ); return Set.of(this.shardHealthService.get()); } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/allocation/shards/ShardsAvailabilityHealthIndicatorServiceIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/allocation/shards/ShardsAvailabilityHealthIndicatorServiceIT.java index d4294ac759433..05b33098c3bca 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/allocation/shards/ShardsAvailabilityHealthIndicatorServiceIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/allocation/shards/ShardsAvailabilityHealthIndicatorServiceIT.java @@ -13,6 +13,7 @@ import org.elasticsearch.cluster.ClusterChangedEvent; import org.elasticsearch.cluster.ClusterStateListener; import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.project.ProjectResolver; import org.elasticsearch.cluster.routing.RoutingNodes; import org.elasticsearch.cluster.routing.allocation.AllocationService; import org.elasticsearch.cluster.service.ClusterService; @@ -133,8 +134,9 @@ private void assertHealthDuring(Matcher statusMatcher, Runnable ac var clusterService = internalCluster().getCurrentMasterNodeInstance(ClusterService.class); var allocationService = internalCluster().getCurrentMasterNodeInstance(AllocationService.class); var systemIndices = internalCluster().getCurrentMasterNodeInstance(SystemIndices.class); + var projectResolver = internalCluster().getCurrentMasterNodeInstance(ProjectResolver.class); - var service = new ShardsAvailabilityHealthIndicatorService(clusterService, allocationService, systemIndices); + var service = new ShardsAvailabilityHealthIndicatorService(clusterService, allocationService, systemIndices, projectResolver); var states = new ArrayList(); var listener = new ClusterStateListener() { @Override diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/shards/ShardsAvailabilityHealthIndicatorService.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/shards/ShardsAvailabilityHealthIndicatorService.java index 6267ec8a01a63..1ce1522acb90c 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/shards/ShardsAvailabilityHealthIndicatorService.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/shards/ShardsAvailabilityHealthIndicatorService.java @@ -18,13 +18,16 @@ import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.metadata.NodesShutdownMetadata; +import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.cluster.metadata.SingleNodeShutdownMetadata; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodeFilters; import org.elasticsearch.cluster.node.DiscoveryNodeRole; +import org.elasticsearch.cluster.project.ProjectResolver; import org.elasticsearch.cluster.routing.IndexRoutingTable; import org.elasticsearch.cluster.routing.IndexShardRoutingTable; import org.elasticsearch.cluster.routing.RoutingNode; +import org.elasticsearch.cluster.routing.RoutingTable; import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.cluster.routing.UnassignedInfo; import org.elasticsearch.cluster.routing.allocation.AllocateUnassignedDecision; @@ -55,6 +58,7 @@ import org.elasticsearch.health.ImpactArea; import org.elasticsearch.health.SimpleHealthIndicatorDetails; import org.elasticsearch.health.node.HealthInfo; +import org.elasticsearch.health.node.ProjectIndexName; import org.elasticsearch.indices.SystemIndices; import org.elasticsearch.snapshots.SearchableSnapshotsSettings; import org.elasticsearch.snapshots.SnapshotShardSizeInfo; @@ -72,7 +76,6 @@ import java.util.function.Predicate; import java.util.stream.Collectors; import java.util.stream.Stream; -import java.util.stream.StreamSupport; import static java.util.stream.Collectors.joining; import static java.util.stream.Collectors.toMap; @@ -90,8 +93,8 @@ import static org.elasticsearch.health.HealthStatus.GREEN; import static org.elasticsearch.health.HealthStatus.RED; import static org.elasticsearch.health.HealthStatus.YELLOW; -import static org.elasticsearch.health.node.HealthIndicatorDisplayValues.getTruncatedIndices; -import static org.elasticsearch.health.node.HealthIndicatorDisplayValues.indicesComparatorByPriorityAndName; +import static org.elasticsearch.health.node.HealthIndicatorDisplayValues.getTruncatedProjectIndices; +import static org.elasticsearch.health.node.HealthIndicatorDisplayValues.indicesComparatorByPriorityAndProjectIndex; /** * This indicator reports health for shards. @@ -132,19 +135,22 @@ public class ShardsAvailabilityHealthIndicatorService implements HealthIndicator private final AllocationService allocationService; private final SystemIndices systemIndices; + protected final ProjectResolver projectResolver; private volatile TimeValue replicaUnassignedBufferTime; public ShardsAvailabilityHealthIndicatorService( ClusterService clusterService, AllocationService allocationService, - SystemIndices systemIndices + SystemIndices systemIndices, + ProjectResolver projectResolver ) { this.clusterService = clusterService; this.allocationService = allocationService; this.systemIndices = systemIndices; this.replicaUnassignedBufferTime = REPLICA_UNASSIGNED_BUFFER_TIME.get(clusterService.getSettings()); clusterService.getClusterSettings().addSettingsUpdateConsumer(REPLICA_UNASSIGNED_BUFFER_TIME, this::setReplicaUnassignedBufferTime); + this.projectResolver = projectResolver; } private void setReplicaUnassignedBufferTime(TimeValue replicaUnassignedBufferTime) { @@ -189,12 +195,17 @@ static void updateShardAllocationStatus( boolean verbose, TimeValue replicaUnassignedBufferTime ) { - for (IndexRoutingTable indexShardRouting : state.globalRoutingTable().indexRouting()) { - for (int i = 0; i < indexShardRouting.size(); i++) { - IndexShardRoutingTable shardRouting = indexShardRouting.shard(i); - status.addPrimary(shardRouting.primaryShard(), state, shutdown, verbose); - for (ShardRouting replicaShard : shardRouting.replicaShards()) { - status.addReplica(replicaShard, state, shutdown, verbose, replicaUnassignedBufferTime); + for (Map.Entry entries : state.globalRoutingTable().routingTables().entrySet()) { + ProjectId projectId = entries.getKey(); + RoutingTable projectRoutingTable = entries.getValue(); + + for (IndexRoutingTable indexShardRouting : projectRoutingTable.indicesRouting().values()) { + for (int i = 0; i < indexShardRouting.size(); i++) { + IndexShardRoutingTable shardRouting = indexShardRouting.shard(i); + status.addPrimary(projectId, shardRouting.primaryShard(), state, shutdown, verbose); + for (ShardRouting replicaShard : shardRouting.replicaShards()) { + status.addReplica(projectId, replicaShard, state, shutdown, verbose, replicaUnassignedBufferTime); + } } } } @@ -460,35 +471,37 @@ public class ShardAllocationCounts { int initializing = 0; int started = 0; int relocating = 0; - public final Set indicesWithUnavailableShards = new HashSet<>(); - public final Set indicesWithAllShardsUnavailable = new HashSet<>(); + public final Set indicesWithUnavailableShards = new HashSet<>(); + public final Set indicesWithAllShardsUnavailable = new HashSet<>(); // We keep the searchable snapshots separately as long as the original index is still available // This is checked during the post-processing public SearchableSnapshotsState searchableSnapshotsState = new SearchableSnapshotsState(); - final Map> diagnosisDefinitions = new HashMap<>(); + final Map> diagnosisDefinitions = new HashMap<>(); public void increment( + ProjectId projectId, ShardRouting routing, ClusterState state, NodesShutdownMetadata shutdowns, boolean verbose, TimeValue replicaUnassignedBufferTime ) { - boolean isNew = isUnassignedDueToNewInitialization(routing, state); + boolean isNew = isUnassignedDueToNewInitialization(projectId, routing, state); boolean isRestarting = isUnassignedDueToTimelyRestart(routing, shutdowns); long replicaUnassignedCutoffTime = Instant.now().toEpochMilli() - replicaUnassignedBufferTime.millis(); - boolean allUnavailable = areAllShardsOfThisTypeUnavailable(routing, state) - && isNewlyCreatedAndInitializingReplica(routing, state, replicaUnassignedCutoffTime) == false; + boolean allUnavailable = areAllShardsOfThisTypeUnavailable(projectId, routing, state) + && isNewlyCreatedAndInitializingReplica(projectId, routing, state, replicaUnassignedCutoffTime) == false; + + ProjectIndexName projectIndex = new ProjectIndexName(projectId, routing.getIndexName()); if (allUnavailable) { - indicesWithAllShardsUnavailable.add(routing.getIndexName()); + indicesWithAllShardsUnavailable.add(projectIndex); } if ((routing.active() || isRestarting || isNew) == false) { - String indexName = routing.getIndexName(); - Settings indexSettings = state.metadata().indexMetadata(routing.index()).getSettings(); + Settings indexSettings = state.metadata().getProject(projectId).index(routing.index()).getSettings(); if (SearchableSnapshotsSettings.isSearchableSnapshotStore(indexSettings)) { - searchableSnapshotsState.addSearchableSnapshotWithUnavailableShard(indexName); + searchableSnapshotsState.addSearchableSnapshotWithUnavailableShard(projectIndex); } else { - indicesWithUnavailableShards.add(indexName); + indicesWithUnavailableShards.add(projectIndex); } } @@ -501,16 +514,14 @@ public void increment( } else { unassigned++; if (verbose) { - diagnoseUnassignedShardRouting(routing, state).forEach( - definition -> addDefinition(definition, routing.getIndexName()) - ); + diagnoseUnassignedShardRouting(routing, state).forEach(definition -> addDefinition(definition, projectIndex)); } } } case INITIALIZING -> { initializing++; if (verbose) { - addDefinition(DIAGNOSIS_WAIT_FOR_INITIALIZATION, routing.getIndexName()); + addDefinition(DIAGNOSIS_WAIT_FOR_INITIALIZATION, projectIndex); } } case STARTED -> started++; @@ -526,8 +537,8 @@ public boolean doAnyIndicesHaveAllUnavailable() { return indicesWithAllShardsUnavailable.isEmpty() == false; } - private void addDefinition(Diagnosis.Definition diagnosisDefinition, String indexName) { - diagnosisDefinitions.computeIfAbsent(diagnosisDefinition, (k) -> new HashSet<>()).add(indexName); + private void addDefinition(Diagnosis.Definition diagnosisDefinition, ProjectIndexName projectIndexName) { + diagnosisDefinitions.computeIfAbsent(diagnosisDefinition, (k) -> new HashSet<>()).add(projectIndexName); } } @@ -536,11 +547,10 @@ private void addDefinition(Diagnosis.Definition diagnosisDefinition, String inde * example: if a replica is passed then this will return true if ALL replicas are unassigned, * but if at least one is assigned, it will return false. */ - boolean areAllShardsOfThisTypeUnavailable(ShardRouting routing, ClusterState state) { - return StreamSupport.stream( - state.routingTable().allActiveShardsGrouped(new String[] { routing.getIndexName() }, true).spliterator(), - false - ) + boolean areAllShardsOfThisTypeUnavailable(ProjectId projectId, ShardRouting routing, ClusterState state) { + return state.routingTable(projectId) + .allActiveShardsGrouped(new String[] { routing.getIndexName() }, true) + .stream() .flatMap(shardIter -> shardIter.getShardRoutings().stream()) .filter(sr -> sr.shardId().equals(routing.shardId())) .filter(sr -> sr.primary() == routing.primary()) @@ -551,19 +561,23 @@ boolean areAllShardsOfThisTypeUnavailable(ShardRouting routing, ClusterState sta * Returns true if the given shard is a replica that is only unassigned due to its primary being * newly created. See {@link ClusterShardHealth#getInactivePrimaryHealth(ShardRouting)} for more * information. - * * We use this information when considering whether a cluster should turn red. For some cases * (a newly created index having unassigned replicas for example), we don't want the cluster * to turn "unhealthy" for the tiny amount of time before the shards are allocated. */ - static boolean isNewlyCreatedAndInitializingReplica(ShardRouting routing, ClusterState state, long replicaUnassignedCutoffTime) { + static boolean isNewlyCreatedAndInitializingReplica( + ProjectId projectId, + ShardRouting routing, + ClusterState state, + long replicaUnassignedCutoffTime + ) { if (routing.active()) { return false; } if (routing.primary()) { return false; } - ShardRouting primary = state.routingTable().shardRoutingTable(routing.shardId()).primaryShard(); + ShardRouting primary = state.routingTable(projectId).shardRoutingTable(routing.shardId()).primaryShard(); if (primary.active() == false) { return ClusterShardHealth.getInactivePrimaryHealth(primary) == ClusterHealthStatus.YELLOW; } @@ -589,13 +603,15 @@ private static boolean isUnassignedDueToTimelyRestart(ShardRouting routing, Node return now - restartingAllocationDelayExpiration <= 0; } - private static boolean isUnassignedDueToNewInitialization(ShardRouting routing, ClusterState state) { + private static boolean isUnassignedDueToNewInitialization(ProjectId projectId, ShardRouting routing, ClusterState state) { if (routing.active()) { return false; } // If the primary is inactive for unexceptional events in the cluster lifecycle, both the primary and the // replica are considered new initializations. - ShardRouting primary = routing.primary() ? routing : state.routingTable().shardRoutingTable(routing.shardId()).primaryShard(); + ShardRouting primary = routing.primary() + ? routing + : state.routingTable(projectId).shardRoutingTable(routing.shardId()).primaryShard(); return primary.active() == false && getInactivePrimaryHealth(primary) == ClusterHealthStatus.YELLOW; } @@ -950,18 +966,19 @@ public ShardAllocationStatus(Metadata clusterMetadata) { this.clusterMetadata = clusterMetadata; } - void addPrimary(ShardRouting routing, ClusterState state, NodesShutdownMetadata shutdowns, boolean verbose) { - primaries.increment(routing, state, shutdowns, verbose, TimeValue.MINUS_ONE); + void addPrimary(ProjectId projectId, ShardRouting routing, ClusterState state, NodesShutdownMetadata shutdowns, boolean verbose) { + primaries.increment(projectId, routing, state, shutdowns, verbose, TimeValue.MINUS_ONE); } void addReplica( + ProjectId projectId, ShardRouting routing, ClusterState state, NodesShutdownMetadata shutdowns, boolean verbose, TimeValue replicaUnassignedBufferTime ) { - replicas.increment(routing, state, shutdowns, verbose, replicaUnassignedBufferTime); + replicas.increment(projectId, routing, state, shutdowns, verbose, replicaUnassignedBufferTime); } void updateSearchableSnapshotsOfAvailableIndices() { @@ -1068,7 +1085,11 @@ public List getImpacts() { "Cannot add data to %d %s [%s]. Searches might return incomplete results.", primaries.indicesWithUnavailableShards.size(), primaries.indicesWithUnavailableShards.size() == 1 ? "index" : "indices", - getTruncatedIndices(primaries.indicesWithUnavailableShards, clusterMetadata) + getTruncatedProjectIndices( + primaries.indicesWithUnavailableShards, + clusterMetadata, + projectResolver.supportsMultipleProjects() + ) ); impacts.add( new HealthIndicatorImpact( @@ -1080,14 +1101,18 @@ public List getImpacts() { ) ); } - Set readOnlyIndicesWithUnavailableShards = primaries.searchableSnapshotsState.getRedSearchableSnapshots(); + Set readOnlyIndicesWithUnavailableShards = primaries.searchableSnapshotsState.getRedSearchableSnapshots(); if (readOnlyIndicesWithUnavailableShards.isEmpty() == false) { String impactDescription = String.format( Locale.ROOT, "Searching %d %s [%s] might return incomplete results.", readOnlyIndicesWithUnavailableShards.size(), readOnlyIndicesWithUnavailableShards.size() == 1 ? "index" : "indices", - getTruncatedIndices(readOnlyIndicesWithUnavailableShards, clusterMetadata) + getTruncatedProjectIndices( + readOnlyIndicesWithUnavailableShards, + clusterMetadata, + projectResolver.supportsMultipleProjects() + ) ); impacts.add( new HealthIndicatorImpact( @@ -1104,7 +1129,7 @@ public List getImpacts() { * that is reported as unavailable. That replica is likely being promoted to primary. The only impact that matters at this * point is the one above, which has already been reported for this index. */ - Set indicesWithUnavailableReplicasOnly = new HashSet<>(replicas.indicesWithUnavailableShards); + Set indicesWithUnavailableReplicasOnly = new HashSet<>(replicas.indicesWithUnavailableShards); indicesWithUnavailableReplicasOnly.removeAll(primaries.indicesWithUnavailableShards); if (indicesWithUnavailableReplicasOnly.isEmpty() == false) { String impactDescription = String.format( @@ -1112,7 +1137,11 @@ public List getImpacts() { "Searches might be slower than usual. Fewer redundant copies of the data exist on %d %s [%s].", indicesWithUnavailableReplicasOnly.size(), indicesWithUnavailableReplicasOnly.size() == 1 ? "index" : "indices", - getTruncatedIndices(indicesWithUnavailableReplicasOnly, clusterMetadata) + getTruncatedProjectIndices( + indicesWithUnavailableReplicasOnly, + clusterMetadata, + projectResolver.supportsMultipleProjects() + ) ); impacts.add( new HealthIndicatorImpact(NAME, REPLICA_UNASSIGNED_IMPACT_ID, 2, impactDescription, List.of(ImpactArea.SEARCH)) @@ -1129,9 +1158,9 @@ public List getImpacts() { */ public List getDiagnosis(boolean verbose, int maxAffectedResourcesCount) { if (verbose) { - Map> diagnosisToAffectedIndices = new HashMap<>(primaries.diagnosisDefinitions); + Map> diagnosisToAffectedIndices = new HashMap<>(primaries.diagnosisDefinitions); replicas.diagnosisDefinitions.forEach((diagnosisDef, indicesWithReplicasUnassigned) -> { - Set indicesWithPrimariesUnassigned = diagnosisToAffectedIndices.get(diagnosisDef); + Set indicesWithPrimariesUnassigned = diagnosisToAffectedIndices.get(diagnosisDef); if (indicesWithPrimariesUnassigned == null) { diagnosisToAffectedIndices.put(diagnosisDef, indicesWithReplicasUnassigned); } else { @@ -1145,13 +1174,14 @@ public List getDiagnosis(boolean verbose, int maxAffectedResourcesCou return diagnosisToAffectedIndices.entrySet().stream().map(e -> { List affectedResources = new ArrayList<>(1); if (e.getKey().equals(ACTION_RESTORE_FROM_SNAPSHOT)) { - Set restoreFromSnapshotIndices = e.getValue(); + Set restoreFromSnapshotIndices = e.getValue(); if (restoreFromSnapshotIndices != null && restoreFromSnapshotIndices.isEmpty() == false) { affectedResources = getRestoreFromSnapshotAffectedResources( clusterMetadata, systemIndices, restoreFromSnapshotIndices, - maxAffectedResourcesCount + maxAffectedResourcesCount, + projectResolver.supportsMultipleProjects() ); } } else { @@ -1160,7 +1190,13 @@ public List getDiagnosis(boolean verbose, int maxAffectedResourcesCou INDEX, e.getValue() .stream() - .sorted(indicesComparatorByPriorityAndName(clusterMetadata)) + .sorted( + indicesComparatorByPriorityAndProjectIndex( + clusterMetadata, + projectResolver.supportsMultipleProjects() + ) + ) + .map(projectIndex -> projectIndex.toString(projectResolver.supportsMultipleProjects())) .limit(Math.min(e.getValue().size(), maxAffectedResourcesCount)) .collect(Collectors.toList()) ) @@ -1183,27 +1219,23 @@ public List getDiagnosis(boolean verbose, int maxAffectedResourcesCou static List getRestoreFromSnapshotAffectedResources( Metadata metadata, SystemIndices systemIndices, - Set restoreFromSnapshotIndices, - int maxAffectedResourcesCount + Set restoreFromSnapshotIndices, + int maxAffectedResourcesCount, + boolean supportsMultipleProjects ) { List affectedResources = new ArrayList<>(2); - - Set affectedIndices = new HashSet<>(restoreFromSnapshotIndices); + Set affectedProjects = restoreFromSnapshotIndices.stream().map(ProjectIndexName::projectId).collect(toSet()); + Set affectedIndices = new HashSet<>(restoreFromSnapshotIndices); Set affectedFeatureStates = new HashSet<>(); - Map> featureToSystemIndices = systemIndices.getFeatures() - .stream() - .collect( - toMap( - SystemIndices.Feature::getName, - feature -> feature.getIndexDescriptors() - .stream() - .flatMap(descriptor -> descriptor.getMatchingIndices(metadata.getProject()).stream()) - .collect(toSet()) - ) - ); - for (Map.Entry> featureToIndices : featureToSystemIndices.entrySet()) { - for (String featureIndex : featureToIndices.getValue()) { + Map> featureToSystemIndices = getSystemIndicesForProjects( + systemIndices, + affectedProjects, + metadata + ); + + for (Map.Entry> featureToIndices : featureToSystemIndices.entrySet()) { + for (ProjectIndexName featureIndex : featureToIndices.getValue()) { if (restoreFromSnapshotIndices.contains(featureIndex)) { affectedFeatureStates.add(featureToIndices.getKey()); affectedIndices.remove(featureIndex); @@ -1211,22 +1243,16 @@ static List getRestoreFromSnapshotAffectedResources( } } - Map> featureToDsBackingIndices = systemIndices.getFeatures() - .stream() - .collect( - toMap( - SystemIndices.Feature::getName, - feature -> feature.getDataStreamDescriptors() - .stream() - .flatMap(descriptor -> descriptor.getBackingIndexNames(metadata).stream()) - .collect(toSet()) - ) - ); + Map> featureToDsBackingIndices = getSystemDsBackingIndicesForProjects( + systemIndices, + affectedProjects, + metadata + ); // the shards_availability indicator works with indices so let's remove the feature states data streams backing indices from // the list of affected indices (the feature state will cover the restore of these indices too) - for (Map.Entry> featureToBackingIndices : featureToDsBackingIndices.entrySet()) { - for (String featureIndex : featureToBackingIndices.getValue()) { + for (Map.Entry> featureToBackingIndices : featureToDsBackingIndices.entrySet()) { + for (ProjectIndexName featureIndex : featureToBackingIndices.getValue()) { if (restoreFromSnapshotIndices.contains(featureIndex)) { affectedFeatureStates.add(featureToBackingIndices.getKey()); affectedIndices.remove(featureIndex); @@ -1235,7 +1261,16 @@ static List getRestoreFromSnapshotAffectedResources( } if (affectedIndices.isEmpty() == false) { - affectedResources.add(new Diagnosis.Resource(INDEX, affectedIndices.stream().limit(maxAffectedResourcesCount).toList())); + affectedResources.add( + new Diagnosis.Resource( + INDEX, + affectedIndices.stream() + .sorted(indicesComparatorByPriorityAndProjectIndex(metadata, supportsMultipleProjects)) + .map(index -> index.toString(supportsMultipleProjects)) + .limit(maxAffectedResourcesCount) + .toList() + ) + ); } if (affectedFeatureStates.isEmpty() == false) { affectedResources.add( @@ -1244,35 +1279,97 @@ static List getRestoreFromSnapshotAffectedResources( } return affectedResources; } + + /** + * Retrieve the system indices for the projects and group them by Feature + */ + private static Map> getSystemIndicesForProjects( + SystemIndices systemIndices, + Set projects, + Metadata metadata + ) { + return systemIndices.getFeatures() + .stream() + .collect( + Collectors.toMap( + SystemIndices.Feature::getName, + feature -> feature.getIndexDescriptors() + .stream() + .flatMap( + descriptor -> projects.stream() + .flatMap( + projectId -> descriptor.getMatchingIndices(metadata.getProject(projectId)) + .stream() + .map(index -> new ProjectIndexName(projectId, index)) + ) + ) + .collect(Collectors.toSet()) + ) + ); + } + + /** + * Retrieve the backing indices for system data stream for the projects and group them by Feature + */ + private static Map> getSystemDsBackingIndicesForProjects( + SystemIndices systemIndices, + Set projects, + Metadata metadata + ) { + return systemIndices.getFeatures() + .stream() + .collect( + toMap( + SystemIndices.Feature::getName, + feature -> feature.getDataStreamDescriptors() + .stream() + .flatMap( + descriptor -> projects.stream() + .flatMap( + projectId -> descriptor.getBackingIndexNames(metadata.getProject(projectId)) + .stream() + .map(index -> new ProjectIndexName(projectId, index)) + ) + ) + .collect(Collectors.toSet()) + ) + ); + } } public static class SearchableSnapshotsState { - private final Set searchableSnapshotWithUnavailableShard = new HashSet<>(); - private final Set searchableSnapshotWithOriginalIndexAvailable = new HashSet<>(); + private final Set searchableSnapshotWithUnavailableShard = new HashSet<>(); + private final Set searchableSnapshotWithOriginalIndexAvailable = new HashSet<>(); - void addSearchableSnapshotWithUnavailableShard(String indexName) { + void addSearchableSnapshotWithUnavailableShard(ProjectIndexName indexName) { searchableSnapshotWithUnavailableShard.add(indexName); } - void addSearchableSnapshotWithOriginalIndexAvailable(String indexName) { + void addSearchableSnapshotWithOriginalIndexAvailable(ProjectIndexName indexName) { searchableSnapshotWithOriginalIndexAvailable.add(indexName); } - public Set getRedSearchableSnapshots() { + public Set getRedSearchableSnapshots() { return Sets.difference(searchableSnapshotWithUnavailableShard, searchableSnapshotWithOriginalIndexAvailable); } // If the original index of a searchable snapshot with unavailable shards is available then we remove the searchable snapshot // from the list of the unavailable searchable snapshots because the data is available via the original index. - void updateSearchableSnapshotWithAvailableIndices(Metadata clusterMetadata, Set indicesWithUnavailableShards) { - for (String index : searchableSnapshotWithUnavailableShard) { - assert clusterMetadata.getProject().index(index) != null : "Index metadata of index '" + index + "' should not be null"; - Settings indexSettings = clusterMetadata.getProject().index(index).getSettings(); + void updateSearchableSnapshotWithAvailableIndices(Metadata clusterMetadata, Set indicesWithUnavailableShards) { + for (ProjectIndexName projectIndex : searchableSnapshotWithUnavailableShard) { + ProjectId projectId = projectIndex.projectId(); + String index = projectIndex.indexName(); + + assert clusterMetadata.getProject(projectId).index(index) != null + : "Index metadata of index '" + index + "' should not be null"; + + Settings indexSettings = clusterMetadata.getProject(projectId).index(index).getSettings(); String originalIndex = indexSettings.get(SearchableSnapshotsSettings.SEARCHABLE_SNAPSHOT_INDEX_NAME_SETTING_KEY); + ProjectIndexName originalProjectIndex = new ProjectIndexName(projectId, originalIndex); if (originalIndex != null - && clusterMetadata.getProject().indices().containsKey(originalIndex) != false - && indicesWithUnavailableShards.contains(originalIndex) == false) { - addSearchableSnapshotWithOriginalIndexAvailable(index); + && clusterMetadata.getProject(projectId).indices().containsKey(originalIndex) != false + && indicesWithUnavailableShards.contains(originalProjectIndex) == false) { + addSearchableSnapshotWithOriginalIndexAvailable(projectIndex); } } } diff --git a/server/src/main/java/org/elasticsearch/health/RestGetHealthAction.java b/server/src/main/java/org/elasticsearch/health/RestGetHealthAction.java index 6e5cf36da7acf..1f839fa1c2d32 100644 --- a/server/src/main/java/org/elasticsearch/health/RestGetHealthAction.java +++ b/server/src/main/java/org/elasticsearch/health/RestGetHealthAction.java @@ -10,6 +10,7 @@ package org.elasticsearch.health; import org.elasticsearch.client.internal.node.NodeClient; +import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.Scope; @@ -19,6 +20,7 @@ import java.io.IOException; import java.util.List; +import java.util.Set; import static org.elasticsearch.rest.RestRequest.Method.GET; @@ -29,6 +31,8 @@ public class RestGetHealthAction extends BaseRestHandler { private static final String SIZE_PARAM = "size"; + private static final String CAPABILITY_MULTI_PROJECT_SHARDS_AVAILABILITY = "multi_project_shards_availability"; + @Override public String getName() { // TODO: Existing - "cluster_health_action", "cat_health_action" @@ -57,4 +61,9 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli public boolean canTripCircuitBreaker() { return false; } + + @Override + public Set supportedCapabilities() { + return Sets.union(Set.of(CAPABILITY_MULTI_PROJECT_SHARDS_AVAILABILITY), super.supportedCapabilities()); + } } diff --git a/server/src/main/java/org/elasticsearch/health/node/HealthIndicatorDisplayValues.java b/server/src/main/java/org/elasticsearch/health/node/HealthIndicatorDisplayValues.java index 82460601764cb..e470f9f6e24f3 100644 --- a/server/src/main/java/org/elasticsearch/health/node/HealthIndicatorDisplayValues.java +++ b/server/src/main/java/org/elasticsearch/health/node/HealthIndicatorDisplayValues.java @@ -11,6 +11,7 @@ import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.cluster.metadata.ProjectMetadata; import org.elasticsearch.cluster.node.DiscoveryNode; import java.util.Collection; @@ -47,6 +48,7 @@ public static String getNodeName(DiscoveryNode node) { * logging or user messages. The indices are sorted by priority and then by name to ensure a * deterministic message. If there are more indices than 10, it adds the '...' suffix. */ + @Deprecated public static String getTruncatedIndices(Set indices, Metadata clusterMetadata) { final int maxIndices = 10; String truncatedIndicesString = indices.stream() @@ -59,6 +61,28 @@ public static String getTruncatedIndices(Set indices, Metadata clusterMe return truncatedIndicesString; } + /** + * Creates a string that displays max 10 indices from the given set to be used as examples in + * logging or user messages. The indices are sorted by priority and then by name to ensure a + * deterministic message. If there are more indices than 10, it adds the '...' suffix. + */ + public static String getTruncatedProjectIndices( + Set indices, + Metadata clusterMetadata, + boolean supportsMultipleProjects + ) { + final int maxIndices = 10; + String truncatedIndicesString = indices.stream() + .sorted(indicesComparatorByPriorityAndProjectIndex(clusterMetadata, supportsMultipleProjects)) + .limit(maxIndices) + .map(projectIndexName -> projectIndexName.toString(supportsMultipleProjects)) + .collect(joining(", ")); + if (maxIndices < indices.size()) { + truncatedIndicesString = truncatedIndicesString + ", ..."; + } + return truncatedIndicesString; + } + /** * Creates a string that displays all the values that fulfilled the predicate sorted in the natural order. * @param values, the values to be displayed @@ -119,6 +143,7 @@ public static String regularVerb(String verb, int count) { * @param clusterMetadata Used to look up index priority. * @return Comparator instance */ + @Deprecated public static Comparator indicesComparatorByPriorityAndName(Metadata clusterMetadata) { // We want to show indices with a numerically higher index.priority first (since lower priority ones might get truncated): return Comparator.comparingInt((String indexName) -> { @@ -126,4 +151,23 @@ public static Comparator indicesComparatorByPriorityAndName(Metadata clu return indexMetadata == null ? -1 : indexMetadata.priority(); }).reversed().thenComparing(Comparator.naturalOrder()); } + + /** + * Sorts index names by their priority first, then alphabetically by name. If the priority cannot be determined for an index then + * a priority of -1 is used to sort it behind other index names. + * @param clusterMetadata Used to look up index priority. + * @param supportsMultipleProjects Whether cluster supports multi-project + * @return Comparator instance + */ + public static Comparator indicesComparatorByPriorityAndProjectIndex( + Metadata clusterMetadata, + boolean supportsMultipleProjects + ) { + // We want to show indices with a numerically higher index.priority first (since lower priority ones might get truncated): + return Comparator.comparingInt((ProjectIndexName projectIndexName) -> { + ProjectMetadata projectMetadata = clusterMetadata.getProject(projectIndexName.projectId()); + IndexMetadata indexMetadata = projectMetadata.index(projectIndexName.indexName()); + return indexMetadata == null ? -1 : indexMetadata.priority(); + }).reversed().thenComparing(projectIndex -> projectIndex.toString(supportsMultipleProjects)); + } } diff --git a/server/src/main/java/org/elasticsearch/health/node/ProjectIndexName.java b/server/src/main/java/org/elasticsearch/health/node/ProjectIndexName.java new file mode 100644 index 0000000000000..a733f8bcf0ee3 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/health/node/ProjectIndexName.java @@ -0,0 +1,35 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.health.node; + +import org.elasticsearch.cluster.metadata.ProjectId; + +public record ProjectIndexName(ProjectId projectId, String indexName) implements Comparable { + // VisibleForTesting + public static final String DELIMITER = "/"; + + @Override + public String toString() { + return toString(true); + } + + public String toString(boolean withProjectId) { + if (withProjectId) { + return projectId.id() + DELIMITER + indexName; + } else { + return indexName; + } + } + + @Override + public int compareTo(ProjectIndexName other) { + return this.toString().compareTo(other.toString()); + } +} diff --git a/server/src/main/java/org/elasticsearch/indices/SystemDataStreamDescriptor.java b/server/src/main/java/org/elasticsearch/indices/SystemDataStreamDescriptor.java index 72ba6d2dc3a45..2bbdaaaae7946 100644 --- a/server/src/main/java/org/elasticsearch/indices/SystemDataStreamDescriptor.java +++ b/server/src/main/java/org/elasticsearch/indices/SystemDataStreamDescriptor.java @@ -13,6 +13,7 @@ import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.cluster.metadata.ProjectMetadata; import org.elasticsearch.index.Index; import java.util.Collections; @@ -106,8 +107,18 @@ public String getDataStreamName() { * @param metadata Metadata in which to look for indices * @return List of names of backing indices */ + @Deprecated public List getBackingIndexNames(Metadata metadata) { - DataStream dataStream = metadata.getProject().dataStreams().get(dataStreamName); + return getBackingIndexNames(metadata.getProject()); + } + + /** + * Retrieve backing indices for this system data stream + * @param projectMetadata Project metadata in which to look for indices + * @return List of names of backing indices + */ + public List getBackingIndexNames(ProjectMetadata projectMetadata) { + DataStream dataStream = projectMetadata.dataStreams().get(dataStreamName); if (dataStream == null) { return Collections.emptyList(); } diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/ShardsAvailabilityActionGuideTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/ShardsAvailabilityActionGuideTests.java index 87a792e4b32fd..a5c21e711f15c 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/ShardsAvailabilityActionGuideTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/ShardsAvailabilityActionGuideTests.java @@ -9,6 +9,7 @@ package org.elasticsearch.cluster.routing.allocation; +import org.elasticsearch.cluster.project.ProjectResolver; import org.elasticsearch.cluster.routing.allocation.shards.ShardsAvailabilityHealthIndicatorService; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.ClusterSettings; @@ -46,7 +47,12 @@ public ShardsAvailabilityActionGuideTests() { ClusterService clusterService = mock(ClusterService.class); when(clusterService.getClusterSettings()).thenReturn(ClusterSettings.createBuiltInClusterSettings()); when(clusterService.getSettings()).thenReturn(Settings.EMPTY); - service = new ShardsAvailabilityHealthIndicatorService(clusterService, mock(AllocationService.class), mock(SystemIndices.class)); + service = new ShardsAvailabilityHealthIndicatorService( + clusterService, + mock(AllocationService.class), + mock(SystemIndices.class), + mock(ProjectResolver.class) + ); } public void testRestoreFromSnapshotAction() { diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/shards/ShardsAvailabilityHealthIndicatorServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/shards/ShardsAvailabilityHealthIndicatorServiceTests.java index d1e7138e0d3db..5045cc6f498de 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/shards/ShardsAvailabilityHealthIndicatorServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/shards/ShardsAvailabilityHealthIndicatorServiceTests.java @@ -15,11 +15,17 @@ import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.metadata.NodesShutdownMetadata; +import org.elasticsearch.cluster.metadata.ProjectId; +import org.elasticsearch.cluster.metadata.ProjectMetadata; import org.elasticsearch.cluster.metadata.SingleNodeShutdownMetadata; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodeRole; import org.elasticsearch.cluster.node.DiscoveryNodeUtils; import org.elasticsearch.cluster.node.DiscoveryNodes; +import org.elasticsearch.cluster.project.DefaultProjectResolver; +import org.elasticsearch.cluster.project.ProjectResolver; +import org.elasticsearch.cluster.project.TestProjectResolvers; +import org.elasticsearch.cluster.routing.GlobalRoutingTable; import org.elasticsearch.cluster.routing.IndexRoutingTable; import org.elasticsearch.cluster.routing.RecoverySource; import org.elasticsearch.cluster.routing.RoutingTable; @@ -54,6 +60,7 @@ import org.elasticsearch.health.ImpactArea; import org.elasticsearch.health.SimpleHealthIndicatorDetails; import org.elasticsearch.health.node.HealthInfo; +import org.elasticsearch.health.node.ProjectIndexName; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexModule; import org.elasticsearch.index.IndexVersion; @@ -70,13 +77,15 @@ import java.time.Instant; import java.util.ArrayList; import java.util.Collections; +import java.util.Comparator; import java.util.HashMap; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.UUID; -import java.util.stream.Collectors; +import java.util.stream.Stream; import static java.util.Collections.emptyList; import static java.util.stream.Collectors.toMap; @@ -129,14 +138,16 @@ public class ShardsAvailabilityHealthIndicatorServiceTests extends ESTestCase { public void testShouldBeGreenWhenAllPrimariesAndReplicasAreStarted() { + ProjectId projectId = randomProjectIdOrDefault(); var clusterState = createClusterStateWith( + projectId, List.of( index("replicated-index", new ShardAllocation(randomNodeId(), AVAILABLE), new ShardAllocation(randomNodeId(), AVAILABLE)), index("unreplicated-index", new ShardAllocation(randomNodeId(), AVAILABLE)) ), List.of() ); - var service = createShardsAvailabilityIndicatorService(clusterState); + var service = createShardsAvailabilityIndicatorService(projectId, clusterState); assertThat( service.calculate(true, HealthInfo.EMPTY_HEALTH_INFO), @@ -153,13 +164,15 @@ public void testShouldBeGreenWhenAllPrimariesAndReplicasAreStarted() { } public void testShouldBeYellowWhenReplicaIsInitializing() { + ProjectId projectId = randomProjectIdOrDefault(); var clusterState = createClusterStateWith( + projectId, List.of( index("replicated-index", new ShardAllocation(randomNodeId(), AVAILABLE), new ShardAllocation(randomNodeId(), INITIALIZING)) ), List.of() ); - var service = createShardsAvailabilityIndicatorService(clusterState); + var service = createShardsAvailabilityIndicatorService(projectId, clusterState); assertThat( service.calculate(true, HealthInfo.EMPTY_HEALTH_INFO), @@ -189,11 +202,13 @@ public void testShouldBeYellowWhenReplicaIsInitializing() { } public void testShouldBeRedWhenPrimaryIsInitializing() { + ProjectId projectId = randomProjectIdOrDefault(); var clusterState = createClusterStateWith( + projectId, List.of(index("unreplicated-index", new ShardAllocation(randomNodeId(), INITIALIZING))), List.of() ); - var service = createShardsAvailabilityIndicatorService(clusterState); + var service = createShardsAvailabilityIndicatorService(projectId, clusterState); HealthIndicatorResult calculate = service.calculate(true, HealthInfo.EMPTY_HEALTH_INFO); assertThat( @@ -224,11 +239,13 @@ public void testShouldBeRedWhenPrimaryIsInitializing() { } public void testShouldBeGreenWhenAllPrimariesAreCreating() { + ProjectId projectId = randomProjectIdOrDefault(); var clusterState = createClusterStateWith( + projectId, List.of(index("unreplicated-index", new ShardAllocation(randomNodeId(), CREATING))), List.of() ); - var service = createShardsAvailabilityIndicatorService(clusterState); + var service = createShardsAvailabilityIndicatorService(projectId, clusterState); assertThat( service.calculate(true, HealthInfo.EMPTY_HEALTH_INFO), @@ -248,7 +265,9 @@ public void testShouldBeYellowWhenThereAreUnassignedReplicas() { var availableReplicas = randomList(0, 5, () -> new ShardAllocation(randomNodeId(), AVAILABLE)); var unavailableReplicas = randomList(1, 5, () -> new ShardAllocation(randomNodeId(), UNAVAILABLE)); + ProjectId projectId = randomProjectIdOrDefault(); var clusterState = createClusterStateWith( + projectId, List.of( index( "yellow-index", @@ -258,7 +277,7 @@ public void testShouldBeYellowWhenThereAreUnassignedReplicas() { ), List.of() ); - var service = createShardsAvailabilityIndicatorService(clusterState); + var service = createShardsAvailabilityIndicatorService(projectId, clusterState); assertThat( service.calculate(true, HealthInfo.EMPTY_HEALTH_INFO), @@ -294,11 +313,13 @@ public void testShouldBeYellowWhenThereAreUnassignedReplicas() { } public void testShouldBeRedWhenThereAreUnassignedPrimariesAndUnassignedReplicas() { + ProjectId projectId = randomProjectIdOrDefault(); var clusterState = createClusterStateWith( + projectId, List.of(index("red-index", new ShardAllocation(randomNodeId(), UNAVAILABLE), new ShardAllocation(randomNodeId(), UNAVAILABLE))), List.of() ); - var service = createShardsAvailabilityIndicatorService(clusterState); + var service = createShardsAvailabilityIndicatorService(projectId, clusterState); assertThat( service.calculate(true, HealthInfo.EMPTY_HEALTH_INFO), @@ -326,7 +347,9 @@ public void testShouldBeRedWhenThereAreUnassignedPrimariesAndUnassignedReplicas( public void testAllReplicasUnassigned() { { - ClusterState clusterState = createClusterStateWith( + ProjectId projectId = randomProjectIdOrDefault(); + var clusterState = createClusterStateWith( + projectId, List.of( index( "myindex", @@ -337,7 +360,7 @@ public void testAllReplicasUnassigned() { ), List.of() ); - var service = createShardsAvailabilityIndicatorService(clusterState); + var service = createShardsAvailabilityIndicatorService(projectId, clusterState); ShardAllocationStatus status = service.createNewStatus(clusterState.metadata()); ShardsAvailabilityHealthIndicatorService.updateShardAllocationStatus( status, @@ -349,7 +372,9 @@ public void testAllReplicasUnassigned() { assertFalse(status.replicas.doAnyIndicesHaveAllUnavailable()); } { - ClusterState clusterState = createClusterStateWith( + ProjectId projectId = randomProjectIdOrDefault(); + var clusterState = createClusterStateWith( + projectId, List.of( index( "myindex", @@ -360,7 +385,7 @@ public void testAllReplicasUnassigned() { ), List.of() ); - var service = createShardsAvailabilityIndicatorService(clusterState); + var service = createShardsAvailabilityIndicatorService(projectId, clusterState); ShardAllocationStatus status = service.createNewStatus(clusterState.metadata()); ShardsAvailabilityHealthIndicatorService.updateShardAllocationStatus( status, @@ -372,7 +397,9 @@ public void testAllReplicasUnassigned() { assertFalse(status.replicas.doAnyIndicesHaveAllUnavailable()); } { - ClusterState clusterState = createClusterStateWith( + ProjectId projectId = randomProjectIdOrDefault(); + var clusterState = createClusterStateWith( + projectId, List.of( index( "myindex", @@ -383,7 +410,7 @@ public void testAllReplicasUnassigned() { ), List.of() ); - var service = createShardsAvailabilityIndicatorService(clusterState); + var service = createShardsAvailabilityIndicatorService(projectId, clusterState); ShardAllocationStatus status = service.createNewStatus(clusterState.metadata()); ShardsAvailabilityHealthIndicatorService.updateShardAllocationStatus( status, @@ -395,7 +422,9 @@ public void testAllReplicasUnassigned() { assertTrue(status.replicas.doAnyIndicesHaveAllUnavailable()); } { - ClusterState clusterState = createClusterStateWith( + ProjectId projectId = randomProjectIdOrDefault(); + var clusterState = createClusterStateWith( + projectId, List.of( indexWithTwoPrimaryOneReplicaShard( "myindex", @@ -408,7 +437,7 @@ public void testAllReplicasUnassigned() { List.of() ); - var service = createShardsAvailabilityIndicatorService(clusterState); + var service = createShardsAvailabilityIndicatorService(projectId, clusterState); ShardAllocationStatus status = service.createNewStatus(clusterState.metadata()); ShardsAvailabilityHealthIndicatorService.updateShardAllocationStatus( status, @@ -420,7 +449,9 @@ public void testAllReplicasUnassigned() { assertTrue(status.replicas.doAnyIndicesHaveAllUnavailable()); } { - ClusterState clusterState = createClusterStateWith( + ProjectId projectId = randomProjectIdOrDefault(); + var clusterState = createClusterStateWith( + projectId, List.of( indexNewlyCreated( "myindex", @@ -445,7 +476,7 @@ public void testAllReplicasUnassigned() { ), List.of() ); - var service = createShardsAvailabilityIndicatorService(clusterState); + var service = createShardsAvailabilityIndicatorService(projectId, clusterState); ShardAllocationStatus status = service.createNewStatus(clusterState.metadata()); ShardsAvailabilityHealthIndicatorService.updateShardAllocationStatus( status, @@ -474,8 +505,9 @@ public void testAllReplicasUnassigned() { new ShardAllocation(randomNodeId(), AVAILABLE), // Primary 2 new ShardAllocation(randomNodeId(), UNAVAILABLE) // Replica 2 ); - ClusterState clusterState = createClusterStateWith(List.of(routingTable), List.of()); - var service = createShardsAvailabilityIndicatorService(clusterState); + ProjectId projectId = randomProjectIdOrDefault(); + var clusterState = createClusterStateWith(projectId, List.of(routingTable), List.of()); + var service = createShardsAvailabilityIndicatorService(projectId, clusterState); ShardAllocationStatus status = service.createNewStatus(clusterState.metadata()); ShardsAvailabilityHealthIndicatorService.updateShardAllocationStatus( status, @@ -485,10 +517,12 @@ public void testAllReplicasUnassigned() { timeValueSeconds(0) ); var shardRouting = routingTable.shardsWithState(ShardRoutingState.UNASSIGNED).get(0); - assertTrue(service.areAllShardsOfThisTypeUnavailable(shardRouting, clusterState)); + assertTrue(service.areAllShardsOfThisTypeUnavailable(projectId, shardRouting, clusterState)); } { - ClusterState clusterState = createClusterStateWith( + ProjectId projectId = randomProjectIdOrDefault(); + var clusterState = createClusterStateWith( + projectId, List.of( index( "myindex", @@ -499,7 +533,7 @@ public void testAllReplicasUnassigned() { ), List.of() ); - var service = createShardsAvailabilityIndicatorService(clusterState); + var service = createShardsAvailabilityIndicatorService(projectId, clusterState); ShardAllocationStatus status = service.createNewStatus(clusterState.metadata()); ShardsAvailabilityHealthIndicatorService.updateShardAllocationStatus( status, @@ -508,14 +542,19 @@ public void testAllReplicasUnassigned() { randomBoolean(), timeValueSeconds(0) ); - var shardRouting = clusterState.routingTable().index("myindex").shardsWithState(ShardRoutingState.UNASSIGNED).get(0); - assertFalse(service.areAllShardsOfThisTypeUnavailable(shardRouting, clusterState)); + var shardRouting = clusterState.routingTable(projectId).index("myindex").shardsWithState(ShardRoutingState.UNASSIGNED).get(0); + assertFalse(service.areAllShardsOfThisTypeUnavailable(projectId, shardRouting, clusterState)); } } public void testShouldBeRedWhenThereAreUnassignedPrimariesAndNoReplicas() { - var clusterState = createClusterStateWith(List.of(index("red-index", new ShardAllocation(randomNodeId(), UNAVAILABLE))), List.of()); - var service = createShardsAvailabilityIndicatorService(clusterState); + ProjectId projectId = randomProjectIdOrDefault(); + var clusterState = createClusterStateWith( + projectId, + List.of(index("red-index", new ShardAllocation(randomNodeId(), UNAVAILABLE))), + List.of() + ); + var service = createShardsAvailabilityIndicatorService(projectId, clusterState); assertThat( service.calculate(true, HealthInfo.EMPTY_HEALTH_INFO), @@ -542,11 +581,13 @@ public void testShouldBeRedWhenThereAreUnassignedPrimariesAndNoReplicas() { } public void testShouldBeRedWhenThereAreUnassignedPrimariesAndUnassignedReplicasOnSameIndex() { + ProjectId projectId = randomProjectIdOrDefault(); var clusterState = createClusterStateWith( + projectId, List.of(index("red-index", new ShardAllocation(randomNodeId(), UNAVAILABLE), new ShardAllocation(randomNodeId(), UNAVAILABLE))), List.of() ); - var service = createShardsAvailabilityIndicatorService(clusterState); + var service = createShardsAvailabilityIndicatorService(projectId, clusterState); HealthIndicatorResult result = service.calculate(true, HealthInfo.EMPTY_HEALTH_INFO); assertEquals(RED, result.status()); @@ -568,7 +609,9 @@ public void testShouldBeRedWhenThereAreUnassignedPrimariesAndUnassignedReplicasO List indexMetadataList = createIndexMetadataForIndexNameToPriorityMap( Map.of("red-index", 3, "yellow-index-1", 5, "yellow-index-2", 8) ); + ProjectId projectId = randomProjectIdOrDefault(); var clusterState = createClusterStateWith( + projectId, indexMetadataList, List.of( index("red-index", new ShardAllocation(randomNodeId(), UNAVAILABLE)), @@ -578,7 +621,7 @@ public void testShouldBeRedWhenThereAreUnassignedPrimariesAndUnassignedReplicasO List.of(), List.of() ); - var service = createShardsAvailabilityIndicatorService(clusterState); + var service = createShardsAvailabilityIndicatorService(projectId, clusterState); HealthIndicatorResult result = service.calculate(true, HealthInfo.EMPTY_HEALTH_INFO); assertEquals(RED, result.status()); @@ -616,7 +659,9 @@ public void testSortByIndexPriority() { List indexMetadataList = createIndexMetadataForIndexNameToPriorityMap( Map.of("index-3", lowPriority, "index-1", lowPriority, "index-2", highPriority) ); + ProjectId projectId = randomProjectIdOrDefault(); var clusterState = createClusterStateWith( + projectId, indexMetadataList, List.of( index("index-3", new ShardAllocation(randomNodeId(), AVAILABLE), new ShardAllocation(randomNodeId(), UNAVAILABLE)), @@ -626,7 +671,7 @@ public void testSortByIndexPriority() { List.of(), List.of() ); - var service = createShardsAvailabilityIndicatorService(clusterState); + var service = createShardsAvailabilityIndicatorService(projectId, clusterState); HealthIndicatorResult result = service.calculate(true, HealthInfo.EMPTY_HEALTH_INFO); // index-2 has the higher priority so it ought to be listed first, followed by index-1 then index-3 which have the same priority: @@ -646,7 +691,9 @@ public void testSortByIndexPriority() { } public void testShouldBeGreenWhenThereAreRestartingReplicas() { + ProjectId projectId = randomProjectIdOrDefault(); var clusterState = createClusterStateWith( + projectId, List.of( index( "restarting-index", @@ -656,7 +703,7 @@ public void testShouldBeGreenWhenThereAreRestartingReplicas() { ), List.of(new NodeShutdown("node-0", RESTART, 60)) ); - var service = createShardsAvailabilityIndicatorService(clusterState); + var service = createShardsAvailabilityIndicatorService(projectId, clusterState); assertThat( service.calculate(true, HealthInfo.EMPTY_HEALTH_INFO), @@ -673,11 +720,13 @@ public void testShouldBeGreenWhenThereAreRestartingReplicas() { } public void testShouldBeGreenWhenThereAreNoReplicasExpected() { + ProjectId projectId = randomProjectIdOrDefault(); var clusterState = createClusterStateWith( + projectId, List.of(index("primaries-only-index", new ShardAllocation(randomNodeId(), AVAILABLE))), List.of(new NodeShutdown("node-0", RESTART, 60)) ); - var service = createShardsAvailabilityIndicatorService(clusterState); + var service = createShardsAvailabilityIndicatorService(projectId, clusterState); assertThat( service.calculate(true, HealthInfo.EMPTY_HEALTH_INFO), @@ -694,7 +743,9 @@ public void testShouldBeGreenWhenThereAreNoReplicasExpected() { } public void testShouldBeYellowWhenRestartingReplicasReachedAllocationDelay() { + ProjectId projectId = randomProjectIdOrDefault(); var clusterState = createClusterStateWith( + projectId, List.of( index( "restarting-index", @@ -704,7 +755,7 @@ public void testShouldBeYellowWhenRestartingReplicasReachedAllocationDelay() { ), List.of(new NodeShutdown("node-0", RESTART, 60)) ); - var service = createShardsAvailabilityIndicatorService(clusterState); + var service = createShardsAvailabilityIndicatorService(projectId, clusterState); assertThat( service.calculate(true, HealthInfo.EMPTY_HEALTH_INFO), @@ -735,11 +786,13 @@ public void testShouldBeYellowWhenRestartingReplicasReachedAllocationDelay() { } public void testShouldBeGreenWhenThereAreInitializingPrimariesAndReplicas() { + ProjectId projectId = randomProjectIdOrDefault(); var clusterState = createClusterStateWith( + projectId, List.of(index("restarting-index", new ShardAllocation("node-0", CREATING), new ShardAllocation("node-1", CREATING))), List.of() ); - var service = createShardsAvailabilityIndicatorService(clusterState); + var service = createShardsAvailabilityIndicatorService(projectId, clusterState); assertThat( service.calculate(true, HealthInfo.EMPTY_HEALTH_INFO), @@ -756,11 +809,13 @@ public void testShouldBeGreenWhenThereAreInitializingPrimariesAndReplicas() { } public void testShouldBeGreenWhenThereAreRestartingPrimaries() { + ProjectId projectId = randomProjectIdOrDefault(); var clusterState = createClusterStateWith( + projectId, List.of(index("restarting-index", new ShardAllocation("node-0", RESTARTING, System.nanoTime()))), List.of(new NodeShutdown("node-0", RESTART, 60)) ); - var service = createShardsAvailabilityIndicatorService(clusterState); + var service = createShardsAvailabilityIndicatorService(projectId, clusterState); assertThat( service.calculate(true, HealthInfo.EMPTY_HEALTH_INFO), @@ -777,7 +832,9 @@ public void testShouldBeGreenWhenThereAreRestartingPrimaries() { } public void testShouldBeRedWhenRestartingPrimariesReachedAllocationDelayAndNoReplicas() { + ProjectId projectId = randomProjectIdOrDefault(); var clusterState = createClusterStateWith( + projectId, List.of( index( "restarting-index", @@ -786,7 +843,7 @@ public void testShouldBeRedWhenRestartingPrimariesReachedAllocationDelayAndNoRep ), List.of(new NodeShutdown("node-0", RESTART, 60)) ); - var service = createShardsAvailabilityIndicatorService(clusterState); + var service = createShardsAvailabilityIndicatorService(projectId, clusterState); assertThat( service.calculate(true, HealthInfo.EMPTY_HEALTH_INFO), @@ -824,14 +881,16 @@ public void testDiagnosisNotGeneratedWhenNotDrillingDown() { .build(); // Cluster state with index, but its only shard is unassigned because there is no shard copy + ProjectId projectId = randomProjectIdOrDefault(); var clusterState = createClusterStateWith( + projectId, List.of(indexMetadata), List.of(index("red-index", new ShardAllocation(randomNodeId(), UNAVAILABLE, noShardCopy()))), List.of(), List.of() ); - var service = createShardsAvailabilityIndicatorService(clusterState); + var service = createShardsAvailabilityIndicatorService(projectId, clusterState); assertThat( service.calculate(false, HealthInfo.EMPTY_HEALTH_INFO), @@ -854,6 +913,7 @@ public void testDiagnosisNotGeneratedWhenNotDrillingDown() { } public void testDiagnoseRestoreIndexAfterDataLoss() { + ProjectId projectId = randomProjectIdOrDefault(); // Index definition, 1 primary no replicas IndexMetadata indexMetadata = IndexMetadata.builder("red-index") .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()).build()) @@ -867,7 +927,7 @@ public void testDiagnoseRestoreIndexAfterDataLoss() { new ShardAllocation(randomNodeId(), UNAVAILABLE, noShardCopy()) ); - var service = createShardsAvailabilityIndicatorService(); + var service = createShardsAvailabilityIndicatorService(projectId); List definitions = service.diagnoseUnassignedShardRouting(shardRouting, ClusterState.EMPTY_STATE); assertThat(definitions, hasSize(1)); @@ -878,7 +938,7 @@ public void testRestoreFromSnapshotReportsFeatureStates() { // this test adds a mix of regular and system indices and data streams // we'll test the `shards_availability` indicator correctly reports the // affected feature states and indices - + ProjectId projectId = randomProjectIdOrDefault(); IndexMetadata featureIndex = IndexMetadata.builder(".feature-index") .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()).build()) .numberOfShards(1) @@ -913,6 +973,7 @@ public void testRestoreFromSnapshotReportsFeatureStates() { ); var clusterState = createClusterStateWith( + projectId, List.of(featureIndex, regularIndex, backingIndex), List.of( IndexRoutingTable.builder(featureIndex.getIndex()).addShard(featureIndexRouting).build(), @@ -922,17 +983,17 @@ public void testRestoreFromSnapshotReportsFeatureStates() { List.of(), List.of() ); - // add the data stream to the cluster state - Metadata.Builder mdBuilder = Metadata.builder(clusterState.metadata()) + var projectBuilder = ProjectMetadata.builder(clusterState.metadata().getProject(projectId)) .put(newInstance(featureDataStreamName, List.of(backingIndex.getIndex()))); - ClusterState state = ClusterState.builder(clusterState).metadata(mdBuilder).build(); + ClusterState state = ClusterState.builder(clusterState).putProjectMetadata(projectBuilder).build(); var service = createAllocationHealthIndicatorService( Settings.EMPTY, state, Map.of(), - getSystemIndices(featureDataStreamName, ".test-ds-*", ".feature-*") + getSystemIndices(featureDataStreamName, ".test-ds-*", ".feature-*"), + TestProjectResolvers.singleProjectOnly(projectId) ); HealthIndicatorResult result = service.calculate(true, HealthInfo.EMPTY_HEALTH_INFO); @@ -952,6 +1013,7 @@ public void testRestoreFromSnapshotReportsFeatureStates() { } public void testGetRestoreFromSnapshotAffectedResources() { + ProjectId projectId = randomProjectIdOrDefault(); String featureDataStreamName = ".test-ds-feature"; IndexMetadata backingIndex = createBackingIndex(featureDataStreamName, 1).build(); @@ -974,15 +1036,23 @@ public void testGetRestoreFromSnapshotAffectedResources() { for (IndexMetadata indexMetadata : indexMetadataList) { indexMetadataMap.put(indexMetadata.getIndex().getName(), indexMetadata); } - metadataBuilder.indices(indexMetadataMap); - metadataBuilder.put(newInstance(featureDataStreamName, List.of(backingIndex.getIndex()))); - Metadata metadata = metadataBuilder.build(); + + Metadata metadata = metadataBuilder.put( + ProjectMetadata.builder(projectId) + .indices(indexMetadataMap) + .put(newInstance(featureDataStreamName, List.of(backingIndex.getIndex()))) + ).build(); { List affectedResources = ShardAllocationStatus.getRestoreFromSnapshotAffectedResources( metadata, getSystemIndices(featureDataStreamName, ".test-ds-*", ".feature-*"), - Set.of(backingIndex.getIndex().getName(), ".feature-index", "regular-index"), - 10 + Set.of( + new ProjectIndexName(projectId, backingIndex.getIndex().getName()), + new ProjectIndexName(projectId, ".feature-index"), + new ProjectIndexName(projectId, "regular-index") + ), + 10, + false ); assertThat(affectedResources.size(), is(2)); @@ -1000,8 +1070,13 @@ public void testGetRestoreFromSnapshotAffectedResources() { List affectedResources = ShardAllocationStatus.getRestoreFromSnapshotAffectedResources( metadata, getSystemIndices(featureDataStreamName, ".test-ds-*", ".feature-*"), - Set.of(backingIndex.getIndex().getName(), ".feature-index", "regular-index"), - 0 + Set.of( + new ProjectIndexName(projectId, backingIndex.getIndex().getName()), + new ProjectIndexName(projectId, ".feature-index"), + new ProjectIndexName(projectId, "regular-index") + ), + 0, + false ); assertThat(affectedResources.size(), is(2)); @@ -1018,6 +1093,7 @@ public void testGetRestoreFromSnapshotAffectedResources() { } public void testDiagnoseUnknownAllocationDeciderIssue() { + ProjectId projectId = randomProjectIdOrDefault(); // Index definition, 1 primary no replicas IndexMetadata indexMetadata = IndexMetadata.builder("red-index") .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()).build()) @@ -1027,6 +1103,7 @@ public void testDiagnoseUnknownAllocationDeciderIssue() { // Cluster state with index, but its only shard is unassigned (Either deciders said no, or a node left) var clusterState = createClusterStateWith( + projectId, List.of(indexMetadata), List.of(index("red-index", new ShardAllocation(randomNodeId(), UNAVAILABLE, randomFrom(decidersNo(), nodeLeft())))), List.of(), @@ -1058,10 +1135,10 @@ public void testDiagnoseUnknownAllocationDeciderIssue() { MoveDecision.NOT_TAKEN ) ); - var service = createShardsAvailabilityIndicatorService(clusterState, decisionMap); + var service = createShardsAvailabilityIndicatorService(projectId, clusterState, decisionMap); // Get the list of user actions that are generated for this unassigned index shard - ShardRouting shardRouting = clusterState.getRoutingTable().index(indexMetadata.getIndex()).shard(0).primaryShard(); + ShardRouting shardRouting = clusterState.routingTable(projectId).index(indexMetadata.getIndex()).shard(0).primaryShard(); List actions = service.diagnoseUnassignedShardRouting(shardRouting, clusterState); assertThat(actions, hasSize(1)); @@ -1069,6 +1146,7 @@ public void testDiagnoseUnknownAllocationDeciderIssue() { } public void testDiagnoseEnableIndexAllocation() { + ProjectId projectId = randomProjectIdOrDefault(); // Index definition, 1 primary no replicas, allocation is not allowed IndexMetadata indexMetadata = IndexMetadata.builder("red-index") .settings( @@ -1081,7 +1159,7 @@ public void testDiagnoseEnableIndexAllocation() { .numberOfReplicas(0) .build(); - var service = createShardsAvailabilityIndicatorService(); + var service = createShardsAvailabilityIndicatorService(projectId); // Get the list of user actions that are generated for this unassigned index shard List actions = service.checkIsAllocationDisabled( @@ -1101,6 +1179,7 @@ public void testDiagnoseEnableIndexAllocation() { } public void testNodeAllocationResultWithNullDecision() { + ProjectId projectId = randomProjectIdOrDefault(); // Index definition, 1 primary no replicas, allocation is not allowed IndexMetadata indexMetadata = IndexMetadata.builder("red-index") .settings( @@ -1113,7 +1192,7 @@ public void testNodeAllocationResultWithNullDecision() { .numberOfReplicas(0) .build(); - var service = createShardsAvailabilityIndicatorService(); + var service = createShardsAvailabilityIndicatorService(projectId); // Get the list of user actions that are generated for this unassigned index shard List actions = service.checkIsAllocationDisabled( @@ -1132,6 +1211,7 @@ public void testNodeAllocationResultWithNullDecision() { } public void testDiagnoseEnableClusterAllocation() { + ProjectId projectId = randomProjectIdOrDefault(); // Index definition, 1 primary no replicas IndexMetadata indexMetadata = IndexMetadata.builder("red-index") .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()).build()) @@ -1141,6 +1221,7 @@ public void testDiagnoseEnableClusterAllocation() { // Disallow allocations in cluster settings var service = createShardsAvailabilityIndicatorService( + projectId, Settings.builder().put(EnableAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ENABLE_SETTING.getKey(), "none").build(), ClusterState.EMPTY_STATE, Map.of() @@ -1164,6 +1245,7 @@ public void testDiagnoseEnableClusterAllocation() { } public void testDiagnoseEnableRoutingAllocation() { + ProjectId projectId = randomProjectIdOrDefault(); // Index definition, 1 primary no replicas, allocation is not allowed IndexMetadata indexMetadata = IndexMetadata.builder("red-index") .settings( @@ -1178,6 +1260,7 @@ public void testDiagnoseEnableRoutingAllocation() { // Disallow allocations in cluster settings var service = createShardsAvailabilityIndicatorService( + projectId, Settings.builder().put(EnableAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ENABLE_SETTING.getKey(), "none").build(), ClusterState.EMPTY_STATE, Map.of() @@ -1202,6 +1285,7 @@ public void testDiagnoseEnableRoutingAllocation() { } public void testDiagnoseEnableDataTiers() { + ProjectId projectId = randomProjectIdOrDefault(); // Index definition, 1 primary no replicas, in the hot tier IndexMetadata indexMetadata = IndexMetadata.builder("red-index") .settings( @@ -1214,7 +1298,7 @@ public void testDiagnoseEnableDataTiers() { .numberOfReplicas(0) .build(); - var service = createShardsAvailabilityIndicatorService(); + var service = createShardsAvailabilityIndicatorService(projectId); // Get the list of user actions that are generated for this unassigned index shard List actions = service.checkNodeRoleRelatedIssues( @@ -1258,7 +1342,9 @@ public void testDiagnoseIncreaseShardLimitIndexSettingInTier() { Set.of(DiscoveryNodeRole.DATA_HOT_NODE_ROLE) ); + ProjectId projectId = randomProjectIdOrDefault(); var clusterState = createClusterStateWith( + projectId, List.of(indexMetadata), List.of( IndexRoutingTable.builder(index) @@ -1273,7 +1359,7 @@ public void testDiagnoseIncreaseShardLimitIndexSettingInTier() { List.of(), List.of(hotNode) ); - var service = createShardsAvailabilityIndicatorService(); + var service = createShardsAvailabilityIndicatorService(projectId); // Get the list of user actions that are generated for this unassigned index shard List actions = service.checkNodeRoleRelatedIssues( @@ -1317,7 +1403,9 @@ public void testDiagnoseIncreaseShardLimitClusterSettingInTier() { Set.of(DiscoveryNodeRole.DATA_HOT_NODE_ROLE) ); + ProjectId projectId = randomProjectIdOrDefault(); var clusterState = createClusterStateWith( + projectId, List.of(indexMetadata), List.of( IndexRoutingTable.builder(index) @@ -1335,6 +1423,7 @@ public void testDiagnoseIncreaseShardLimitClusterSettingInTier() { // Configure at most 1 shard per node var service = createShardsAvailabilityIndicatorService( + projectId, Settings.builder().put(CLUSTER_TOTAL_SHARDS_PER_NODE_SETTING.getKey(), 1).build(), clusterState, Map.of() @@ -1383,7 +1472,9 @@ public void testDiagnoseIncreaseShardLimitIndexSettingInGeneral() { Set.of(DiscoveryNodeRole.DATA_ROLE) ); + ProjectId projectId = randomProjectIdOrDefault(); var clusterState = createClusterStateWith( + projectId, List.of(indexMetadata), List.of( IndexRoutingTable.builder(index) @@ -1398,7 +1489,7 @@ public void testDiagnoseIncreaseShardLimitIndexSettingInGeneral() { List.of(), List.of(dataNode) ); - var service = createShardsAvailabilityIndicatorService(); + var service = createShardsAvailabilityIndicatorService(projectId); // Get the list of user actions that are generated for this unassigned index shard List actions = service.checkNodeRoleRelatedIssues( @@ -1442,7 +1533,9 @@ public void testDiagnoseIncreaseShardLimitClusterSettingInGeneral() { Set.of(DiscoveryNodeRole.DATA_ROLE) ); + ProjectId projectId = randomProjectIdOrDefault(); var clusterState = createClusterStateWith( + projectId, List.of(indexMetadata), List.of( IndexRoutingTable.builder(index) @@ -1460,6 +1553,7 @@ public void testDiagnoseIncreaseShardLimitClusterSettingInGeneral() { // Configure at most 1 shard per node var service = createShardsAvailabilityIndicatorService( + projectId, Settings.builder().put(CLUSTER_TOTAL_SHARDS_PER_NODE_SETTING.getKey(), 1).build(), clusterState, Map.of() @@ -1486,6 +1580,7 @@ public void testDiagnoseIncreaseShardLimitClusterSettingInGeneral() { } public void testDiagnoseMigrateDataRequiredToDataTiers() { + ProjectId projectId = randomProjectIdOrDefault(); // Index definition, 1 primary no replicas, in the hot tier, with require attribute data:hot IndexMetadata indexMetadata = IndexMetadata.builder("red-index") .settings( @@ -1499,7 +1594,7 @@ public void testDiagnoseMigrateDataRequiredToDataTiers() { .numberOfReplicas(0) .build(); - var service = createShardsAvailabilityIndicatorService(); + var service = createShardsAvailabilityIndicatorService(projectId); // Get the list of user actions that are generated for this unassigned index shard List actions = service.checkNodeRoleRelatedIssues( @@ -1523,6 +1618,7 @@ public void testDiagnoseMigrateDataRequiredToDataTiers() { } public void testDiagnoseMigrateDataIncludedToDataTiers() { + ProjectId projectId = randomProjectIdOrDefault(); // Index definition, 1 primary no replicas, in the hot tier, with include attribute data:hot IndexMetadata indexMetadata = IndexMetadata.builder("red-index") .settings( @@ -1536,7 +1632,7 @@ public void testDiagnoseMigrateDataIncludedToDataTiers() { .numberOfReplicas(0) .build(); - var service = createShardsAvailabilityIndicatorService(); + var service = createShardsAvailabilityIndicatorService(projectId); // Get the list of user actions that are generated for this unassigned index shard List actions = service.checkNodeRoleRelatedIssues( @@ -1560,6 +1656,7 @@ public void testDiagnoseMigrateDataIncludedToDataTiers() { } public void testDiagnoseOtherFilteringIssue() { + ProjectId projectId = randomProjectIdOrDefault(); // Index definition, 1 primary no replicas, in the hot tier IndexMetadata indexMetadata = IndexMetadata.builder("red-index") .settings( @@ -1572,7 +1669,7 @@ public void testDiagnoseOtherFilteringIssue() { .numberOfReplicas(0) .build(); - var service = createShardsAvailabilityIndicatorService(); + var service = createShardsAvailabilityIndicatorService(projectId); // Get the list of user actions that are generated for this unassigned index shard List actions = service.checkNodeRoleRelatedIssues( @@ -1596,6 +1693,7 @@ public void testDiagnoseOtherFilteringIssue() { } public void testDiagnoseIncreaseTierCapacity() { + ProjectId projectId = randomProjectIdOrDefault(); // Index definition, 1 primary no replicas, in the hot tier IndexMetadata indexMetadata = IndexMetadata.builder("red-index") .settings( @@ -1608,7 +1706,7 @@ public void testDiagnoseIncreaseTierCapacity() { .numberOfReplicas(0) .build(); - var service = createShardsAvailabilityIndicatorService(); + var service = createShardsAvailabilityIndicatorService(projectId); // Get the list of user actions that are generated for this unassigned index shard List actions = service.checkNodeRoleRelatedIssues( @@ -1636,6 +1734,7 @@ public void testDiagnoseIncreaseTierCapacity() { } public void testDiagnoseIncreaseNodeCapacity() { + ProjectId projectId = randomProjectIdOrDefault(); // Index definition, 1 primary no replicas, in the hot tier IndexMetadata indexMetadata = IndexMetadata.builder("red-index") .settings( @@ -1648,7 +1747,7 @@ public void testDiagnoseIncreaseNodeCapacity() { .numberOfReplicas(0) .build(); - var service = createShardsAvailabilityIndicatorService(); + var service = createShardsAvailabilityIndicatorService(projectId); // Get the list of user actions that are generated for this unassigned index shard List actions = service.checkNodeRoleRelatedIssues( @@ -1676,7 +1775,9 @@ public void testDiagnoseIncreaseNodeCapacity() { } public void testLimitNumberOfAffectedResources() { + ProjectId projectId = randomProjectIdOrDefault(); var clusterState = createClusterStateWith( + projectId, List.of( index("red-index1", new ShardAllocation(randomNodeId(), UNAVAILABLE)), index("red-index2", new ShardAllocation(randomNodeId(), UNAVAILABLE)), @@ -1686,7 +1787,7 @@ public void testLimitNumberOfAffectedResources() { ), List.of() ); - var service = createShardsAvailabilityIndicatorService(clusterState); + var service = createShardsAvailabilityIndicatorService(projectId, clusterState); { // assert the full result to check that details, impacts, and symptoms use the correct count of affected indices (5) @@ -1748,7 +1849,9 @@ public void testLimitNumberOfAffectedResources() { public void testShouldBeGreenWhenFrozenIndexIsUnassignedAndOriginalIsAvailable() { String originalIndex = "logs-2023.07.11-000024"; String restoredIndex = "restored-logs-2023.07.11-000024"; + ProjectId projectId = randomProjectIdOrDefault(); var clusterState = createClusterStateWith( + projectId, List.of( IndexMetadata.builder(restoredIndex) .settings( @@ -1775,7 +1878,7 @@ public void testShouldBeGreenWhenFrozenIndexIsUnassignedAndOriginalIsAvailable() List.of(), List.of() ); - var service = createShardsAvailabilityIndicatorService(clusterState); + var service = createShardsAvailabilityIndicatorService(projectId, clusterState); HealthIndicatorResult result = service.calculate(true, HealthInfo.EMPTY_HEALTH_INFO); assertThat( @@ -1817,8 +1920,9 @@ public void testShouldBeRedWhenFrozenIndexIsUnassignedAndOriginalIsUnavailable() routes.add(index(restoredIndex, new ShardAllocation(randomNodeId(), UNAVAILABLE))); // When original does not exist { - var clusterState = createClusterStateWith(indexMetadata, routes, List.of(), List.of()); - var service = createShardsAvailabilityIndicatorService(clusterState); + ProjectId projectId = randomProjectIdOrDefault(); + var clusterState = createClusterStateWith(projectId, indexMetadata, routes, List.of(), List.of()); + var service = createShardsAvailabilityIndicatorService(projectId, clusterState); HealthIndicatorResult result = service.calculate(true, HealthInfo.EMPTY_HEALTH_INFO); assertThat( @@ -1857,8 +1961,9 @@ public void testShouldBeRedWhenFrozenIndexIsUnassignedAndOriginalIsUnavailable() .build() ); routes.add(index(originalIndex, new ShardAllocation(randomNodeId(), UNAVAILABLE))); - var clusterState = createClusterStateWith(indexMetadata, routes, List.of(), List.of()); - var service = createShardsAvailabilityIndicatorService(clusterState); + ProjectId projectId = randomProjectIdOrDefault(); + var clusterState = createClusterStateWith(projectId, indexMetadata, routes, List.of(), List.of()); + var service = createShardsAvailabilityIndicatorService(projectId, clusterState); HealthIndicatorResult result = service.calculate(true, HealthInfo.EMPTY_HEALTH_INFO); assertThat( @@ -1992,7 +2097,8 @@ public void testMappedFieldsForTelemetry() { var service = new ShardsAvailabilityHealthIndicatorService( clusterService, mock(AllocationService.class), - mock(SystemIndices.class) + mock(SystemIndices.class), + DefaultProjectResolver.INSTANCE ); for (String tier : List.of("data_content", "data_hot", "data_warm", "data_cold", "data_frozen")) { assertThat( @@ -2023,7 +2129,7 @@ public void testMappedFieldsForTelemetry() { } public void testIsNewlyCreatedAndInitializingReplica() { - + ProjectId projectId = randomProjectIdOrDefault(); ShardId id = new ShardId("index", "uuid", 0); IndexMetadata idxMeta = IndexMetadata.builder("index") .numberOfShards(1) @@ -2048,9 +2154,10 @@ public void testIsNewlyCreatedAndInitializingReplica() { boolean primary = randomBoolean(); ShardAllocation primaryAllocation = new ShardAllocation("node", AVAILABLE); ShardRouting shard = createShardRouting(id, primary, primaryAllocation); - state = createClusterStateWith(List.of(index("index", primaryAllocation)), List.of()); + state = createClusterStateWith(projectId, List.of(index("index", primaryAllocation)), List.of()); assertFalse( ShardsAvailabilityHealthIndicatorService.isNewlyCreatedAndInitializingReplica( + projectId, shard, state, Instant.now().toEpochMilli() - replicaUnassignedThreshold.millis() @@ -2061,9 +2168,10 @@ public void testIsNewlyCreatedAndInitializingReplica() { { // primary, but not active var primaryAllocation = new ShardAllocation("node", INITIALIZING); ShardRouting primary = createShardRouting(id, true, primaryAllocation); - state = createClusterStateWith(List.of(index("index", primaryAllocation)), List.of()); + state = createClusterStateWith(projectId, List.of(index("index", primaryAllocation)), List.of()); assertFalse( ShardsAvailabilityHealthIndicatorService.isNewlyCreatedAndInitializingReplica( + projectId, primary, state, Instant.now().toEpochMilli() - replicaUnassignedThreshold.millis() @@ -2082,6 +2190,7 @@ public void testIsNewlyCreatedAndInitializingReplica() { ShardRouting unallocatedReplica = createShardRouting(id, false, replicaAllocation); state = createClusterStateWith( + projectId, List.of(idxMeta), List.of(index(idxMeta, primaryAllocation, replicaAllocation)), List.of(), @@ -2089,6 +2198,7 @@ public void testIsNewlyCreatedAndInitializingReplica() { ); assertFalse( ShardsAvailabilityHealthIndicatorService.isNewlyCreatedAndInitializingReplica( + projectId, unallocatedReplica, state, now - replicaUnassignedThreshold.millis() @@ -2103,6 +2213,7 @@ public void testIsNewlyCreatedAndInitializingReplica() { ShardRouting unallocatedReplica = createShardRouting(id, false, replicaAllocation); state = createClusterStateWith( + projectId, List.of(idxMeta), List.of(index(idxMeta, primaryAllocation, replicaAllocation)), List.of(), @@ -2110,6 +2221,7 @@ public void testIsNewlyCreatedAndInitializingReplica() { ); assertTrue( ShardsAvailabilityHealthIndicatorService.isNewlyCreatedAndInitializingReplica( + projectId, unallocatedReplica, state, now - replicaUnassignedThreshold.millis() @@ -2143,6 +2255,7 @@ public void testIsNewlyCreatedAndInitializingReplica() { var primaryAllocation = new ShardAllocation("node", config.v1()); ShardRouting unallocatedReplica = createShardRouting(id, false, replicaAllocation); state = createClusterStateWith( + projectId, List.of(idxMeta), List.of(index(idxMeta, primaryAllocation, replicaAllocation)), List.of(), @@ -2150,6 +2263,7 @@ public void testIsNewlyCreatedAndInitializingReplica() { ); assertFalse( ShardsAvailabilityHealthIndicatorService.isNewlyCreatedAndInitializingReplica( + projectId, unallocatedReplica, state, now - replicaUnassignedThreshold.millis() @@ -2177,9 +2291,10 @@ public void testIsNewlyCreatedAndInitializingReplica() { ShardRouting unallocatedReplica = createShardRouting(id, false, replicaAllocation); IndexRoutingTable index = index(idxMeta, primaryAllocation, replicaAllocation); - state = createClusterStateWith(List.of(idxMeta), List.of(index), List.of(), List.of()); + state = createClusterStateWith(projectId, List.of(idxMeta), List.of(index), List.of(), List.of()); assertTrue( ShardsAvailabilityHealthIndicatorService.isNewlyCreatedAndInitializingReplica( + projectId, unallocatedReplica, state, now - replicaUnassignedThreshold.millis() @@ -2189,6 +2304,66 @@ public void testIsNewlyCreatedAndInitializingReplica() { } } + public void testMultiProjectShouldDisplayProjectId() { + ProjectId projectId1 = randomUniqueProjectId(); + ProjectId projectId2 = randomUniqueProjectId(); + String index1 = "index-1"; + String index2 = "index-2"; + + // 3 indices from 2 projects + var clusterState = createClusterStateWith( + Map.of( + projectId1, + List.of( + index(index1, new ShardAllocation(randomNodeId(), UNAVAILABLE)), + index(index2, new ShardAllocation(randomNodeId(), UNAVAILABLE)) + ), + projectId2, + List.of(index(index1, new ShardAllocation(randomNodeId(), UNAVAILABLE))) + ), + List.of() + ); + + var service = createAllocationHealthIndicatorService( + Settings.EMPTY, + clusterState, + Collections.emptyMap(), + new SystemIndices(List.of()), + TestProjectResolvers.allProjects() + ); + + List indexDisplayNames = Stream.of( + projectId1 + ProjectIndexName.DELIMITER + index1, + projectId1 + ProjectIndexName.DELIMITER + index2, + projectId2 + ProjectIndexName.DELIMITER + index1 + ).sorted(Comparator.naturalOrder()).toList(); + + assertThat( + service.calculate(true, HealthInfo.EMPTY_HEALTH_INFO), + equalTo( + createExpectedResult( + RED, + "This cluster has 3 unavailable primary shards.", + Map.of("unassigned_primaries", 3), + List.of( + new HealthIndicatorImpact( + NAME, + ShardsAvailabilityHealthIndicatorService.PRIMARY_UNASSIGNED_IMPACT_ID, + 1, + String.format( + Locale.ROOT, + "Cannot add data to 3 indices [%s]. Searches might return incomplete results.", + String.join(", ", indexDisplayNames) + ), + List.of(ImpactArea.INGEST, ImpactArea.SEARCH) + ) + ), + List.of(new Diagnosis(ACTION_CHECK_ALLOCATION_EXPLAIN_API, List.of(new Diagnosis.Resource(INDEX, indexDisplayNames)))) + ) + ) + ); + } + private HealthIndicatorResult createExpectedResult( HealthStatus status, String symptom, @@ -2210,17 +2385,34 @@ private HealthIndicatorResult createExpectedTruncatedResult(HealthStatus status, return new HealthIndicatorResult(NAME, status, symptom, HealthIndicatorDetails.EMPTY, impacts, emptyList()); } - private static ClusterState createClusterStateWith(List indexRoutes, List nodeShutdowns) { - List indices = indexRoutes.stream() - .map( - table -> IndexMetadata.builder(table.getIndex().getName()) - .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()).build()) - .numberOfShards(1) - .numberOfReplicas(table.size() - 1) - .build() - ) - .collect(Collectors.toList()); - return createClusterStateWith(indices, indexRoutes, nodeShutdowns, List.of()); + private static ClusterState createClusterStateWith( + ProjectId projectId, + List indexRoutes, + List nodeShutdowns + ) { + Map> defaultProjectRoutes = Map.of(projectId, indexRoutes); + return createClusterStateWith(defaultProjectRoutes, nodeShutdowns); + } + + private static ClusterState createClusterStateWith( + Map> projectIndexRoutes, + List nodeShutdowns + ) { + Map> projectIndices = new HashMap<>(); + for (var entry : projectIndexRoutes.entrySet()) { + List indices = entry.getValue() + .stream() + .map( + table -> IndexMetadata.builder(table.getIndex().getName()) + .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()).build()) + .numberOfShards(1) + .numberOfReplicas(table.size() - 1) + .build() + ) + .toList(); + projectIndices.put(entry.getKey(), indices); + } + return createClusterStateWith(projectIndices, projectIndexRoutes, nodeShutdowns, List.of()); } private static List createIndexMetadataForIndexNameToPriorityMap(Map indexNameToPriorityMap) { @@ -2241,14 +2433,48 @@ private static List createIndexMetadataForIndexNameToPriorityMap( } private static ClusterState createClusterStateWith( + ProjectId projectId, List indexMetadataList, List indexRoutingTables, List nodeShutdowns, List nodes ) { - var routingTableBuilder = RoutingTable.builder(); - for (IndexRoutingTable indexRoutingTable : indexRoutingTables) { - routingTableBuilder.add(indexRoutingTable); + Map> defaultProjectMetadata = Map.of(projectId, indexMetadataList); + Map> defaultProjectRoutes = Map.of(projectId, indexRoutingTables); + return createClusterStateWith(defaultProjectMetadata, defaultProjectRoutes, nodeShutdowns, nodes); + } + + private static ClusterState createClusterStateWith( + Map> projectIndexMetadataList, + Map> projectIndexRoutingTables, + List nodeShutdowns, + List nodes + ) { + // build routing tables + var globalRoutingTableBuilder = GlobalRoutingTable.builder(); + for (var entries : projectIndexRoutingTables.entrySet()) { + ProjectId projectId = entries.getKey(); + List routingTables = entries.getValue(); + + var builder = RoutingTable.builder(); + for (IndexRoutingTable indexRoutingTable : routingTables) { + builder.add(indexRoutingTable); + } + + globalRoutingTableBuilder.put(projectId, builder.build()); + } + + // build Metadata + Metadata.Builder metadataBuilder = Metadata.builder(); + + for (var entries : projectIndexMetadataList.entrySet()) { + ProjectId projectId = entries.getKey(); + List indexMetadataList = entries.getValue(); + Map indexMetadataMap = indexMetadataList.stream() + .collect(toMap(indexMetadata -> indexMetadata.getIndex().getName(), indexMetadata -> indexMetadata)); + + ProjectMetadata projectMetadata = ProjectMetadata.builder(projectId).indices(indexMetadataMap).build(); + metadataBuilder.put(projectMetadata); } var nodesShutdownMetadata = new NodesShutdownMetadata( @@ -2268,19 +2494,14 @@ private static ClusterState createClusterStateWith( ) ) ); - Metadata.Builder metadataBuilder = Metadata.builder(); - Map indexMetadataMap = new HashMap<>(); - for (IndexMetadata indexMetadata : indexMetadataList) { - indexMetadataMap.put(indexMetadata.getIndex().getName(), indexMetadata); - } - metadataBuilder.indices(indexMetadataMap); metadataBuilder.putCustom(NodesShutdownMetadata.TYPE, nodesShutdownMetadata); + // build nodes DiscoveryNodes.Builder discoveryNodesBuilder = DiscoveryNodes.builder(); nodes.forEach(discoveryNodesBuilder::add); return ClusterState.builder(new ClusterName("test-cluster")) - .routingTable(routingTableBuilder.build()) + .routingTable(globalRoutingTableBuilder.build()) .nodes(discoveryNodesBuilder) .metadata(metadataBuilder.build()) .build(); @@ -2531,34 +2752,52 @@ private static UnassignedInfo decidersNo(TimeValue unassignedTime) { private record ShardRoutingKey(String index, int shard, boolean primary) {} - private static ShardsAvailabilityHealthIndicatorService createShardsAvailabilityIndicatorService() { - return createShardsAvailabilityIndicatorService(ClusterState.EMPTY_STATE, Collections.emptyMap()); + private static ShardsAvailabilityHealthIndicatorService createShardsAvailabilityIndicatorService(ProjectId projectId) { + return createShardsAvailabilityIndicatorService(projectId, ClusterState.EMPTY_STATE, Collections.emptyMap()); } - private static ShardsAvailabilityHealthIndicatorService createShardsAvailabilityIndicatorService(ClusterState clusterState) { - return createShardsAvailabilityIndicatorService(clusterState, Collections.emptyMap()); + private static ShardsAvailabilityHealthIndicatorService createShardsAvailabilityIndicatorService( + ProjectId projectId, + ClusterState clusterState + ) { + return createShardsAvailabilityIndicatorService(projectId, clusterState, Collections.emptyMap()); } private static ShardsAvailabilityHealthIndicatorService createShardsAvailabilityIndicatorService( + ProjectId projectId, ClusterState clusterState, final Map decisions ) { - return createAllocationHealthIndicatorService(Settings.EMPTY, clusterState, decisions, new SystemIndices(List.of())); + return createAllocationHealthIndicatorService( + Settings.EMPTY, + clusterState, + decisions, + new SystemIndices(List.of()), + TestProjectResolvers.singleProjectOnly(projectId) + ); } private static ShardsAvailabilityHealthIndicatorService createShardsAvailabilityIndicatorService( + ProjectId projectId, Settings nodeSettings, ClusterState clusterState, final Map decisions ) { - return createAllocationHealthIndicatorService(nodeSettings, clusterState, decisions, new SystemIndices(List.of())); + return createAllocationHealthIndicatorService( + nodeSettings, + clusterState, + decisions, + new SystemIndices(List.of()), + TestProjectResolvers.singleProjectOnly(projectId) + ); } private static ShardsAvailabilityHealthIndicatorService createAllocationHealthIndicatorService( Settings nodeSettings, ClusterState clusterState, final Map decisions, - SystemIndices systemIndices + SystemIndices systemIndices, + ProjectResolver projectResolver ) { var clusterService = mock(ClusterService.class); when(clusterService.state()).thenReturn(clusterState); @@ -2573,6 +2812,6 @@ private static ShardsAvailabilityHealthIndicatorService createAllocationHealthIn return decisions.getOrDefault(key, ShardAllocationDecision.NOT_TAKEN); } ); - return new ShardsAvailabilityHealthIndicatorService(clusterService, allocationService, systemIndices); + return new ShardsAvailabilityHealthIndicatorService(clusterService, allocationService, systemIndices, projectResolver); } } diff --git a/x-pack/qa/multi-project/xpack-rest-tests-with-multiple-projects/src/yamlRestTest/resources/rest-api-spec/test/health/10_multi_project.yml b/x-pack/qa/multi-project/xpack-rest-tests-with-multiple-projects/src/yamlRestTest/resources/rest-api-spec/test/health/10_multi_project.yml new file mode 100644 index 0000000000000..583753ce365e5 --- /dev/null +++ b/x-pack/qa/multi-project/xpack-rest-tests-with-multiple-projects/src/yamlRestTest/resources/rest-api-spec/test/health/10_multi_project.yml @@ -0,0 +1,40 @@ +--- +"Health indicator shards_availability for multi-project enabled cluster": + - requires: + test_runner_features: capabilities + capabilities: + - method: GET + path: /_health_report + capabilities: [ multi_project_shards_availability ] + reason: Capability required to run test + - do: + health_report: + feature: master_is_stable + + - is_true: cluster_name + - match: { indicators.master_is_stable.status: "green" } + + - do: + indices.create: + index: red_index + master_timeout: 1s + timeout: 1s + body: + settings: + number_of_shards: 1 + number_of_replicas: 0 + index.routing.allocation.enable: none + + - do: + health_report: + feature: shards_availability + + - is_true: cluster_name + - match: { indicators.shards_availability.status: "red" } + - match: { indicators.shards_availability.symptom: "This cluster has 1 unavailable primary shard." } + - is_true: indicators.shards_availability.diagnosis + - length: { indicators.shards_availability.diagnosis: 1 } + - is_true: indicators.shards_availability.diagnosis.0.affected_resources + - length: { indicators.shards_availability.diagnosis.0.affected_resources: 1 } + # regex match project index name + - match: { indicators.shards_availability.diagnosis.0.affected_resources.indices.0: "/.*\\/red_index/" }