From d4c3e22ac0ee97c22ad69e7f45db8819edd41540 Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Tue, 24 Jun 2025 17:05:34 +1000 Subject: [PATCH 01/12] Make SnapshotsService and related APIs support multi-project --- .../snapshots/SnapshotsServiceIT.java | 5 +- .../TransportCleanupRepositoryAction.java | 23 +- .../clone/TransportCloneSnapshotAction.java | 10 +- .../create/TransportCreateSnapshotAction.java | 16 +- .../delete/TransportDeleteSnapshotAction.java | 10 +- .../get/TransportGetSnapshotsAction.java | 46 ++- .../TransportSnapshotsStatusAction.java | 111 ++++-- .../cluster/SnapshotDeletionsInProgress.java | 3 +- .../cluster/SnapshotsInProgress.java | 27 +- .../repositories/FinalizeSnapshotContext.java | 3 +- .../blobstore/BlobStoreRepository.java | 31 +- .../elasticsearch/snapshots/SnapshotInfo.java | 5 + .../snapshots/SnapshotsService.java | 364 ++++++++++-------- .../TransportSnapshotsStatusActionTests.java | 5 +- .../snapshots/SnapshotResiliencyTests.java | 45 ++- 15 files changed, 429 insertions(+), 275 deletions(-) diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotsServiceIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotsServiceIT.java index 24539a270c03e..cf2e4c54f121b 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotsServiceIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotsServiceIT.java @@ -16,6 +16,7 @@ import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.cluster.SnapshotDeletionsInProgress; import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.cluster.service.MasterService; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.snapshots.mockstore.MockRepository; @@ -176,10 +177,10 @@ private SubscribableListener createSnapshotDeletionListener(String reposit return false; } if (deleteHasStarted.get() == false) { - deleteHasStarted.set(deletionsInProgress.hasExecutingDeletion(repositoryName)); + deleteHasStarted.set(deletionsInProgress.hasExecutingDeletion(ProjectId.DEFAULT, repositoryName)); return false; } else { - return deletionsInProgress.hasExecutingDeletion(repositoryName) == false; + return deletionsInProgress.hasExecutingDeletion(ProjectId.DEFAULT, repositoryName) == false; } }); } diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/cleanup/TransportCleanupRepositoryAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/cleanup/TransportCleanupRepositoryAction.java index cd749901eff92..9dda15a21ce30 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/cleanup/TransportCleanupRepositoryAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/cleanup/TransportCleanupRepositoryAction.java @@ -22,12 +22,13 @@ import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.metadata.ProjectId; +import org.elasticsearch.cluster.metadata.ProjectMetadata; import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.project.ProjectResolver; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.blobstore.DeleteResult; import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.common.util.concurrent.ListenableFuture; -import org.elasticsearch.core.FixForMultiProject; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.injection.guice.Inject; @@ -66,6 +67,7 @@ public final class TransportCleanupRepositoryAction extends TransportMasterNodeA private static final Logger logger = LogManager.getLogger(TransportCleanupRepositoryAction.class); private final RepositoriesService repositoriesService; + private final ProjectResolver projectResolver; @Inject public TransportCleanupRepositoryAction( @@ -73,7 +75,8 @@ public TransportCleanupRepositoryAction( ClusterService clusterService, RepositoriesService repositoriesService, ThreadPool threadPool, - ActionFilters actionFilters + ActionFilters actionFilters, + ProjectResolver projectResolver ) { super( TYPE.name(), @@ -86,6 +89,7 @@ public TransportCleanupRepositoryAction( EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.repositoriesService = repositoriesService; + this.projectResolver = projectResolver; // We add a state applier that will remove any dangling repository cleanup actions on master failover. // This is safe to do since cleanups will increment the repository state id before executing any operations to prevent concurrent // operations from corrupting the repository. This is the same safety mechanism used by snapshot deletes. @@ -134,7 +138,7 @@ protected void masterOperation( ClusterState state, ActionListener listener ) { - cleanupRepo(request.name(), listener.map(CleanupRepositoryResponse::new)); + cleanupRepo(projectResolver.getProjectId(), request.name(), listener.map(CleanupRepositoryResponse::new)); } @Override @@ -145,11 +149,12 @@ protected ClusterBlockException checkBlock(CleanupRepositoryRequest request, Clu /** * Runs cleanup operations on the given repository. + * @param projectId Project for the repository * @param repositoryName Repository to clean up * @param listener Listener for cleanup result */ - private void cleanupRepo(String repositoryName, ActionListener listener) { - final Repository repository = repositoriesService.repository(repositoryName); + private void cleanupRepo(ProjectId projectId, String repositoryName, ActionListener listener) { + final Repository repository = repositoriesService.repository(projectId, repositoryName); if (repository instanceof BlobStoreRepository == false) { listener.onFailure(new IllegalArgumentException("Repository [" + repositoryName + "] does not support repository cleanup")); return; @@ -172,8 +177,10 @@ private void cleanupRepo(String repositoryName, ActionListener TYPE = new ActionType<>("cluster:admin/snapshot/clone"); private final SnapshotsService snapshotsService; + private final ProjectResolver projectResolver; @Inject public TransportCloneSnapshotAction( @@ -39,7 +41,8 @@ public TransportCloneSnapshotAction( ClusterService clusterService, ThreadPool threadPool, SnapshotsService snapshotsService, - ActionFilters actionFilters + ActionFilters actionFilters, + ProjectResolver projectResolver ) { super( TYPE.name(), @@ -51,12 +54,13 @@ public TransportCloneSnapshotAction( EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.snapshotsService = snapshotsService; + this.projectResolver = projectResolver; } @Override protected ClusterBlockException checkBlock(CloneSnapshotRequest request, ClusterState state) { // Cluster is not affected but we look up repositories in metadata - return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_READ); + return state.blocks().globalBlockedException(projectResolver.getProjectId(), ClusterBlockLevel.METADATA_READ); } @Override @@ -66,6 +70,6 @@ protected void masterOperation( ClusterState state, final ActionListener listener ) { - snapshotsService.cloneSnapshot(request, listener.map(v -> AcknowledgedResponse.TRUE)); + snapshotsService.cloneSnapshot(projectResolver.getProjectId(), request, listener.map(v -> AcknowledgedResponse.TRUE)); } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/TransportCreateSnapshotAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/TransportCreateSnapshotAction.java index f544e07ff3d51..05942d65b6e81 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/TransportCreateSnapshotAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/TransportCreateSnapshotAction.java @@ -16,6 +16,7 @@ import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.block.ClusterBlockLevel; +import org.elasticsearch.cluster.project.ProjectResolver; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.injection.guice.Inject; @@ -31,6 +32,7 @@ public class TransportCreateSnapshotAction extends TransportMasterNodeAction { public static final ActionType TYPE = new ActionType<>("cluster:admin/snapshot/create"); private final SnapshotsService snapshotsService; + private final ProjectResolver projectResolver; @Inject public TransportCreateSnapshotAction( @@ -38,7 +40,8 @@ public TransportCreateSnapshotAction( ClusterService clusterService, ThreadPool threadPool, SnapshotsService snapshotsService, - ActionFilters actionFilters + ActionFilters actionFilters, + ProjectResolver projectResolver ) { super( TYPE.name(), @@ -51,12 +54,13 @@ public TransportCreateSnapshotAction( EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.snapshotsService = snapshotsService; + this.projectResolver = projectResolver; } @Override protected ClusterBlockException checkBlock(CreateSnapshotRequest request, ClusterState state) { // We only check metadata block, as we want to snapshot closed indices (which have a read block) - return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_READ); + return state.blocks().globalBlockedException(projectResolver.getProjectId(), ClusterBlockLevel.METADATA_READ); } @Override @@ -67,9 +71,13 @@ protected void masterOperation( final ActionListener listener ) { if (request.waitForCompletion()) { - snapshotsService.executeSnapshot(request, listener.map(CreateSnapshotResponse::new)); + snapshotsService.executeSnapshot(projectResolver.getProjectId(), request, listener.map(CreateSnapshotResponse::new)); } else { - snapshotsService.createSnapshot(request, listener.map(snapshot -> new CreateSnapshotResponse((SnapshotInfo) null))); + snapshotsService.createSnapshot( + projectResolver.getProjectId(), + request, + listener.map(snapshot -> new CreateSnapshotResponse((SnapshotInfo) null)) + ); } } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/delete/TransportDeleteSnapshotAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/delete/TransportDeleteSnapshotAction.java index 639452c833197..9fffdac13fae4 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/delete/TransportDeleteSnapshotAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/delete/TransportDeleteSnapshotAction.java @@ -18,6 +18,7 @@ import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.block.ClusterBlockLevel; +import org.elasticsearch.cluster.project.ProjectResolver; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.injection.guice.Inject; @@ -32,6 +33,7 @@ public class TransportDeleteSnapshotAction extends AcknowledgedTransportMasterNodeAction { public static final ActionType TYPE = new ActionType<>("cluster:admin/snapshot/delete"); private final SnapshotsService snapshotsService; + private final ProjectResolver projectResolver; @Inject public TransportDeleteSnapshotAction( @@ -39,7 +41,8 @@ public TransportDeleteSnapshotAction( ClusterService clusterService, ThreadPool threadPool, SnapshotsService snapshotsService, - ActionFilters actionFilters + ActionFilters actionFilters, + ProjectResolver projectResolver ) { super( TYPE.name(), @@ -51,12 +54,13 @@ public TransportDeleteSnapshotAction( EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.snapshotsService = snapshotsService; + this.projectResolver = projectResolver; } @Override protected ClusterBlockException checkBlock(DeleteSnapshotRequest request, ClusterState state) { // Cluster is not affected but we look up repositories in metadata - return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_READ); + return state.blocks().globalBlockedException(projectResolver.getProjectId(), ClusterBlockLevel.METADATA_READ); } @Override @@ -74,6 +78,6 @@ protected void masterOperation( ClusterState state, final ActionListener listener ) { - snapshotsService.deleteSnapshots(request, listener.map(v -> AcknowledgedResponse.TRUE)); + snapshotsService.deleteSnapshots(projectResolver.getProjectId(), request, listener.map(v -> AcknowledgedResponse.TRUE)); } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java index 7393ab23d8896..a98e364bb3d56 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java @@ -19,7 +19,9 @@ import org.elasticsearch.cluster.SnapshotsInProgress; import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.block.ClusterBlockLevel; +import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.cluster.metadata.RepositoryMetadata; +import org.elasticsearch.cluster.project.ProjectResolver; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Strings; import org.elasticsearch.common.collect.Iterators; @@ -78,6 +80,7 @@ public class TransportGetSnapshotsAction extends TransportMasterNodeAction repositories; @@ -217,6 +225,7 @@ private class GetSnapshotsOperation { GetSnapshotsOperation( CancellableTask cancellableTask, + ProjectId projectId, List repositories, String[] snapshots, boolean ignoreUnavailable, @@ -233,6 +242,7 @@ private class GetSnapshotsOperation { EnumSet states ) { this.cancellableTask = cancellableTask; + this.projectId = projectId; this.repositories = repositories; this.ignoreUnavailable = ignoreUnavailable; this.sortBy = sortBy; @@ -310,7 +320,10 @@ private void populateResults(ActionListener listener) { assert ThreadPool.assertCurrentThreadPool(ThreadPool.Names.MANAGEMENT); cancellableTask.ensureNotCancelled(); ensureRequiredNamesPresent(repositoryName, repositoryData); - return getAsyncSnapshotInfoIterator(repositoriesService.repository(repositoryName), repositoryData); + return getAsyncSnapshotInfoIterator( + repositoriesService.repository(projectId, repositoryName), + repositoryData + ); }) .addListener(asyncRepositoryContentsListener) ), @@ -362,7 +375,7 @@ private void maybeGetRepositoryData(String repositoryName, ActionListener(snapshotNamePredicate.requiredNames()); - for (final var snapshotInProgress : snapshotsInProgress.forRepo(repositoryName)) { + for (final var snapshotInProgress : snapshotsInProgress.forRepo(projectId, repositoryName)) { unmatchedRequiredNames.remove(snapshotInProgress.snapshot().getSnapshotId().getName()); } if (unmatchedRequiredNames.isEmpty()) { @@ -464,7 +477,7 @@ public void getSnapshotInfo(ActionListener listener) { ActionListener.completeWith( listener, () -> new SnapshotInfo( - new Snapshot(repository.getMetadata().name(), snapshotId), + new Snapshot(repository.getProjectId(), repository.getMetadata().name(), snapshotId), indicesLookup.getOrDefault(snapshotId, Collections.emptyList()), Collections.emptyList(), Collections.emptyList(), @@ -491,15 +504,18 @@ private Iterator getAsyncSnapshotInfoIterator(Repository repo return Iterators.concat( // matching in-progress snapshots first Iterators.map( - Iterators.filter(snapshotsInProgress.forRepo(repository.getMetadata().name()).iterator(), snapshotInProgress -> { - final var snapshotId = snapshotInProgress.snapshot().getSnapshotId(); - if (snapshotNamePredicate.test(snapshotId.getName(), true)) { - matchingInProgressSnapshots.add(snapshotId); - return true; - } else { - return false; + Iterators.filter( + snapshotsInProgress.forRepo(repository.getProjectId(), repository.getMetadata().name()).iterator(), + snapshotInProgress -> { + final var snapshotId = snapshotInProgress.snapshot().getSnapshotId(); + if (snapshotNamePredicate.test(snapshotId.getName(), true)) { + matchingInProgressSnapshots.add(snapshotId); + return true; + } else { + return false; + } } - }), + ), this::forSnapshotInProgress ), repositoryData == null diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportSnapshotsStatusAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportSnapshotsStatusAction.java index fbb3815eab079..a05a183d7f7e2 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportSnapshotsStatusAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportSnapshotsStatusAction.java @@ -25,12 +25,12 @@ import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.ProjectId; +import org.elasticsearch.cluster.project.ProjectResolver; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Strings; import org.elasticsearch.common.util.CollectionUtils; import org.elasticsearch.common.util.concurrent.ListenableFuture; import org.elasticsearch.common.util.set.Sets; -import org.elasticsearch.core.FixForMultiProject; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.index.snapshots.IndexShardSnapshotStatus; import org.elasticsearch.injection.guice.Inject; @@ -76,6 +76,7 @@ public class TransportSnapshotsStatusAction extends TransportMasterNodeAction currentSnapshots = SnapshotsService.currentSnapshots( snapshotsInProgress, + projectId, request.repository(), Arrays.asList(request.snapshots()) ); if (currentSnapshots.isEmpty()) { - buildResponse(snapshotsInProgress, request, currentSnapshots, null, state.getMinTransportVersion(), cancellableTask, listener); + buildResponse( + snapshotsInProgress, + projectId, + request, + currentSnapshots, + null, + state.getMinTransportVersion(), + cancellableTask, + listener + ); return; } @@ -153,6 +167,7 @@ protected void masterOperation( listener.delegateFailureAndWrap( (l, nodeSnapshotStatuses) -> buildResponse( snapshotsInProgress, + projectId, request, currentSnapshots, nodeSnapshotStatuses, @@ -165,7 +180,16 @@ protected void masterOperation( ); } else { // We don't have any in-progress shards, just return current stats - buildResponse(snapshotsInProgress, request, currentSnapshots, null, state.getMinTransportVersion(), cancellableTask, listener); + buildResponse( + snapshotsInProgress, + projectId, + request, + currentSnapshots, + null, + state.getMinTransportVersion(), + cancellableTask, + listener + ); } } @@ -173,6 +197,7 @@ protected void masterOperation( // Package access for testing. void buildResponse( SnapshotsInProgress snapshotsInProgress, + ProjectId projectId, SnapshotsStatusRequest request, List currentSnapshotEntries, TransportNodesSnapshotsStatus.NodesSnapshotStatus nodeSnapshotStatuses, @@ -282,7 +307,7 @@ void buildResponse( // Now add snapshots on disk that are not currently running final String repositoryName = request.repository(); if (Strings.hasText(repositoryName) && CollectionUtils.isEmpty(request.snapshots()) == false) { - loadRepositoryData(snapshotsInProgress, request, builder, currentSnapshotNames, repositoryName, task, listener); + loadRepositoryData(snapshotsInProgress, request, builder, currentSnapshotNames, projectId, repositoryName, task, listener); } else { listener.onResponse(new SnapshotsStatusResponse(Collections.unmodifiableList(builder))); } @@ -293,14 +318,13 @@ private void loadRepositoryData( SnapshotsStatusRequest request, List builder, Set currentSnapshotNames, + ProjectId projectId, String repositoryName, CancellableTask task, ActionListener listener ) { final Set requestedSnapshotNames = Sets.newHashSet(request.snapshots()); final ListenableFuture repositoryDataListener = new ListenableFuture<>(); - @FixForMultiProject(description = "resolve the actual projectId, ES-10166") - final var projectId = ProjectId.DEFAULT; repositoriesService.getRepositoryData(projectId, repositoryName, repositoryDataListener); final Collection snapshotIdsToLoad = new ArrayList<>(); repositoryDataListener.addListener(listener.delegateFailureAndWrap((delegate, repositoryData) -> { @@ -329,7 +353,7 @@ private void loadRepositoryData( throw new SnapshotMissingException(repositoryName, snapshotName); } } - if (snapshotsInProgress.snapshot(new Snapshot(repositoryName, snapshotId)) == null) { + if (snapshotsInProgress.snapshot(new Snapshot(projectId, repositoryName, snapshotId)) == null) { snapshotIdsToLoad.add(snapshotId); } } @@ -338,38 +362,39 @@ private void loadRepositoryData( delegate.onResponse(new SnapshotsStatusResponse(Collections.unmodifiableList(builder))); } else { final List threadSafeBuilder = Collections.synchronizedList(builder); - repositoriesService.repository(repositoryName).getSnapshotInfo(snapshotIdsToLoad, true, task::isCancelled, snapshotInfo -> { - List shardStatusBuilder = new ArrayList<>(); - final Map shardStatuses; - shardStatuses = snapshotShards(repositoryName, repositoryData, task, snapshotInfo); - // an exception here stops further fetches of snapshotInfo since context is fail-fast - for (final var shardStatus : shardStatuses.entrySet()) { - IndexShardSnapshotStatus.Copy lastSnapshotStatus = shardStatus.getValue(); - shardStatusBuilder.add(new SnapshotIndexShardStatus(shardStatus.getKey(), lastSnapshotStatus)); - } - final SnapshotsInProgress.State state = switch (snapshotInfo.state()) { - case FAILED -> SnapshotsInProgress.State.FAILED; - case SUCCESS, PARTIAL -> - // Both of these means the snapshot has completed. - SnapshotsInProgress.State.SUCCESS; - default -> throw new IllegalArgumentException("Unexpected snapshot state " + snapshotInfo.state()); - }; - final long startTime = snapshotInfo.startTime(); - final long endTime = snapshotInfo.endTime(); - assert endTime >= startTime || (endTime == 0L && snapshotInfo.state().completed() == false) - : "Inconsistent timestamps found in SnapshotInfo [" + snapshotInfo + "]"; - threadSafeBuilder.add( - new SnapshotStatus( - new Snapshot(repositoryName, snapshotInfo.snapshotId()), - state, - Collections.unmodifiableList(shardStatusBuilder), - snapshotInfo.includeGlobalState(), - startTime, - // Use current time to calculate overall runtime for in-progress snapshots that have endTime == 0 - (endTime == 0 ? threadPool.absoluteTimeInMillis() : endTime) - startTime - ) - ); - }, delegate.map(v -> new SnapshotsStatusResponse(List.copyOf(threadSafeBuilder)))); + repositoriesService.repository(projectId, repositoryName) + .getSnapshotInfo(snapshotIdsToLoad, true, task::isCancelled, snapshotInfo -> { + List shardStatusBuilder = new ArrayList<>(); + final Map shardStatuses; + shardStatuses = snapshotShards(projectId, repositoryName, repositoryData, task, snapshotInfo); + // an exception here stops further fetches of snapshotInfo since context is fail-fast + for (final var shardStatus : shardStatuses.entrySet()) { + IndexShardSnapshotStatus.Copy lastSnapshotStatus = shardStatus.getValue(); + shardStatusBuilder.add(new SnapshotIndexShardStatus(shardStatus.getKey(), lastSnapshotStatus)); + } + final SnapshotsInProgress.State state = switch (snapshotInfo.state()) { + case FAILED -> SnapshotsInProgress.State.FAILED; + case SUCCESS, PARTIAL -> + // Both of these means the snapshot has completed. + SnapshotsInProgress.State.SUCCESS; + default -> throw new IllegalArgumentException("Unexpected snapshot state " + snapshotInfo.state()); + }; + final long startTime = snapshotInfo.startTime(); + final long endTime = snapshotInfo.endTime(); + assert endTime >= startTime || (endTime == 0L && snapshotInfo.state().completed() == false) + : "Inconsistent timestamps found in SnapshotInfo [" + snapshotInfo + "]"; + threadSafeBuilder.add( + new SnapshotStatus( + new Snapshot(projectId, repositoryName, snapshotInfo.snapshotId()), + state, + Collections.unmodifiableList(shardStatusBuilder), + snapshotInfo.includeGlobalState(), + startTime, + // Use current time to calculate overall runtime for in-progress snapshots that have endTime == 0 + (endTime == 0 ? threadPool.absoluteTimeInMillis() : endTime) - startTime + ) + ); + }, delegate.map(v -> new SnapshotsStatusResponse(List.copyOf(threadSafeBuilder)))); } })); } @@ -382,17 +407,19 @@ private void loadRepositoryData( * returns similar information but for already finished snapshots. *

* + * @param projectId project for the repository * @param repositoryName repository name * @param snapshotInfo snapshot info * @return map of shard id to snapshot status */ private Map snapshotShards( + final ProjectId projectId, final String repositoryName, final RepositoryData repositoryData, final CancellableTask task, final SnapshotInfo snapshotInfo ) throws IOException { - final Repository repository = repositoriesService.repository(repositoryName); + final Repository repository = repositoriesService.repository(projectId, repositoryName); final Map shardStatus = new HashMap<>(); for (String index : snapshotInfo.indices()) { IndexId indexId = repositoryData.resolveIndexId(index); diff --git a/server/src/main/java/org/elasticsearch/cluster/SnapshotDeletionsInProgress.java b/server/src/main/java/org/elasticsearch/cluster/SnapshotDeletionsInProgress.java index 8136719612ce6..cbd4c2ed14ca4 100644 --- a/server/src/main/java/org/elasticsearch/cluster/SnapshotDeletionsInProgress.java +++ b/server/src/main/java/org/elasticsearch/cluster/SnapshotDeletionsInProgress.java @@ -114,9 +114,10 @@ public List getEntries() { /** * Checks if there is an actively executing delete operation for the given repository * + * @param projectId project to look for the repository * @param repository repository name */ - public boolean hasExecutingDeletion(String repository) { + public boolean hasExecutingDeletion(ProjectId projectId, String repository) { for (Entry entry : entries) { if (entry.state() == State.STARTED && entry.repository().equals(repository)) { return true; diff --git a/server/src/main/java/org/elasticsearch/cluster/SnapshotsInProgress.java b/server/src/main/java/org/elasticsearch/cluster/SnapshotsInProgress.java index 8390c00fc420a..146de4ee37f2c 100644 --- a/server/src/main/java/org/elasticsearch/cluster/SnapshotsInProgress.java +++ b/server/src/main/java/org/elasticsearch/cluster/SnapshotsInProgress.java @@ -174,11 +174,15 @@ public List forRepo(String repository) { return forRepo(Metadata.DEFAULT_PROJECT_ID, repository); } + public List forRepo(ProjectId projectId, String repository) { + return forRepo(new ProjectRepo(projectId, repository)); + } + /** * Returns the list of snapshots in the specified repository. */ - public List forRepo(ProjectId projectId, String repository) { - return entries.getOrDefault(new ProjectRepo(projectId, repository), ByRepo.EMPTY).entries; + public List forRepo(ProjectRepo projectRepo) { + return entries.getOrDefault(projectRepo, ByRepo.EMPTY).entries; } public boolean isEmpty() { @@ -221,25 +225,6 @@ private static Entry findSnapshotInList(Snapshot snapshotToFind, List for return null; } - /** - * Computes a map of repository shard id to set of shard generations, containing all shard generations that became obsolete and may be - * deleted from the repository as the cluster state moves from the given old value of {@link SnapshotsInProgress} to this instance. - *

- * An unique shard generation is created for every in-progress shard snapshot. The shard generation file contains information about all - * the files needed by pre-existing and any new shard snapshots that were in-progress. When a shard snapshot is finalized, its file list - * is promoted to the official shard snapshot list for the index shard. This final list will contain metadata about any other - * in-progress shard snapshots that were not yet finalized when it began. All these other in-progress shard snapshot lists are scheduled - * for deletion now. - */ - @FixForMultiProject - @Deprecated(forRemoval = true) - public Map> obsoleteGenerations( - String repository, - SnapshotsInProgress oldClusterStateSnapshots - ) { - return obsoleteGenerations(Metadata.DEFAULT_PROJECT_ID, repository, oldClusterStateSnapshots); - } - /** * Computes a map of repository shard id to set of shard generations, containing all shard generations that became obsolete and may be * deleted from the repository as the cluster state moves from the given old value of {@link SnapshotsInProgress} to this instance. diff --git a/server/src/main/java/org/elasticsearch/repositories/FinalizeSnapshotContext.java b/server/src/main/java/org/elasticsearch/repositories/FinalizeSnapshotContext.java index b508238406373..d643448b691f0 100644 --- a/server/src/main/java/org/elasticsearch/repositories/FinalizeSnapshotContext.java +++ b/server/src/main/java/org/elasticsearch/repositories/FinalizeSnapshotContext.java @@ -107,7 +107,8 @@ public ClusterState updatedClusterState(ClusterState state) { // Now that the updated cluster state may have changed in-progress shard snapshots' shard generations to the latest shard // generation, let's mark any now unreferenced shard generations as obsolete and ready to be deleted. obsoleteGenerations.set( - SnapshotsInProgress.get(updatedState).obsoleteGenerations(snapshotInfo.repository(), SnapshotsInProgress.get(state)) + SnapshotsInProgress.get(updatedState) + .obsoleteGenerations(snapshotInfo.projectId(), snapshotInfo.repository(), SnapshotsInProgress.get(state)) ); return updatedState; } diff --git a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java index 996c9cda4deab..cb3fa6f575c8c 100644 --- a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java +++ b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java @@ -710,7 +710,7 @@ public boolean canUpdateInPlace(Settings updatedSettings, Set ignoredSet @Override public void updateState(ClusterState state) { final Settings previousSettings = metadata.settings(); - metadata = getRepoMetadata(state); + metadata = getRepoMetadata(state.metadata().getProject(getProjectId())); final Settings updatedSettings = metadata.settings(); if (updatedSettings.equals(previousSettings) == false) { snapshotRateLimiter = getSnapshotRateLimiter(); @@ -725,7 +725,7 @@ public void updateState(ClusterState state) { return; } if (bestEffortConsistency) { - long bestGenerationFromCS = bestGeneration(SnapshotsInProgress.get(state).forRepo(this.metadata.name())); + long bestGenerationFromCS = bestGeneration(SnapshotsInProgress.get(state).forRepo(getProjectId(), this.metadata.name())); // Don't use generation from the delete task if we already found a generation for an in progress snapshot. // In this case, the generation points at the generation the repo will be in after the snapshot finishes so it may not yet // exist @@ -1823,7 +1823,7 @@ record RootBlobUpdateResult(RepositoryData oldRepositoryData, RepositoryData new // Write the index metadata for each index in the snapshot for (IndexId index : indices) { executor.execute(ActionRunnable.run(allMetaListeners.acquire(), () -> { - final IndexMetadata indexMetaData = clusterMetadata.getProject().index(index.getName()); + final IndexMetadata indexMetaData = clusterMetadata.getProject(getProjectId()).index(index.getName()); if (writeIndexGens) { final String identifiers = IndexMetaDataGenerations.buildUniqueIdentifier(indexMetaData); String metaUUID = existingRepositoryData.indexMetaDataGenerations().getIndexMetaBlobId(identifiers); @@ -1836,7 +1836,7 @@ record RootBlobUpdateResult(RepositoryData oldRepositoryData, RepositoryData new metadataWriteResult.indexMetas().put(index, identifiers); } else { INDEX_METADATA_FORMAT.write( - clusterMetadata.getProject().index(index.getName()), + clusterMetadata.getProject(getProjectId()).index(index.getName()), indexContainer(index), snapshotId.getUUID(), compress @@ -2403,11 +2403,11 @@ public void clusterStateProcessed(ClusterState oldState, ClusterState newState) private ClusterState getClusterStateWithUpdatedRepositoryGeneration(ClusterState currentState, RepositoryData repoData) { // In theory we might have failed over to a different master which initialized the repo and then failed back to this node, so we // must check the repository generation in the cluster state is still unknown here. - final RepositoryMetadata repoMetadata = getRepoMetadata(currentState); + final var project = currentState.metadata().getProject(getProjectId()); + final RepositoryMetadata repoMetadata = getRepoMetadata(project); if (repoMetadata.generation() != RepositoryData.UNKNOWN_REPO_GEN) { throw new RepositoryException(repoMetadata.name(), "Found unexpected initialized repo metadata [" + repoMetadata + "]"); } - final var project = currentState.metadata().getProject(getProjectId()); return ClusterState.builder(currentState) .putProjectMetadata( ProjectMetadata.builder(project) @@ -2594,7 +2594,7 @@ private void markRepoCorrupted(long corruptedGeneration, Exception originalExcep new ClusterStateUpdateTask() { @Override public ClusterState execute(ClusterState currentState) { - final var project = currentState.metadata().getDefaultProject(); + final var project = currentState.metadata().getProject(projectId); final RepositoriesMetadata state = RepositoriesMetadata.get(project); final RepositoryMetadata repoState = state.repository(metadata.name()); if (repoState.generation() != corruptedGeneration) { @@ -2748,7 +2748,8 @@ protected void writeIndexGen( @Override public ClusterState execute(ClusterState currentState) { - final RepositoryMetadata meta = getRepoMetadata(currentState); + final var project = currentState.metadata().getProject(projectId); + final RepositoryMetadata meta = getRepoMetadata(project); final String repoName = metadata.name(); if (RepositoriesService.isReadOnly(meta.settings())) { @@ -2788,7 +2789,6 @@ public ClusterState execute(ClusterState currentState) { + "] must be larger than latest known generation [" + latestKnownRepoGen.get() + "]"; - final var project = currentState.metadata().getDefaultProject(); return ClusterState.builder(currentState) .putProjectMetadata( ProjectMetadata.builder(project) @@ -2914,7 +2914,8 @@ public void onFailure(Exception e) { submitUnbatchedTask(setSafeGenerationSource, new ClusterStateUpdateTask() { @Override public ClusterState execute(ClusterState currentState) { - final RepositoryMetadata meta = getRepoMetadata(currentState); + final var project = currentState.metadata().getProject(projectId); + final RepositoryMetadata meta = getRepoMetadata(project); if (meta.generation() != expectedGen) { throw new IllegalStateException( "Tried to update repo generation to [" + newGen + "] but saw unexpected generation in state [" + meta + "]" @@ -2929,7 +2930,6 @@ public ClusterState execute(ClusterState currentState) { + "]" ); } - final var project = currentState.metadata().getDefaultProject(); final RepositoriesMetadata withGenerations = RepositoriesMetadata.get(project) .withUpdatedGeneration(metadata.name(), newGen, newGen); final RepositoriesMetadata withUuid = meta.uuid().equals(newRepositoryData.getUuid()) @@ -3089,7 +3089,7 @@ private ClusterState updateRepositoryGenerationsIfNecessary(ClusterState state, boolean changedSnapshots = false; final List snapshotEntries = new ArrayList<>(); final SnapshotsInProgress snapshotsInProgress = SnapshotsInProgress.get(state); - for (SnapshotsInProgress.Entry entry : snapshotsInProgress.forRepo(repoName)) { + for (SnapshotsInProgress.Entry entry : snapshotsInProgress.forRepo(getProjectId(), repoName)) { if (entry.repositoryStateId() == oldGen) { snapshotEntries.add(entry.withRepoGen(newGen)); changedSnapshots = true; @@ -3098,7 +3098,7 @@ private ClusterState updateRepositoryGenerationsIfNecessary(ClusterState state, } } updatedSnapshotsInProgress = changedSnapshots - ? snapshotsInProgress.createCopyWithUpdatedEntriesForRepo(repoName, snapshotEntries) + ? snapshotsInProgress.createCopyWithUpdatedEntriesForRepo(getProjectId(), repoName, snapshotEntries) : null; final SnapshotDeletionsInProgress updatedDeletionsInProgress; boolean changedDeletions = false; @@ -3115,9 +3115,8 @@ private ClusterState updateRepositoryGenerationsIfNecessary(ClusterState state, return SnapshotsService.updateWithSnapshots(state, updatedSnapshotsInProgress, updatedDeletionsInProgress); } - private RepositoryMetadata getRepoMetadata(ClusterState state) { - final RepositoryMetadata repositoryMetadata = RepositoriesMetadata.get(state.getMetadata().getProject(getProjectId())) - .repository(metadata.name()); + private RepositoryMetadata getRepoMetadata(ProjectMetadata projectMetadata) { + final RepositoryMetadata repositoryMetadata = RepositoriesMetadata.get(projectMetadata).repository(metadata.name()); assert repositoryMetadata != null || lifecycle.stoppedOrClosed() : "did not find metadata for repo [" + metadata.name() + "] in state [" + lifecycleState() + "]"; return repositoryMetadata; diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java index 2cedf4506c7c6..2bc75665da2a5 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java @@ -12,6 +12,7 @@ import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsRequest; import org.elasticsearch.cluster.SnapshotsInProgress; import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; @@ -355,6 +356,10 @@ public SnapshotId snapshotId() { return snapshot.getSnapshotId(); } + public ProjectId projectId() { + return snapshot.getProjectId(); + } + public String repository() { return snapshot.getRepository(); } diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java index 8a1f1ad79a17f..396287af20208 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java @@ -97,6 +97,7 @@ import org.elasticsearch.repositories.RepositoryData; import org.elasticsearch.repositories.RepositoryException; import org.elasticsearch.repositories.RepositoryMissingException; +import org.elasticsearch.repositories.RepositoryOperation.ProjectRepo; import org.elasticsearch.repositories.RepositoryShardId; import org.elasticsearch.repositories.ShardGeneration; import org.elasticsearch.repositories.ShardGenerations; @@ -181,7 +182,7 @@ public final class SnapshotsService extends AbstractLifecycleComponent implement private final Map>> snapshotDeletionListeners = new ConcurrentHashMap<>(); // Set of repositories currently running either a snapshot finalization or a snapshot delete. - private final Set currentlyFinalizing = Collections.synchronizedSet(new HashSet<>()); + private final Set currentlyFinalizing = Collections.synchronizedSet(new HashSet<>()); // Set of snapshots that are currently being ended by this node private final Set endingSnapshots = Collections.synchronizedSet(new HashSet<>()); @@ -255,14 +256,19 @@ public SnapshotsService( } /** - * Same as {@link #createSnapshot(CreateSnapshotRequest, ActionListener)} but invokes its callback on completion of + * Same as {@link #createSnapshot(ProjectId, CreateSnapshotRequest, ActionListener)} but invokes its callback on completion of * the snapshot. * + * @param projectId the project for creating the snapshot * @param request snapshot request * @param listener snapshot completion listener */ - public void executeSnapshot(final CreateSnapshotRequest request, final ActionListener listener) { - createSnapshot(request, listener.delegateFailureAndWrap((l, snapshot) -> addListener(snapshot, l))); + public void executeSnapshot( + final ProjectId projectId, + final CreateSnapshotRequest request, + final ActionListener listener + ) { + createSnapshot(projectId, request, listener.delegateFailureAndWrap((l, snapshot) -> addListener(snapshot, l))); } /** @@ -271,20 +277,27 @@ public void executeSnapshot(final CreateSnapshotRequest request, final ActionLis * This method is used by clients to start snapshot. It makes sure that there is no snapshots are currently running and * creates a snapshot record in cluster state metadata. * + * @param projectId the project for creating the snapshot * @param request snapshot request * @param listener snapshot creation listener */ - public void createSnapshot(final CreateSnapshotRequest request, final ActionListener listener) { + public void createSnapshot(final ProjectId projectId, final CreateSnapshotRequest request, final ActionListener listener) { final String repositoryName = request.repository(); final String snapshotName = IndexNameExpressionResolver.resolveDateMathExpression(request.snapshot()); validate(repositoryName, snapshotName); final SnapshotId snapshotId = new SnapshotId(snapshotName, request.uuid()); - Repository repository = repositoriesService.repository(request.repository()); + Repository repository = repositoriesService.repository(projectId, request.repository()); if (repository.isReadOnly()) { listener.onFailure(new RepositoryException(repository.getMetadata().name(), "cannot create snapshot in a readonly repository")); return; } - submitCreateSnapshotRequest(request, listener, repository, new Snapshot(repositoryName, snapshotId), repository.getMetadata()); + submitCreateSnapshotRequest( + request, + listener, + repository, + new Snapshot(projectId, repositoryName, snapshotId), + repository.getMetadata() + ); } /** @@ -309,17 +322,24 @@ private void submitCreateSnapshotRequest( ); } - private static void ensureSnapshotNameNotRunning(SnapshotsInProgress runningSnapshots, String repositoryName, String snapshotName) { - if (runningSnapshots.forRepo(repositoryName).stream().anyMatch(s -> s.snapshot().getSnapshotId().getName().equals(snapshotName))) { + private static void ensureSnapshotNameNotRunning( + SnapshotsInProgress runningSnapshots, + ProjectId projectId, + String repositoryName, + String snapshotName + ) { + if (runningSnapshots.forRepo(projectId, repositoryName) + .stream() + .anyMatch(s -> s.snapshot().getSnapshotId().getName().equals(snapshotName))) { throw new SnapshotNameAlreadyInUseException(repositoryName, snapshotName, "snapshot with the same name is already in-progress"); } } // TODO: It is worth revisiting the design choice of creating a placeholder entry in snapshots-in-progress here once we have a cache // for repository metadata and loading it has predictable performance - public void cloneSnapshot(CloneSnapshotRequest request, ActionListener listener) { + public void cloneSnapshot(ProjectId projectId, CloneSnapshotRequest request, ActionListener listener) { final String repositoryName = request.repository(); - Repository repository = repositoriesService.repository(repositoryName); + Repository repository = repositoriesService.repository(projectId, repositoryName); if (repository.isReadOnly()) { listener.onFailure(new RepositoryException(repositoryName, "cannot create snapshot in a readonly repository")); return; @@ -329,7 +349,7 @@ public void cloneSnapshot(CloneSnapshotRequest request, ActionListener lis // TODO: create snapshot UUID in CloneSnapshotRequest and make this operation idempotent to cleanly deal with transport layer // retries final SnapshotId snapshotId = new SnapshotId(snapshotName, UUIDs.randomBase64UUID()); - final Snapshot snapshot = new Snapshot(repositoryName, snapshotId); + final Snapshot snapshot = new Snapshot(projectId, repositoryName, snapshotId); initializingClones.add(snapshot); executeConsistentStateUpdate(repository, repositoryData -> new ClusterStateUpdateTask(request.masterNodeTimeout()) { @@ -337,13 +357,14 @@ public void cloneSnapshot(CloneSnapshotRequest request, ActionListener lis @Override public ClusterState execute(ClusterState currentState) { - ensureRepositoryExists(repositoryName, currentState); + final var projectMetadata = currentState.metadata().getProject(projectId); + ensureRepositoryExists(repositoryName, projectMetadata); ensureSnapshotNameAvailableInRepo(repositoryData, snapshotName, repository); ensureNoCleanupInProgress(currentState, repositoryName, snapshotName, "clone snapshot"); - ensureNotReadOnly(currentState, repositoryName); + ensureNotReadOnly(projectMetadata, repositoryName); final SnapshotsInProgress snapshots = SnapshotsInProgress.get(currentState); - ensureSnapshotNameNotRunning(snapshots, repositoryName, snapshotName); - validate(repositoryName, snapshotName, currentState); + ensureSnapshotNameNotRunning(snapshots, projectId, repositoryName, snapshotName); + validate(repositoryName, snapshotName, projectMetadata); final SnapshotId sourceSnapshotId = repositoryData.getSnapshotIds() .stream() @@ -372,7 +393,7 @@ public ClusterState execute(ClusterState currentState) { ); if (matchingIndices.isEmpty()) { throw new SnapshotException( - new Snapshot(repositoryName, sourceSnapshotId), + new Snapshot(projectId, repositoryName, sourceSnapshotId), "No indices in the source snapshot [" + sourceSnapshotId + "] matched requested pattern [" @@ -413,6 +434,7 @@ public void clusterStateProcessed(ClusterState oldState, final ClusterState newS /** * Checks the cluster state for any in-progress repository cleanup tasks ({@link RepositoryCleanupInProgress}). + * Repository cleanup is intentionally cluster wide exclusive */ private static void ensureNoCleanupInProgress( final ClusterState currentState, @@ -436,8 +458,8 @@ private static void ensureNoCleanupInProgress( } } - public static void ensureNotReadOnly(final ClusterState currentState, final String repositoryName) { - final var repositoryMetadata = RepositoriesMetadata.get(currentState).repository(repositoryName); + public static void ensureNotReadOnly(final ProjectMetadata projectMetadata, final String repositoryName) { + final var repositoryMetadata = RepositoriesMetadata.get(projectMetadata).repository(repositoryName); if (RepositoriesService.isReadOnly(repositoryMetadata.settings())) { throw new RepositoryException(repositoryMetadata.name(), "repository is readonly"); } @@ -516,8 +538,9 @@ private void startCloning(Repository repository, SnapshotsInProgress.Entry clone @Override public ClusterState execute(ClusterState currentState) { final SnapshotsInProgress snapshotsInProgress = SnapshotsInProgress.get(currentState); + final ProjectId projectId = cloneEntry.projectId(); final String repoName = cloneEntry.repository(); - final List existingEntries = snapshotsInProgress.forRepo(repoName); + final List existingEntries = snapshotsInProgress.forRepo(projectId, repoName); final List updatedEntries = new ArrayList<>(existingEntries.size()); final String localNodeId = currentState.nodes().getLocalNodeId(); final ShardGenerations shardGenerations = repoData.shardGenerations(); @@ -526,10 +549,12 @@ public ClusterState execute(ClusterState currentState) { final ImmutableOpenMap.Builder clonesBuilder = ImmutableOpenMap .builder(); final boolean readyToExecute = SnapshotDeletionsInProgress.get(currentState) - .hasExecutingDeletion(repoName) == false; + .hasExecutingDeletion(projectId, repoName) == false; final InFlightShardSnapshotStates inFlightShardStates; if (readyToExecute) { - inFlightShardStates = InFlightShardSnapshotStates.forEntries(snapshotsInProgress.forRepo(repoName)); + inFlightShardStates = InFlightShardSnapshotStates.forEntries( + snapshotsInProgress.forRepo(projectId, repoName) + ); } else { // no need to compute these, we'll mark all shards as queued anyway because we wait for the delete inFlightShardStates = null; @@ -563,7 +588,7 @@ public ClusterState execute(ClusterState currentState) { updatedEntries.add(updatedEntry); return updateWithSnapshots( currentState, - snapshotsInProgress.createCopyWithUpdatedEntriesForRepo(repoName, updatedEntries), + snapshotsInProgress.createCopyWithUpdatedEntriesForRepo(projectId, repoName, updatedEntries), null ); } @@ -607,6 +632,7 @@ public String toString() { ); } + // RepositoryShardId is unique across projects because RepositoryShardId.index().getId() is an UUID private final Set currentlyCloning = Collections.synchronizedSet(new HashSet<>()); private void runReadyClone( @@ -677,6 +703,7 @@ private void runReadyClone( } } + // This is limit is cluster wide across all projects private void ensureBelowConcurrencyLimit( String repository, String name, @@ -702,8 +729,8 @@ private void ensureBelowConcurrencyLimit( /** * Throws {@link RepositoryMissingException} if no repository by the given name is found in the given cluster state. */ - public static void ensureRepositoryExists(String repoName, ClusterState state) { - if (RepositoriesMetadata.get(state).repository(repoName) == null) { + public static void ensureRepositoryExists(String repoName, ProjectMetadata projectMetadata) { + if (RepositoriesMetadata.get(projectMetadata).repository(repoName) == null) { throw new RepositoryMissingException(repoName); } } @@ -713,10 +740,10 @@ public static void ensureRepositoryExists(String repoName, ClusterState state) { * * @param repositoryName repository name * @param snapshotName snapshot name - * @param state current cluster state + * @param projectMetadata current project metadata */ - private static void validate(String repositoryName, String snapshotName, ClusterState state) { - if (RepositoriesMetadata.get(state).repository(repositoryName) == null) { + private static void validate(String repositoryName, String snapshotName, ProjectMetadata projectMetadata) { + if (RepositoriesMetadata.get(projectMetadata).repository(repositoryName) == null) { throw new RepositoryMissingException(repositoryName); } validate(repositoryName, snapshotName); @@ -774,9 +801,8 @@ private static UpdatedShardGenerations buildGenerations(SnapshotsInProgress.Entr return new UpdatedShardGenerations(builder.build(), deletedBuilder == null ? ShardGenerations.EMPTY : deletedBuilder.build()); } - private static Metadata metadataForSnapshot(SnapshotsInProgress.Entry snapshot, Metadata metadata) { - @FixForMultiProject - final ProjectMetadata snapshotProject = projectForSnapshot(snapshot, metadata.getProject()); + private static Metadata metadataForSnapshot(SnapshotsInProgress.Entry snapshot, Metadata metadata, ProjectId projectId) { + final ProjectMetadata snapshotProject = projectForSnapshot(snapshot, metadata.getProject(projectId)); final Metadata.Builder builder; if (snapshot.includeGlobalState() == false) { // Remove global state from the cluster state @@ -830,12 +856,14 @@ private static ProjectMetadata projectForSnapshot(SnapshotsInProgress.Entry snap *

* * @param snapshotsInProgress snapshots in progress in the cluster state + * @param projectId project to look for the repository * @param repository repository id * @param snapshots list of snapshots that will be used as a filter, empty list means no snapshots are filtered * @return list of metadata for currently running snapshots */ public static List currentSnapshots( @Nullable SnapshotsInProgress snapshotsInProgress, + ProjectId projectId, String repository, List snapshots ) { @@ -846,10 +874,10 @@ public static List currentSnapshots( return snapshotsInProgress.asStream().toList(); } if (snapshots.isEmpty()) { - return snapshotsInProgress.forRepo(repository); + return snapshotsInProgress.forRepo(projectId, repository); } List builder = new ArrayList<>(); - for (SnapshotsInProgress.Entry entry : snapshotsInProgress.forRepo(repository)) { + for (SnapshotsInProgress.Entry entry : snapshotsInProgress.forRepo(projectId, repository)) { for (String snapshot : snapshots) { if (entry.snapshot().getSnapshotId().getName().equals(snapshot)) { builder.add(entry); @@ -1007,7 +1035,6 @@ private void processExternalChanges(boolean changedNodes, boolean changedShards) @Override public ClusterState execute(ClusterState currentState) { - RoutingTable routingTable = currentState.routingTable(); final SnapshotsInProgress snapshotsInProgress = SnapshotsInProgress.get(currentState); final SnapshotDeletionsInProgress deletesInProgress = SnapshotDeletionsInProgress.get(currentState); DiscoveryNodes nodes = currentState.nodes(); @@ -1035,6 +1062,7 @@ public ClusterState execute(ClusterState currentState) { boolean changed = false; final List updatedEntriesForRepo = new ArrayList<>(); final Map knownFailures = new HashMap<>(); + final ProjectId projectId = snapshotsInRepo.get(0).projectId(); final String repositoryName = snapshotsInRepo.get(0).repository(); for (SnapshotsInProgress.Entry snapshotEntry : snapshotsInRepo) { if (statesToUpdate.contains(snapshotEntry.state())) { @@ -1049,7 +1077,7 @@ public ClusterState execute(ClusterState currentState) { } } else { // see if any clones may have had a shard become available for execution because of failures - if (deletesInProgress.hasExecutingDeletion(repositoryName)) { + if (deletesInProgress.hasExecutingDeletion(projectId, repositoryName)) { // Currently executing a delete for this repo, no need to try and update any clone operations. // The logic for finishing the delete will update running clones with the latest changes. updatedEntriesForRepo.add(snapshotEntry); @@ -1099,7 +1127,7 @@ public ClusterState execute(ClusterState currentState) { ImmutableOpenMap shards = processWaitingShardsAndRemovedNodes( snapshotEntry, - routingTable, + currentState.routingTable(projectId), nodes, snapshotsInProgress::isNodeIdForRemoval, knownFailures @@ -1131,7 +1159,11 @@ public ClusterState execute(ClusterState currentState) { } } if (changed) { - updatedSnapshots = updatedSnapshots.createCopyWithUpdatedEntriesForRepo(repositoryName, updatedEntriesForRepo); + updatedSnapshots = updatedSnapshots.createCopyWithUpdatedEntriesForRepo( + projectId, + repositoryName, + updatedEntriesForRepo + ); } } final ClusterState res = readyDeletions( @@ -1176,7 +1208,7 @@ public void clusterStateProcessed(ClusterState oldState, ClusterState newState) startExecutableClones(SnapshotsInProgress.get(newState), null); // run newly ready deletes for (SnapshotDeletionsInProgress.Entry entry : deletionsToExecute) { - if (tryEnterRepoLoop(entry.repository())) { + if (tryEnterRepoLoop(entry.projectId(), entry.repository())) { deleteSnapshotsFromRepository(entry, newState.nodes().getMaxDataNodeCompatibleIndexVersion()); } } @@ -1306,6 +1338,7 @@ private static boolean waitingShardsStartedOrUnassigned(SnapshotsInProgress snap for (List entries : snapshotsInProgress.entriesByRepo()) { for (SnapshotsInProgress.Entry entry : entries) { if (entry.state() == SnapshotsInProgress.State.STARTED && entry.isClone() == false) { + final ProjectId projectId = entry.projectId(); for (Map.Entry shardStatus : entry.shardSnapshotStatusByRepoShardId() .entrySet()) { final ShardState state = shardStatus.getValue().state(); @@ -1315,7 +1348,7 @@ private static boolean waitingShardsStartedOrUnassigned(SnapshotsInProgress snap final RepositoryShardId shardId = shardStatus.getKey(); final Index index = entry.indexByName(shardId.indexName()); if (event.indexRoutingTableChanged(index)) { - IndexRoutingTable indexShardRoutingTable = event.state().getRoutingTable().index(index); + IndexRoutingTable indexShardRoutingTable = event.state().routingTable(projectId).index(index); if (indexShardRoutingTable == null) { // index got removed concurrently and we have to fail WAITING, QUEUED and PAUSED_FOR_REMOVAL state shards return true; @@ -1374,10 +1407,11 @@ private void endSnapshot(SnapshotsInProgress.Entry entry, Metadata metadata, @Nu } return; } + final ProjectId projectId = snapshot.getProjectId(); final String repoName = snapshot.getRepository(); - if (tryEnterRepoLoop(repoName)) { + if (tryEnterRepoLoop(projectId, repoName)) { if (repositoryData == null) { - repositoriesService.repository(snapshot.getProjectId(), repoName) + repositoriesService.repository(projectId, repoName) .getRepositoryData( EsExecutors.DIRECT_EXECUTOR_SERVICE, // TODO contemplate threading here, do we need to fork, see #101445? new ActionListener<>() { @@ -1386,13 +1420,16 @@ public void onResponse(RepositoryData repositoryData) { if (newFinalization) { finalizeSnapshotEntry(snapshot, metadata, repositoryData); } else { - runNextQueuedOperation(repositoryData, repoName, false); + runNextQueuedOperation(repositoryData, projectId, repoName, false); } } @Override public void onFailure(Exception e) { - submitUnbatchedTask("fail repo tasks for [" + repoName + "]", new FailPendingRepoTasksTask(repoName, e)); + submitUnbatchedTask( + "fail repo tasks for [" + repoName + "]", + new FailPendingRepoTasksTask(projectId, repoName, e) + ); } } ); @@ -1400,7 +1437,7 @@ public void onFailure(Exception e) { if (newFinalization) { finalizeSnapshotEntry(snapshot, metadata, repositoryData); } else { - runNextQueuedOperation(repositoryData, repoName, false); + runNextQueuedOperation(repositoryData, projectId, repoName, false); } } } else { @@ -1413,21 +1450,21 @@ public void onFailure(Exception e) { /** * Try starting to run a snapshot finalization or snapshot delete for the given repository. If this method returns * {@code true} then snapshot finalizations and deletions for the repo may be executed. Once no more operations are - * ready for the repository {@link #leaveRepoLoop(String)} should be invoked so that a subsequent state change that + * ready for the repository {@link #leaveRepoLoop(ProjectId, String)} should be invoked so that a subsequent state change that * causes another operation to become ready can execute. * * @return true if a finalization or snapshot delete may be started at this point */ - private boolean tryEnterRepoLoop(String repository) { - return currentlyFinalizing.add(repository); + private boolean tryEnterRepoLoop(ProjectId projectId, String repository) { + return currentlyFinalizing.add(new ProjectRepo(projectId, repository)); } /** * Stop polling for ready snapshot finalizations or deletes in state {@link SnapshotDeletionsInProgress.State#STARTED} to execute * for the given repository. */ - private void leaveRepoLoop(String repository) { - final boolean removed = currentlyFinalizing.remove(repository); + private void leaveRepoLoop(ProjectId projectId, String repository) { + final boolean removed = currentlyFinalizing.remove(new ProjectRepo(projectId, repository)); assert removed; } @@ -1445,6 +1482,7 @@ private void finalizeSnapshotEntry(Snapshot snapshot, Metadata metadata, Reposit private class SnapshotFinalization extends AbstractRunnable { private final Snapshot snapshot; + @FixForMultiProject(description = "replace with ProjectMetadata") private final Metadata metadata; private final RepositoryData repositoryData; @@ -1457,7 +1495,7 @@ private class SnapshotFinalization extends AbstractRunnable { @Override protected void doRun() { assert ThreadPool.assertCurrentThreadPool(ThreadPool.Names.SNAPSHOT); - assert currentlyFinalizing.contains(snapshot.getRepository()); + assert currentlyFinalizing.contains(new ProjectRepo(snapshot.getProjectId(), snapshot.getRepository())); assert repositoryOperations.assertNotQueued(snapshot); SnapshotsInProgress.Entry entry = SnapshotsInProgress.get(clusterService.state()).snapshot(snapshot); @@ -1484,14 +1522,14 @@ protected void doRun() { assert state == ShardState.SUCCESS; } } + final ProjectId projectId = snapshot.getProjectId(); final String repository = snapshot.getRepository(); final ListenableFuture metadataListener = new ListenableFuture<>(); - final Repository repo = repositoriesService.repository(snapshot.getProjectId(), snapshot.getRepository()); + final Repository repo = repositoriesService.repository(projectId, repository); if (entry.isClone()) { threadPool.executor(ThreadPool.Names.SNAPSHOT).execute(ActionRunnable.supply(metadataListener, () -> { final Metadata existingMetadata = repo.getSnapshotGlobalMetadata(entry.source()); - @FixForMultiProject - final ProjectMetadata existingProject = existingMetadata.getProject(); + final ProjectMetadata existingProject = existingMetadata.getProject(projectId); final ProjectMetadata.Builder projBuilder = ProjectMetadata.builder(existingProject); final Set existingIndices = new HashSet<>(); for (IndexId index : entry.indices().values()) { @@ -1518,7 +1556,7 @@ protected void doRun() { } metadataListener.addListener(ActionListener.wrap(meta -> { assert ThreadPool.assertCurrentThreadPool(ThreadPool.Names.SNAPSHOT); - final Metadata metaForSnapshot = metadataForSnapshot(entry, meta); + final Metadata metaForSnapshot = metadataForSnapshot(entry, meta, projectId); final Map indexSnapshotDetails = Maps.newMapWithExpectedSize( finalIndices.size() @@ -1558,7 +1596,7 @@ protected void doRun() { final SnapshotInfo snapshotInfo = new SnapshotInfo( snapshot, finalIndices, - entry.dataStreams().stream().filter(metaForSnapshot.getProject().dataStreams()::containsKey).toList(), + entry.dataStreams().stream().filter(metaForSnapshot.getProject(projectId).dataStreams()::containsKey).toList(), entry.partial() ? onlySuccessfulFeatureStates(entry, finalIndices) : entry.featureStates(), failure, threadPool.absoluteTimeInMillis(), @@ -1582,7 +1620,7 @@ protected void doRun() { // by the snapshot info callback below and won't be failed needlessly if #runNextQueuedOperation runs into // a fatal like e.g. this node stopped being the master node snapshotListeners.onResponse(endAndGetListenersToResolve(snapshot)); - runNextQueuedOperation(updatedRepositoryData, repository, true); + runNextQueuedOperation(updatedRepositoryData, projectId, repository, true); }, e -> handleFinalizationFailure( e, @@ -1711,18 +1749,19 @@ private void handleFinalizationFailure( * Run the next queued up repository operation for the given repository name. * * @param repositoryData current repository data + * @param projectId project for the repository * @param repository repository name * @param attemptDelete whether to try and run delete operations that are ready in the cluster state if no * snapshot create operations remain to execute */ - private void runNextQueuedOperation(RepositoryData repositoryData, String repository, boolean attemptDelete) { - assert currentlyFinalizing.contains(repository); - final Tuple nextFinalization = repositoryOperations.pollFinalization(repository); + private void runNextQueuedOperation(RepositoryData repositoryData, ProjectId projectId, String repository, boolean attemptDelete) { + assert currentlyFinalizing.contains(new ProjectRepo(projectId, repository)); + final Tuple nextFinalization = repositoryOperations.pollFinalization(projectId, repository); if (nextFinalization == null) { if (attemptDelete) { - runReadyDeletions(repositoryData, repository); + runReadyDeletions(repositoryData, projectId, repository); } else { - leaveRepoLoop(repository); + leaveRepoLoop(projectId, repository); } } else { logger.trace("Moving on to finalizing next snapshot [{}]", nextFinalization); @@ -1735,7 +1774,7 @@ private void runNextQueuedOperation(RepositoryData repositoryData, String reposi * * TODO: optimize this to execute in a single CS update together with finalizing the latest snapshot */ - private void runReadyDeletions(RepositoryData repositoryData, String repository) { + private void runReadyDeletions(RepositoryData repositoryData, ProjectId projectId, String repository) { submitUnbatchedTask("Run ready deletions", new ClusterStateUpdateTask() { private SnapshotDeletionsInProgress.Entry deletionToRun; @@ -1745,7 +1784,9 @@ public ClusterState execute(ClusterState currentState) { assert readyDeletions(currentState).v1() == currentState : "Deletes should have been set to ready by finished snapshot deletes and finalizations"; for (SnapshotDeletionsInProgress.Entry entry : SnapshotDeletionsInProgress.get(currentState).getEntries()) { - if (entry.repository().equals(repository) && entry.state() == SnapshotDeletionsInProgress.State.STARTED) { + if (entry.projectId().equals(projectId) + && entry.repository().equals(repository) + && entry.state() == SnapshotDeletionsInProgress.State.STARTED) { deletionToRun = entry; break; } @@ -1762,7 +1803,7 @@ public void onFailure(Exception e) { @Override public void clusterStateProcessed(ClusterState oldState, ClusterState newState) { if (deletionToRun == null) { - runNextQueuedOperation(repositoryData, repository, false); + runNextQueuedOperation(repositoryData, projectId, repository, false); } else { deleteSnapshotsFromRepository(deletionToRun, repositoryData, newState.nodes().getMaxDataNodeCompatibleIndexVersion()); } @@ -1787,15 +1828,15 @@ private static Tuple> read } final SnapshotsInProgress snapshotsInProgress = currentState.custom(SnapshotsInProgress.TYPE); assert snapshotsInProgress != null; - final Set repositoriesSeen = new HashSet<>(); + final Set repositoriesSeen = new HashSet<>(); boolean changed = false; final ArrayList readyDeletions = new ArrayList<>(); final List newDeletes = new ArrayList<>(); for (SnapshotDeletionsInProgress.Entry entry : deletions.getEntries()) { - final String repo = entry.repository(); - if (repositoriesSeen.add(entry.repository()) + final var projectRepo = new ProjectRepo(entry.projectId(), entry.repository()); + if (repositoriesSeen.add(projectRepo) && entry.state() == SnapshotDeletionsInProgress.State.WAITING - && snapshotsInProgress.forRepo(repo).stream().noneMatch(SnapshotsService::isWritingToRepository)) { + && snapshotsInProgress.forRepo(projectRepo).stream().noneMatch(SnapshotsService::isWritingToRepository)) { changed = true; final SnapshotDeletionsInProgress.Entry newEntry = entry.started(); readyDeletions.add(newEntry); @@ -1836,7 +1877,7 @@ public static ClusterState stateWithoutSnapshot( ClusterState result = state; int indexOfEntry = -1; // Find the in-progress snapshot entry that matches {@code snapshot}. - final List entryList = inProgressSnapshots.forRepo(snapshot.getRepository()); + final List entryList = inProgressSnapshots.forRepo(snapshot.getProjectId(), snapshot.getRepository()); for (int i = 0; i < entryList.size(); i++) { SnapshotsInProgress.Entry entry = entryList.get(i); if (entry.snapshot().equals(snapshot)) { @@ -1935,7 +1976,11 @@ public static ClusterState stateWithoutSnapshot( result = ClusterState.builder(state) .putCustom( SnapshotsInProgress.TYPE, - inProgressSnapshots.createCopyWithUpdatedEntriesForRepo(snapshot.getRepository(), updatedEntries) + inProgressSnapshots.createCopyWithUpdatedEntriesForRepo( + snapshot.getProjectId(), + snapshot.getRepository(), + updatedEntries + ) ) .build(); } @@ -2023,6 +2068,7 @@ public ClusterState execute(ClusterState currentState) { deletionsWithoutSnapshots( SnapshotDeletionsInProgress.get(updatedState), Collections.singletonList(snapshot.getSnapshotId()), + snapshot.getProjectId(), snapshot.getRepository() ) ); @@ -2050,7 +2096,7 @@ public void onFailure(Exception e) { public void clusterStateProcessed(ClusterState oldState, ClusterState newState) { failSnapshotCompletionListeners(snapshot, failure, Runnable::run); if (repositoryData != null) { - runNextQueuedOperation(repositoryData, snapshot.getRepository(), true); + runNextQueuedOperation(repositoryData, snapshot.getProjectId(), snapshot.getRepository(), true); } } }); @@ -2064,6 +2110,7 @@ public void clusterStateProcessed(ClusterState oldState, ClusterState newState) * * @param deletions snapshot deletions to update * @param snapshotIds snapshot ids to remove + * @param projectId project for the repository * @param repository repository that the snapshot ids belong to * @return updated {@link SnapshotDeletionsInProgress} or {@code null} if unchanged */ @@ -2071,12 +2118,13 @@ public void clusterStateProcessed(ClusterState oldState, ClusterState newState) private static SnapshotDeletionsInProgress deletionsWithoutSnapshots( SnapshotDeletionsInProgress deletions, Collection snapshotIds, + ProjectId projectId, String repository ) { boolean changed = false; List updatedEntries = new ArrayList<>(deletions.getEntries().size()); for (SnapshotDeletionsInProgress.Entry entry : deletions.getEntries()) { - if (entry.repository().equals(repository)) { + if (entry.projectId().equals(projectId) && entry.repository().equals(repository)) { final List updatedSnapshotIds = new ArrayList<>(entry.snapshots()); if (updatedSnapshotIds.removeAll(snapshotIds)) { changed = true; @@ -2103,14 +2151,15 @@ private void failSnapshotCompletionListeners(Snapshot snapshot, Exception e, Con * When wait_for_completion is set to true, the passed action listener will only complete when all * matching snapshots are deleted, when it is false it will complete as soon as the deletes are scheduled * + * @param projectId project to look for the snapshot * @param request delete snapshot request * @param listener listener a listener which will be resolved according to the wait_for_completion parameter */ - public void deleteSnapshots(final DeleteSnapshotRequest request, final ActionListener listener) { + public void deleteSnapshots(final ProjectId projectId, final DeleteSnapshotRequest request, final ActionListener listener) { final String repositoryName = request.repository(); final String[] snapshotNames = request.snapshots(); - final Repository repository = repositoriesService.repository(repositoryName); + final Repository repository = repositoriesService.repository(projectId, repositoryName); executeConsistentStateUpdate(repository, repositoryData -> new ClusterStateUpdateTask(request.masterNodeTimeout()) { private SnapshotDeletionsInProgress.Entry newDelete = null; @@ -2127,12 +2176,13 @@ public void deleteSnapshots(final DeleteSnapshotRequest request, final ActionLis @Override public ClusterState execute(ClusterState currentState) { - ensureRepositoryExists(repositoryName, currentState); + final var projectMetadata = currentState.metadata().getProject(projectId); + ensureRepositoryExists(repositoryName, projectMetadata); final Set snapshotIds = new HashSet<>(); // find in-progress snapshots to delete in cluster state final SnapshotsInProgress snapshotsInProgress = SnapshotsInProgress.get(currentState); - for (SnapshotsInProgress.Entry entry : snapshotsInProgress.forRepo(repositoryName)) { + for (SnapshotsInProgress.Entry entry : snapshotsInProgress.forRepo(projectId, repositoryName)) { final SnapshotId snapshotId = entry.snapshot().getSnapshotId(); if (Regex.simpleMatch(snapshotNames, snapshotId.getName())) { snapshotIds.add(snapshotId); @@ -2170,12 +2220,13 @@ public ClusterState execute(ClusterState currentState) { final Set activeCloneSources = snapshotsInProgress.asStream() .filter(SnapshotsInProgress.Entry::isClone) + .filter(entry -> entry.projectId().equals(projectId)) .map(SnapshotsInProgress.Entry::source) .collect(Collectors.toSet()); for (SnapshotId snapshotId : snapshotIds) { if (activeCloneSources.contains(snapshotId)) { throw new ConcurrentSnapshotExecutionException( - new Snapshot(repositoryName, snapshotId), + new Snapshot(projectId, repositoryName, snapshotId), "cannot delete snapshot while it is being cloned" ); } @@ -2188,7 +2239,7 @@ public ClusterState execute(ClusterState currentState) { "delete snapshot" ); - ensureNotReadOnly(currentState, repositoryName); + ensureNotReadOnly(projectMetadata, repositoryName); final SnapshotDeletionsInProgress deletionsInProgress = SnapshotDeletionsInProgress.get(currentState); @@ -2200,7 +2251,7 @@ public ClusterState execute(ClusterState currentState) { for (RestoreInProgress.Entry entry : restoreInProgress) { if (repositoryName.equals(entry.snapshot().getRepository()) && snapshotIds.contains(entry.snapshot().getSnapshotId())) { throw new ConcurrentSnapshotExecutionException( - new Snapshot(repositoryName, snapshotIds.stream().findFirst().get()), + new Snapshot(projectId, repositoryName, snapshotIds.stream().findFirst().get()), "cannot delete snapshot during a restore in progress in [" + restoreInProgress + "]" ); } @@ -2208,8 +2259,9 @@ public ClusterState execute(ClusterState currentState) { // Snapshot ids that will have to be physically deleted from the repository final Set snapshotIdsRequiringCleanup = new HashSet<>(snapshotIds); final SnapshotsInProgress updatedSnapshots = snapshotsInProgress.createCopyWithUpdatedEntriesForRepo( + projectId, repositoryName, - snapshotsInProgress.forRepo(repositoryName).stream().map(existing -> { + snapshotsInProgress.forRepo(projectId, repositoryName).stream().map(existing -> { if (existing.state() == SnapshotsInProgress.State.STARTED && snapshotIdsRequiringCleanup.contains(existing.snapshot().getSnapshotId())) { // snapshot is started - mark every non completed shard as aborted @@ -2238,7 +2290,7 @@ public ClusterState execute(ClusterState currentState) { // add the snapshot deletion to the cluster state final SnapshotDeletionsInProgress.Entry replacedEntry = deletionsInProgress.getEntries() .stream() - .filter(entry -> entry.repository().equals(repositoryName)) + .filter(entry -> entry.projectId().equals(projectId) && entry.repository().equals(repositoryName)) .filter(entry -> entry.state() == SnapshotDeletionsInProgress.State.WAITING) .findFirst() .orElse(null); @@ -2246,7 +2298,8 @@ public ClusterState execute(ClusterState currentState) { final Optional foundDuplicate = deletionsInProgress.getEntries() .stream() .filter( - entry -> entry.repository().equals(repositoryName) + entry -> entry.projectId().equals(projectId) + && entry.repository().equals(repositoryName) && entry.state() == SnapshotDeletionsInProgress.State.STARTED && entry.snapshots().containsAll(snapshotIds) ) @@ -2256,16 +2309,14 @@ public ClusterState execute(ClusterState currentState) { reusedExistingDelete = true; return currentState; } - @FixForMultiProject - final var projectId = ProjectId.DEFAULT; newDelete = new SnapshotDeletionsInProgress.Entry( projectId, repositoryName, List.copyOf(snapshotIdsRequiringCleanup), threadPool.absoluteTimeInMillis(), repositoryData.getGenId(), - updatedSnapshots.forRepo(repositoryName).stream().noneMatch(SnapshotsService::isWritingToRepository) - && deletionsInProgress.hasExecutingDeletion(repositoryName) == false + updatedSnapshots.forRepo(projectId, repositoryName).stream().noneMatch(SnapshotsService::isWritingToRepository) + && deletionsInProgress.hasExecutingDeletion(projectId, repositoryName) == false ? SnapshotDeletionsInProgress.State.STARTED : SnapshotDeletionsInProgress.State.WAITING ); @@ -2312,7 +2363,7 @@ public void clusterStateProcessed(ClusterState oldState, ClusterState newState) return; } if (newDelete.state() == SnapshotDeletionsInProgress.State.STARTED) { - if (tryEnterRepoLoop(repositoryName)) { + if (tryEnterRepoLoop(projectId, repositoryName)) { deleteSnapshotsFromRepository( newDelete, repositoryData, @@ -2457,7 +2508,7 @@ public void onResponse(RepositoryData repositoryData) { public void onFailure(Exception e) { submitUnbatchedTask( "fail repo tasks for [" + deleteEntry.repository() + "]", - new FailPendingRepoTasksTask(deleteEntry.repository(), e) + new FailPendingRepoTasksTask(deleteEntry.projectId(), deleteEntry.repository(), e) ); } }); @@ -2505,8 +2556,9 @@ public ClusterState execute(ClusterState currentState) throws Exception { // updated repository data and state on the retry. We don't want to wait for the write to finish though // because it could fail for any number of reasons so we just retry instead of waiting on the cluster state // to change in any form. + final ProjectMetadata projectMetadata = currentState.metadata().getProject(repository.getProjectId()); if (repositoryMetadataStart.equals( - RepositoriesMetadata.get(currentState).repository(repository.getMetadata().name()) + RepositoriesMetadata.get(projectMetadata).repository(repository.getMetadata().name()) )) { executedTask = true; return updateTask.execute(currentState); @@ -2548,7 +2600,7 @@ private void deleteSnapshotsFromRepository( IndexVersion minNodeVersion ) { if (repositoryOperations.startDeletion(deleteEntry.uuid())) { - assert currentlyFinalizing.contains(deleteEntry.repository()); + assert currentlyFinalizing.contains(new ProjectRepo(deleteEntry.projectId(), deleteEntry.repository())); final List snapshotIds = deleteEntry.snapshots(); assert deleteEntry.state() == SnapshotDeletionsInProgress.State.STARTED : "incorrect state for entry [" + deleteEntry + "]"; if (snapshotIds.isEmpty()) { @@ -2645,6 +2697,7 @@ protected SnapshotDeletionsInProgress filterDeletions(SnapshotDeletionsInProgres final SnapshotDeletionsInProgress updatedDeletions = deletionsWithoutSnapshots( deletions, deleteEntry.snapshots(), + deleteEntry.projectId(), deleteEntry.repository() ); return updatedDeletions == null ? deletions : updatedDeletions; @@ -2761,7 +2814,7 @@ public final void clusterStateProcessed(ClusterState oldState, ClusterState newS readyToResolveListeners.forEach(Runnable::run); if (newFinalizations.isEmpty()) { if (readyDeletions.isEmpty()) { - leaveRepoLoop(deleteEntry.repository()); + leaveRepoLoop(deleteEntry.projectId(), deleteEntry.repository()); } else { for (SnapshotDeletionsInProgress.Entry readyDeletion : readyDeletions) { deleteSnapshotsFromRepository( @@ -2772,7 +2825,7 @@ public final void clusterStateProcessed(ClusterState oldState, ClusterState newS } } } else { - leaveRepoLoop(deleteEntry.repository()); + leaveRepoLoop(deleteEntry.projectId(), deleteEntry.repository()); assert readyDeletions.stream().noneMatch(entry -> entry.repository().equals(deleteEntry.repository())) : "New finalizations " + newFinalizations + " added even though deletes " + readyDeletions + " are ready"; for (SnapshotsInProgress.Entry entry : newFinalizations) { @@ -2821,12 +2874,13 @@ private SnapshotsInProgress updatedSnapshotsInProgress(ClusterState currentState boolean changed = false; final String localNodeId = currentState.nodes().getLocalNodeId(); + final ProjectId projectId = deleteEntry.projectId(); final String repoName = deleteEntry.repository(); InFlightShardSnapshotStates inFlightShardStates = null; // Keep track of IndexId values that may have gone unreferenced due to the delete entry just executed. // See org.elasticsearch.cluster.SnapshotsInProgress.Entry#withUpdatedIndexIds for details. final Set newIndexIdsToRefresh = new HashSet<>(); - for (SnapshotsInProgress.Entry entry : snapshotsInProgress.forRepo(repoName)) { + for (SnapshotsInProgress.Entry entry : snapshotsInProgress.forRepo(projectId, repoName)) { if (entry.state().completed() == false) { // TODO: dry up redundant computation and code between clone and non-clone case, in particular reuse // `inFlightShardStates` across both clone and standard snapshot code @@ -2842,12 +2896,14 @@ private SnapshotsInProgress updatedSnapshotsInProgress(ClusterState currentState } // TODO: the below logic is very similar to that in #startCloning and both could be dried up against each other // also the code for standard snapshots could make use of this breakout as well - if (canBeUpdated.isEmpty() || updatedDeletions.hasExecutingDeletion(repoName)) { + if (canBeUpdated.isEmpty() || updatedDeletions.hasExecutingDeletion(projectId, repoName)) { // No shards can be updated in this snapshot so we just add it as is again snapshotEntries.add(entry); } else { if (inFlightShardStates == null) { - inFlightShardStates = InFlightShardSnapshotStates.forEntries(snapshotsInProgress.forRepo(repoName)); + inFlightShardStates = InFlightShardSnapshotStates.forEntries( + snapshotsInProgress.forRepo(projectId, repoName) + ); } final ImmutableOpenMap.Builder updatedAssignmentsBuilder = ImmutableOpenMap.builder(entry.shardSnapshotStatusByRepoShardId()); @@ -2891,7 +2947,7 @@ private SnapshotsInProgress updatedSnapshotsInProgress(ClusterState currentState final ImmutableOpenMap shardAssignments = shards( snapshotsInProgress, updatedDeletions, - currentState, + currentState.projectState(projectId), entry.indices().values(), entry.version().onOrAfter(SHARD_GEN_IN_REPO_DATA_VERSION), repositoryData, @@ -2904,7 +2960,7 @@ private SnapshotsInProgress updatedSnapshotsInProgress(ClusterState currentState final ShardSnapshotStatus updated = shardAssignments.get(sid); if (updated == null) { // We don't have a new assignment for this shard because its index was concurrently deleted - assert currentState.routingTable().hasIndex(sid.getIndex()) == false + assert currentState.routingTable(projectId).hasIndex(sid.getIndex()) == false : "Missing assignment for [" + sid + "]"; updatedAssignmentsBuilder.put(sid, ShardSnapshotStatus.MISSING); } else { @@ -2936,7 +2992,7 @@ private SnapshotsInProgress updatedSnapshotsInProgress(ClusterState currentState } snapshotEntries.replaceAll(entry -> entry.withUpdatedIndexIds(updatedIndexIds)); } - return changed ? snapshotsInProgress.createCopyWithUpdatedEntriesForRepo(repoName, snapshotEntries) : null; + return changed ? snapshotsInProgress.createCopyWithUpdatedEntriesForRepo(projectId, repoName, snapshotEntries) : null; } private static void markShardReassigned(RepositoryShardId shardId, Set reassignments) { @@ -3010,7 +3066,7 @@ private static void completeListenersIgnoringException(@Nullable List shards( SnapshotsInProgress snapshotsInProgress, SnapshotDeletionsInProgress deletionsInProgress, - ClusterState currentState, + ProjectState currentState, Collection indices, boolean useShardGenerations, RepositoryData repositoryData, @@ -3019,13 +3075,13 @@ private static ImmutableOpenMap builder = ImmutableOpenMap.builder(); final ShardGenerations shardGenerations = repositoryData.shardGenerations(); final InFlightShardSnapshotStates inFlightShardStates = InFlightShardSnapshotStates.forEntries( - snapshotsInProgress.forRepo(repoName) + snapshotsInProgress.forRepo(currentState.projectId(), repoName) ); - final boolean readyToExecute = deletionsInProgress.hasExecutingDeletion(repoName) == false; + final boolean readyToExecute = deletionsInProgress.hasExecutingDeletion(currentState.projectId(), repoName) == false; for (IndexId index : indices) { final String indexName = index.getName(); final boolean isNewIndex = repositoryData.getIndices().containsKey(indexName) == false; - IndexMetadata indexMetadata = currentState.metadata().getProject().index(indexName); + IndexMetadata indexMetadata = currentState.metadata().index(indexName); if (indexMetadata == null) { // The index was deleted before we managed to start the snapshot - mark it as missing. builder.put(new ShardId(indexName, IndexMetadata.INDEX_UUID_NA_VALUE, 0), ShardSnapshotStatus.MISSING); @@ -3247,7 +3303,7 @@ static final class SnapshotShardsUpdateContext { private final Predicate nodeIdRemovalPredicate; /** Updates outstanding to be applied to existing snapshot entries. Maps repository name to shard snapshot updates. */ - private final Map> updatesByRepo; + private final Map> updatesByRepo; /** Updates that were used to update an existing in-progress shard snapshot */ private final Set executedUpdates = new HashSet<>(); @@ -3270,11 +3326,14 @@ static final class SnapshotShardsUpdateContext { this.nodeIdRemovalPredicate = SnapshotsInProgress.get(initialState)::isNodeIdForRemoval; this.completionHandler = completionHandler; - // SnapshotsInProgress is organized by repository name, so organize the shard snapshot updates similarly. + // SnapshotsInProgress is organized by ProjectRepo, so organize the shard snapshot updates similarly. this.updatesByRepo = new HashMap<>(); for (final var taskContext : batchExecutionContext.taskContexts()) { if (taskContext.getTask() instanceof ShardSnapshotUpdate task) { - updatesByRepo.computeIfAbsent(task.snapshot.getRepository(), r -> new ArrayList<>()).add(task); + updatesByRepo.computeIfAbsent( + new ProjectRepo(task.snapshot.getProjectId(), task.snapshot.getRepository()), + r -> new ArrayList<>() + ).add(task); } } } @@ -3287,9 +3346,9 @@ static final class SnapshotShardsUpdateContext { SnapshotsInProgress computeUpdatedState() { final SnapshotsInProgress existing = SnapshotsInProgress.get(initialState); SnapshotsInProgress updated = existing; - for (Map.Entry> updates : updatesByRepo.entrySet()) { - final String repoName = updates.getKey(); - final List oldEntries = existing.forRepo(repoName); + for (Map.Entry> updates : updatesByRepo.entrySet()) { + final var projectRepo = updates.getKey(); + final List oldEntries = existing.forRepo(projectRepo); if (oldEntries.isEmpty()) { continue; } @@ -3303,7 +3362,7 @@ SnapshotsInProgress computeUpdatedState() { newlyCompletedEntries.add(newEntry); } } - updated = updated.createCopyWithUpdatedEntriesForRepo(repoName, newEntries); + updated = updated.createCopyWithUpdatedEntriesForRepo(projectRepo.projectId(), projectRepo.name(), newEntries); } if (changedCount > 0) { @@ -3604,7 +3663,7 @@ private void startShardSnapshot(RepositoryShardId repoShardId, ShardGeneration g + "] because it's a normal snapshot but did not"; // work out the node to run the snapshot task on as it might have changed from the previous operation if it was a clone // or there was a primary failover - final IndexRoutingTable indexRouting = initialState.routingTable().index(index); + final IndexRoutingTable indexRouting = initialState.routingTable(entry.projectId()).index(index); final ShardRouting shardRouting; if (indexRouting == null) { shardRouting = null; @@ -3649,14 +3708,14 @@ interface ShardSnapshotUpdateCompletionHandler { void handleCompletion( ShardSnapshotUpdateResult shardSnapshotUpdateResult, List newlyCompletedEntries, - Set updatedRepositories + Set updatedRepositories ); } private void handleShardSnapshotUpdateCompletion( ShardSnapshotUpdateResult shardSnapshotUpdateResult, List newlyCompletedEntries, - Set updatedRepositories + Set updatedRepositories ) { // Maybe this state update completed one or more snapshots. If we are not already ending them because of some earlier update, end // them now. @@ -3767,13 +3826,13 @@ private void innerUpdateSnapshotState( masterServiceTaskQueue.submitTask("update snapshot state", update, null); } - private void startExecutableClones(SnapshotsInProgress snapshotsInProgress, @Nullable String repoName) { - if (repoName == null) { + private void startExecutableClones(SnapshotsInProgress snapshotsInProgress, @Nullable ProjectRepo projectRepo) { + if (projectRepo == null) { for (List entries : snapshotsInProgress.entriesByRepo()) { startExecutableClones(entries); } } else { - startExecutableClones(snapshotsInProgress.forRepo(repoName)); + startExecutableClones(snapshotsInProgress.forRepo(projectRepo)); } } @@ -3856,9 +3915,11 @@ private final class FailPendingRepoTasksTask extends ClusterStateUpdateTask { // Failure that caused the decision to fail all snapshots and deletes for a repo private final Exception failure; + private final ProjectId projectId; private final String repository; - FailPendingRepoTasksTask(String repository, Exception failure) { + FailPendingRepoTasksTask(ProjectId projectId, String repository, Exception failure) { + this.projectId = projectId; this.repository = repository; this.failure = failure; } @@ -3870,7 +3931,7 @@ public ClusterState execute(ClusterState currentState) { final List remainingEntries = deletionsInProgress.getEntries(); List updatedEntries = new ArrayList<>(remainingEntries.size()); for (SnapshotDeletionsInProgress.Entry entry : remainingEntries) { - if (entry.repository().equals(repository)) { + if (entry.projectId().equals(projectId) && entry.repository().equals(repository)) { changed = true; deletionsToFail.add(entry.uuid()); } else { @@ -3880,14 +3941,14 @@ public ClusterState execute(ClusterState currentState) { final SnapshotDeletionsInProgress updatedDeletions = changed ? SnapshotDeletionsInProgress.of(updatedEntries) : null; final SnapshotsInProgress snapshotsInProgress = SnapshotsInProgress.get(currentState); boolean changedSnapshots = false; - for (SnapshotsInProgress.Entry entry : snapshotsInProgress.forRepo(repository)) { + for (SnapshotsInProgress.Entry entry : snapshotsInProgress.forRepo(projectId, repository)) { // We failed to read repository data for this delete, it is not the job of SnapshotsService to // retry these kinds of issues so we fail all the pending snapshots snapshotsToFail.add(entry.snapshot()); changedSnapshots = true; } final SnapshotsInProgress updatedSnapshotsInProgress = changedSnapshots - ? snapshotsInProgress.createCopyWithUpdatedEntriesForRepo(repository, List.of()) + ? snapshotsInProgress.createCopyWithUpdatedEntriesForRepo(projectId, repository, List.of()) : null; return updateWithSnapshots(currentState, updatedSnapshotsInProgress, updatedDeletions); } @@ -3907,11 +3968,11 @@ public void clusterStateProcessed(ClusterState oldState, ClusterState newState) final List readyToResolveListeners = new ArrayList<>(); synchronized (currentlyFinalizing) { Tuple finalization; - while ((finalization = repositoryOperations.pollFinalization(repository)) != null) { + while ((finalization = repositoryOperations.pollFinalization(projectId, repository)) != null) { assert snapshotsToFail.contains(finalization.v1()) : "[" + finalization.v1() + "] not found in snapshots to fail " + snapshotsToFail; } - leaveRepoLoop(repository); + leaveRepoLoop(projectId, repository); for (Snapshot snapshot : snapshotsToFail) { failSnapshotCompletionListeners(snapshot, failure, readyToResolveListeners::add); } @@ -3932,7 +3993,7 @@ private static final class OngoingRepositoryOperations { * Map of repository name to a deque of {@link Snapshot} that need to be finalized for the repository and the * {@link Metadata to use when finalizing}. */ - private final Map> snapshotsToFinalize = new HashMap<>(); + private final Map> snapshotsToFinalize = new HashMap<>(); /** * Set of delete operations currently being executed against the repository. The values in this set are the delete UUIDs returned @@ -3944,10 +4005,11 @@ private static final class OngoingRepositoryOperations { private Metadata latestKnownMetaData; @Nullable - synchronized Tuple pollFinalization(String repository) { + synchronized Tuple pollFinalization(ProjectId projectId, String repository) { assertConsistent(); + final var projectRepo = new ProjectRepo(projectId, repository); final Snapshot nextEntry; - final Deque queued = snapshotsToFinalize.get(repository); + final Deque queued = snapshotsToFinalize.get(projectRepo); if (queued == null) { return null; } @@ -3955,7 +4017,7 @@ synchronized Tuple pollFinalization(String repository) { assert nextEntry != null; final Tuple res = Tuple.tuple(nextEntry, latestKnownMetaData); if (queued.isEmpty()) { - snapshotsToFinalize.remove(repository); + snapshotsToFinalize.remove(projectRepo); } if (snapshotsToFinalize.isEmpty()) { latestKnownMetaData = null; @@ -3973,7 +4035,8 @@ void finishDeletion(String deleteUUID) { } synchronized void addFinalization(Snapshot snapshot, Metadata metadata) { - snapshotsToFinalize.computeIfAbsent(snapshot.getRepository(), k -> new LinkedList<>()).add(snapshot); + snapshotsToFinalize.computeIfAbsent(new ProjectRepo(snapshot.getProjectId(), snapshot.getRepository()), k -> new LinkedList<>()) + .add(snapshot); this.latestKnownMetaData = metadata; assertConsistent(); } @@ -3993,7 +4056,7 @@ synchronized boolean isEmpty() { } synchronized boolean assertNotQueued(Snapshot snapshot) { - if (snapshotsToFinalize.getOrDefault(snapshot.getRepository(), new LinkedList<>()) + if (snapshotsToFinalize.getOrDefault(new ProjectRepo(snapshot.getProjectId(), snapshot.getRepository()), new LinkedList<>()) .stream() .anyMatch(entry -> entry.equals(snapshot))) { @@ -4094,25 +4157,22 @@ public ClusterState execute(BatchExecutionContext batchExecutionCo // Handle the tasks to apply the shard snapshot updates (ShardSnapshotUpdate tasks). SnapshotsInProgress snapshotsInProgress = shardsUpdateContext.computeUpdatedState(); - final var project = state.metadata().getProject(); - final RegisteredPolicySnapshots.Builder registeredPolicySnapshots = project.custom( - RegisteredPolicySnapshots.TYPE, - RegisteredPolicySnapshots.EMPTY - ).builder(); + final Map registeredPolicySnapshotsBuilders = new HashMap<>(); // Handle the tasks to create new snapshots (CreateSnapshotTask tasks). for (final var taskContext : batchExecutionContext.taskContexts()) { if (taskContext.getTask() instanceof CreateSnapshotTask task) { try { - final var repoMeta = RepositoriesMetadata.get(state).repository(task.snapshot.getRepository()); + final var projectMetadata = state.metadata().getProject(task.snapshot.getProjectId()); + final var repoMeta = RepositoriesMetadata.get(projectMetadata).repository(task.snapshot.getRepository()); if (RepositoriesService.isReadOnly(repoMeta.settings())) { taskContext.onFailure(new RepositoryException(repoMeta.name(), "repository is readonly")); continue; } - registeredPolicySnapshots.addIfSnapshotIsSLMInitiated( - task.createSnapshotRequest.userMetadata(), - task.snapshot.getSnapshotId() - ); + registeredPolicySnapshotsBuilders.computeIfAbsent( + projectMetadata.id(), + ignored -> projectMetadata.custom(RegisteredPolicySnapshots.TYPE, RegisteredPolicySnapshots.EMPTY).builder() + ).addIfSnapshotIsSLMInitiated(task.createSnapshotRequest.userMetadata(), task.snapshot.getSnapshotId()); if (Objects.equals(task.initialRepositoryMetadata, repoMeta)) { snapshotsInProgress = createSnapshot(task, taskContext, state, snapshotsInProgress); } else { @@ -4138,12 +4198,11 @@ public ClusterState execute(BatchExecutionContext batchExecutionCo return state; } - return ClusterState.builder(state) - .putCustom(SnapshotsInProgress.TYPE, snapshotsInProgress) - .putProjectMetadata( - ProjectMetadata.builder(project).putCustom(RegisteredPolicySnapshots.TYPE, registeredPolicySnapshots.build()) - ) - .build(); + final var metadataBuilder = Metadata.builder(state.metadata()); + registeredPolicySnapshotsBuilders.forEach( + (projectId, builder) -> metadataBuilder.getProject(projectId).putCustom(RegisteredPolicySnapshots.TYPE, builder.build()) + ); + return ClusterState.builder(state).putCustom(SnapshotsInProgress.TYPE, snapshotsInProgress).metadata(metadataBuilder).build(); } private SnapshotsInProgress createSnapshot( @@ -4156,11 +4215,12 @@ private SnapshotsInProgress createSnapshot( final Snapshot snapshot = createSnapshotTask.snapshot; final String repositoryName = snapshot.getRepository(); final String snapshotName = snapshot.getSnapshotId().getName(); - ensureRepositoryExists(repositoryName, currentState); + final var projectState = currentState.projectState(snapshot.getProjectId()); + ensureRepositoryExists(repositoryName, projectState.metadata()); final Repository repository = createSnapshotTask.repository; ensureSnapshotNameAvailableInRepo(repositoryData, snapshotName, repository); - ensureSnapshotNameNotRunning(snapshotsInProgress, repositoryName, snapshotName); - validate(repositoryName, snapshotName, currentState); + ensureSnapshotNameNotRunning(snapshotsInProgress, snapshot.getProjectId(), repositoryName, snapshotName); + validate(repositoryName, snapshotName, projectState.metadata()); final SnapshotDeletionsInProgress deletionsInProgress = SnapshotDeletionsInProgress.get(currentState); ensureNoCleanupInProgress(currentState, repositoryName, snapshotName, "create snapshot"); ensureBelowConcurrencyLimit(repositoryName, snapshotName, snapshotsInProgress, deletionsInProgress); @@ -4168,7 +4228,7 @@ private SnapshotsInProgress createSnapshot( final CreateSnapshotRequest request = createSnapshotTask.createSnapshotRequest; // Store newSnapshot here to be processed in clusterStateProcessed Map> requestedIndices = Arrays.stream( - indexNameExpressionResolver.concreteIndexNames(currentState, request) + indexNameExpressionResolver.concreteIndexNames(projectState.metadata(), request) ).collect(Collectors.partitioningBy(systemIndices::isSystemIndex)); List requestedSystemIndices = requestedIndices.get(true); @@ -4228,17 +4288,17 @@ private SnapshotsInProgress createSnapshot( Set featureSystemIndices = feature.getIndexDescriptors() .stream() - .flatMap(descriptor -> descriptor.getMatchingIndices(currentState.metadata().getProject()).stream()) + .flatMap(descriptor -> descriptor.getMatchingIndices(projectState.metadata()).stream()) .collect(Collectors.toSet()); Set featureAssociatedIndices = feature.getAssociatedIndexDescriptors() .stream() - .flatMap(descriptor -> descriptor.getMatchingIndices(currentState.metadata().getProject()).stream()) + .flatMap(descriptor -> descriptor.getMatchingIndices(projectState.metadata()).stream()) .collect(Collectors.toSet()); Set featureSystemDataStreams = new HashSet<>(); Set featureDataStreamBackingIndices = new HashSet<>(); for (SystemDataStreamDescriptor sdd : feature.getDataStreamDescriptors()) { - List backingIndexNames = sdd.getBackingIndexNames(currentState.metadata()); + List backingIndexNames = sdd.getBackingIndexNames(projectState.metadata()); if (backingIndexNames.size() > 0) { featureDataStreamBackingIndices.addAll(backingIndexNames); featureSystemDataStreams.add(sdd.getDataStreamName()); @@ -4267,7 +4327,7 @@ private SnapshotsInProgress createSnapshot( logger.trace("[{}][{}] creating snapshot for indices [{}]", repositoryName, snapshotName, indices); final Map allIndices = new HashMap<>(); - for (SnapshotsInProgress.Entry runningSnapshot : snapshotsInProgress.forRepo(repositoryName)) { + for (SnapshotsInProgress.Entry runningSnapshot : snapshotsInProgress.forRepo(projectState.projectId(), repositoryName)) { allIndices.putAll(runningSnapshot.indices()); } final Map indexIds = repositoryData.resolveNewIndices(indices, allIndices); @@ -4283,7 +4343,7 @@ private SnapshotsInProgress createSnapshot( ImmutableOpenMap shards = shards( snapshotsInProgress, deletionsInProgress, - currentState, + projectState, indexIds.values(), useShardGenerations(version), repositoryData, diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportSnapshotsStatusActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportSnapshotsStatusActionTests.java index adaad4eede0d6..09bf072c7225d 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportSnapshotsStatusActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportSnapshotsStatusActionTests.java @@ -82,7 +82,8 @@ public void initializeComponents() throws Exception { threadPool, repositoriesService, nodeClient, - new ActionFilters(Set.of()) + new ActionFilters(Set.of()), + TestProjectResolvers.DEFAULT_PROJECT_ONLY ); } @@ -198,6 +199,7 @@ public void onFailure(Exception e) { action.buildResponse( SnapshotsInProgress.EMPTY, + ProjectId.DEFAULT, new SnapshotsStatusRequest(TEST_REQUEST_TIMEOUT), currentSnapshotEntries, nodeSnapshotStatuses, @@ -357,6 +359,7 @@ public void onFailure(Exception e) { action.buildResponse( SnapshotsInProgress.EMPTY, + ProjectId.DEFAULT, new SnapshotsStatusRequest(TEST_REQUEST_TIMEOUT), currentSnapshotEntries, nodeSnapshotStatuses, diff --git a/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java b/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java index c005691b212b2..d3f0c1aa2bb42 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java +++ b/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java @@ -2781,19 +2781,47 @@ public boolean clusterHasFeature(ClusterState state, NodeFeature feature) { ); actions.put( TransportCleanupRepositoryAction.TYPE, - new TransportCleanupRepositoryAction(transportService, clusterService, repositoriesService, threadPool, actionFilters) + new TransportCleanupRepositoryAction( + transportService, + clusterService, + repositoriesService, + threadPool, + actionFilters, + TestProjectResolvers.DEFAULT_PROJECT_ONLY + ) ); actions.put( TransportCreateSnapshotAction.TYPE, - new TransportCreateSnapshotAction(transportService, clusterService, threadPool, snapshotsService, actionFilters) + new TransportCreateSnapshotAction( + transportService, + clusterService, + threadPool, + snapshotsService, + actionFilters, + TestProjectResolvers.DEFAULT_PROJECT_ONLY + ) ); actions.put( TransportCloneSnapshotAction.TYPE, - new TransportCloneSnapshotAction(transportService, clusterService, threadPool, snapshotsService, actionFilters) + new TransportCloneSnapshotAction( + transportService, + clusterService, + threadPool, + snapshotsService, + actionFilters, + TestProjectResolvers.DEFAULT_PROJECT_ONLY + ) ); actions.put( TransportGetSnapshotsAction.TYPE, - new TransportGetSnapshotsAction(transportService, clusterService, threadPool, repositoriesService, actionFilters) + new TransportGetSnapshotsAction( + transportService, + clusterService, + threadPool, + repositoriesService, + actionFilters, + TestProjectResolvers.DEFAULT_PROJECT_ONLY + ) ); actions.put( TransportClusterRerouteAction.TYPE, @@ -2843,7 +2871,14 @@ public boolean clusterHasFeature(ClusterState state, NodeFeature feature) { ); actions.put( TransportDeleteSnapshotAction.TYPE, - new TransportDeleteSnapshotAction(transportService, clusterService, threadPool, snapshotsService, actionFilters) + new TransportDeleteSnapshotAction( + transportService, + clusterService, + threadPool, + snapshotsService, + actionFilters, + TestProjectResolvers.DEFAULT_PROJECT_ONLY + ) ); client.initialize( actions, From f3c531ffa2e21b3f633f1f5ad2b40e4c1dce9bcb Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Wed, 25 Jun 2025 17:51:51 +1000 Subject: [PATCH 02/12] Fix more issues and write the correct metadata --- .../repositories/s3/S3Repository.java | 1 + .../snapshots/CloneSnapshotIT.java | 2 +- ...etadataLoadingDuringSnapshotRestoreIT.java | 4 +- .../cluster/SnapshotDeletionsInProgress.java | 13 +- .../cluster/SnapshotsInProgress.java | 24 +++- .../cluster/metadata/ProjectMetadata.java | 3 + .../SnapshotInProgressAllocationDecider.java | 4 +- .../elasticsearch/node/NodeConstruction.java | 3 +- .../repositories/FilterRepository.java | 4 +- .../repositories/FinalizeSnapshotContext.java | 8 ++ .../repositories/InvalidRepository.java | 2 +- .../repositories/ProjectRepo.java | 66 +++++++++ .../repositories/RepositoriesService.java | 2 +- .../repositories/Repository.java | 12 +- .../repositories/RepositoryOperation.java | 48 ------- .../repositories/UnknownTypeRepository.java | 2 +- .../blobstore/BlobStoreRepository.java | 55 +++++--- .../blobstore/ChecksumBlobStoreFormat.java | 23 +-- .../InFlightShardSnapshotStates.java | 2 +- .../snapshots/RestoreService.java | 4 +- .../elasticsearch/snapshots/SnapshotInfo.java | 5 +- .../snapshots/SnapshotsService.java | 132 ++++++++++++------ .../repositories/InvalidRepositoryTests.java | 2 +- .../RepositoriesServiceTests.java | 2 +- .../UnknownTypeRepositoryTests.java | 2 +- .../BlobStoreRepositoryRestoreTests.java | 1 + .../snapshots/BlobStoreFormatTests.java | 12 +- .../SnapshotInfoBlobSerializationTests.java | 3 +- .../snapshots/SnapshotResiliencyTests.java | 3 +- ...SnapshotsInProgressSerializationTests.java | 2 +- .../index/shard/RestoreOnlyRepository.java | 2 +- .../AbstractSnapshotIntegTestCase.java | 5 +- .../xpack/ccr/repository/CcrRepository.java | 2 +- .../SourceOnlySnapshotRepository.java | 1 + .../AsyncRetryDuringSnapshotActionStep.java | 2 +- .../SourceOnlySnapshotShardTests.java | 1 + .../xpack/slm/SLMStatDisruptionIT.java | 1 + .../xpack/slm/SnapshotLifecycleTask.java | 5 +- .../TransportGetSnapshotLifecycleAction.java | 2 +- .../RepositoryVerifyIntegrityIT.java | 10 +- .../RepositoryIntegrityVerifier.java | 2 +- 41 files changed, 318 insertions(+), 161 deletions(-) create mode 100644 server/src/main/java/org/elasticsearch/repositories/ProjectRepo.java diff --git a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java index 92b0dcc2f1cdc..6befdeb2c068a 100644 --- a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java +++ b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java @@ -391,6 +391,7 @@ public void finalizeSnapshot(final FinalizeSnapshotContext finalizeSnapshotConte if (SnapshotsService.useShardGenerations(finalizeSnapshotContext.repositoryMetaVersion()) == false) { final ListenableFuture metadataDone = new ListenableFuture<>(); wrappedFinalizeContext = new FinalizeSnapshotContext( + finalizeSnapshotContext.serializeProjectMetadata(), finalizeSnapshotContext.updatedShardGenerations(), finalizeSnapshotContext.repositoryStateId(), finalizeSnapshotContext.clusterMetadata(), diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/CloneSnapshotIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/CloneSnapshotIT.java index caf7f852cc15e..c93c66e440403 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/CloneSnapshotIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/CloneSnapshotIT.java @@ -890,7 +890,7 @@ private static BlobStoreIndexShardSnapshots readShardGeneration( ShardGeneration generation ) throws IOException { return BlobStoreRepository.INDEX_SHARD_SNAPSHOTS_FORMAT.read( - repository.getMetadata().name(), + repository.getProjectRepo(), repository.shardContainer(repositoryShardId.index(), repositoryShardId.shardId()), generation.getGenerationUUID(), NamedXContentRegistry.EMPTY diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/MetadataLoadingDuringSnapshotRestoreIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/MetadataLoadingDuringSnapshotRestoreIT.java index 89a8af49efdef..480a079eb8b5c 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/MetadataLoadingDuringSnapshotRestoreIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/MetadataLoadingDuringSnapshotRestoreIT.java @@ -189,9 +189,9 @@ public CountingMockRepository( } @Override - public Metadata getSnapshotGlobalMetadata(SnapshotId snapshotId) { + public Metadata getSnapshotGlobalMetadata(SnapshotId snapshotId, boolean fromProjectMetadata) { globalMetadata.computeIfAbsent(snapshotId.getName(), (s) -> new AtomicInteger(0)).incrementAndGet(); - return super.getSnapshotGlobalMetadata(snapshotId); + return super.getSnapshotGlobalMetadata(snapshotId, fromProjectMetadata); } @Override diff --git a/server/src/main/java/org/elasticsearch/cluster/SnapshotDeletionsInProgress.java b/server/src/main/java/org/elasticsearch/cluster/SnapshotDeletionsInProgress.java index cbd4c2ed14ca4..f16e03a7e9594 100644 --- a/server/src/main/java/org/elasticsearch/cluster/SnapshotDeletionsInProgress.java +++ b/server/src/main/java/org/elasticsearch/cluster/SnapshotDeletionsInProgress.java @@ -20,6 +20,7 @@ import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.util.CollectionUtils; import org.elasticsearch.core.SuppressForbidden; +import org.elasticsearch.repositories.ProjectRepo; import org.elasticsearch.repositories.RepositoryOperation; import org.elasticsearch.snapshots.SnapshotId; import org.elasticsearch.xcontent.ToXContent; @@ -65,10 +66,10 @@ public SnapshotDeletionsInProgress(StreamInput in) throws IOException { } private static boolean assertNoConcurrentDeletionsForSameRepository(List entries) { - final Set activeRepositories = new HashSet<>(); + final Set activeRepositories = new HashSet<>(); for (Entry entry : entries) { if (entry.state() == State.STARTED) { - final boolean added = activeRepositories.add(entry.repository()); + final boolean added = activeRepositories.add(new ProjectRepo(entry.projectId(), entry.repository())); assert added : "Found multiple running deletes for a single repository in " + entries; } } @@ -114,12 +115,12 @@ public List getEntries() { /** * Checks if there is an actively executing delete operation for the given repository * - * @param projectId project to look for the repository + * @param projectId project for the repository * @param repository repository name */ public boolean hasExecutingDeletion(ProjectId projectId, String repository) { for (Entry entry : entries) { - if (entry.state() == State.STARTED && entry.repository().equals(repository)) { + if (entry.state() == State.STARTED && entry.projectId.equals(projectId) && entry.repository().equals(repository)) { return true; } } @@ -134,6 +135,10 @@ public boolean hasDeletionsInProgress() { return entries.isEmpty() == false; } + public boolean hasDeletionsInProgress(ProjectId projectId) { + return entries.stream().filter(entry -> entry.projectId().equals(projectId)).findFirst().isPresent(); + } + @Override public String getWriteableName() { return TYPE; diff --git a/server/src/main/java/org/elasticsearch/cluster/SnapshotsInProgress.java b/server/src/main/java/org/elasticsearch/cluster/SnapshotsInProgress.java index 146de4ee37f2c..155de1c889320 100644 --- a/server/src/main/java/org/elasticsearch/cluster/SnapshotsInProgress.java +++ b/server/src/main/java/org/elasticsearch/cluster/SnapshotsInProgress.java @@ -34,8 +34,8 @@ import org.elasticsearch.logging.LogManager; import org.elasticsearch.logging.Logger; import org.elasticsearch.repositories.IndexId; +import org.elasticsearch.repositories.ProjectRepo; import org.elasticsearch.repositories.RepositoryOperation; -import org.elasticsearch.repositories.RepositoryOperation.ProjectRepo; import org.elasticsearch.repositories.RepositoryShardId; import org.elasticsearch.repositories.ShardGeneration; import org.elasticsearch.repositories.ShardSnapshotResult; @@ -60,7 +60,7 @@ import java.util.Set; import java.util.stream.Stream; -import static org.elasticsearch.repositories.RepositoryOperation.PROJECT_REPO_SERIALIZER; +import static org.elasticsearch.repositories.ProjectRepo.PROJECT_REPO_SERIALIZER; /** * Meta data about snapshots that are currently executing @@ -201,10 +201,30 @@ public Iterable> entriesByRepo() { return () -> Iterators.map(entries.values().iterator(), byRepo -> byRepo.entries); } + /** + * Similar to {@link #entriesByRepo()} but only returns entries for the specified project. + */ + public Iterable> entriesByRepo(ProjectId projectId) { + return () -> Iterators.map( + Iterators.filter(entries.entrySet().iterator(), entry -> entry.getKey().projectId().equals(projectId)), + entry -> entry.getValue().entries + ); + } + public Stream asStream() { return entries.values().stream().flatMap(t -> t.entries.stream()); } + /** + * Similar to {@link #asStream()} but only returns entries for the specified project. + */ + public Stream asStream(ProjectId projectId) { + return entries.entrySet() + .stream() + .filter(entry -> entry.getKey().projectId().equals(projectId)) + .flatMap(entry -> entry.getValue().entries.stream()); + } + @Nullable public Entry snapshot(final Snapshot snapshot) { return findSnapshotInList(snapshot, forRepo(snapshot.getProjectId(), snapshot.getRepository())); diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/ProjectMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/ProjectMetadata.java index 4a10964b1be57..4b574b8313c28 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/ProjectMetadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/ProjectMetadata.java @@ -2089,6 +2089,9 @@ static boolean assertDataStreams(Map indices, DataStreamM public static ProjectMetadata fromXContent(XContentParser parser) throws IOException { XContentParser.Token token = parser.currentToken(); + if (token == null) { + token = parser.nextToken(); + } String currentFieldName = null; XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, token, parser); diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/SnapshotInProgressAllocationDecider.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/SnapshotInProgressAllocationDecider.java index 2a76d36466c80..d204dfe8a3841 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/SnapshotInProgressAllocationDecider.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/SnapshotInProgressAllocationDecider.java @@ -10,6 +10,7 @@ package org.elasticsearch.cluster.routing.allocation.decider; import org.elasticsearch.cluster.SnapshotsInProgress; +import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.cluster.metadata.SingleNodeShutdownMetadata; import org.elasticsearch.cluster.routing.RoutingNode; import org.elasticsearch.cluster.routing.ShardRouting; @@ -73,7 +74,8 @@ private static Decision canMove(ShardRouting shardRouting, RoutingAllocation all return YES_NOT_SNAPSHOTTED; } - for (final var entriesByRepo : snapshotsInProgress.entriesByRepo()) { + final ProjectId projectId = allocation.metadata().projectFor(shardRouting.index()).id(); + for (final var entriesByRepo : snapshotsInProgress.entriesByRepo(projectId)) { for (final var entry : entriesByRepo) { if (entry.isClone()) { // clones do not run on data nodes diff --git a/server/src/main/java/org/elasticsearch/node/NodeConstruction.java b/server/src/main/java/org/elasticsearch/node/NodeConstruction.java index 73a2efabb9e81..9e1facab0ea56 100644 --- a/server/src/main/java/org/elasticsearch/node/NodeConstruction.java +++ b/server/src/main/java/org/elasticsearch/node/NodeConstruction.java @@ -1115,7 +1115,8 @@ public Map queryFields() { repositoriesService, transportService, actionModule.getActionFilters(), - systemIndices + systemIndices, + projectResolver.supportsMultipleProjects() ); SnapshotShardsService snapshotShardsService = new SnapshotShardsService( settings, diff --git a/server/src/main/java/org/elasticsearch/repositories/FilterRepository.java b/server/src/main/java/org/elasticsearch/repositories/FilterRepository.java index 34803a12d66cd..b0a55d1f66411 100644 --- a/server/src/main/java/org/elasticsearch/repositories/FilterRepository.java +++ b/server/src/main/java/org/elasticsearch/repositories/FilterRepository.java @@ -67,8 +67,8 @@ public void getSnapshotInfo( } @Override - public Metadata getSnapshotGlobalMetadata(SnapshotId snapshotId) { - return in.getSnapshotGlobalMetadata(snapshotId); + public Metadata getSnapshotGlobalMetadata(SnapshotId snapshotId, boolean fromProjectMetadata) { + return in.getSnapshotGlobalMetadata(snapshotId, fromProjectMetadata); } @Override diff --git a/server/src/main/java/org/elasticsearch/repositories/FinalizeSnapshotContext.java b/server/src/main/java/org/elasticsearch/repositories/FinalizeSnapshotContext.java index d643448b691f0..c9d8a703bbfdd 100644 --- a/server/src/main/java/org/elasticsearch/repositories/FinalizeSnapshotContext.java +++ b/server/src/main/java/org/elasticsearch/repositories/FinalizeSnapshotContext.java @@ -27,6 +27,7 @@ */ public final class FinalizeSnapshotContext extends DelegatingActionListener { + private final boolean serializeProjectMetadata; private final UpdatedShardGenerations updatedShardGenerations; /** @@ -46,6 +47,7 @@ public final class FinalizeSnapshotContext extends DelegatingActionListener PROJECT_REPO_SERIALIZER = new DiffableUtils.KeySerializer<>() { + @Override + public void writeKey(ProjectRepo key, StreamOutput out) throws IOException { + key.writeTo(out); + } + + @Override + public ProjectRepo readKey(StreamInput in) throws IOException { + return new ProjectRepo(in); + } + }; + + public ProjectRepo(StreamInput in) throws IOException { + this(ProjectId.readFrom(in), in.readString()); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + projectId.writeTo(out); + out.writeString(name); + } + + @Override + public String toString() { + return projectRepoString(projectId, name); + } + + public static String projectRepoString(ProjectId projectId, String repositoryName) { + return "[" + projectId + "][" + repositoryName + "]"; + } +} diff --git a/server/src/main/java/org/elasticsearch/repositories/RepositoriesService.java b/server/src/main/java/org/elasticsearch/repositories/RepositoriesService.java index 6248e74bee109..f5d8cad85a290 100644 --- a/server/src/main/java/org/elasticsearch/repositories/RepositoriesService.java +++ b/server/src/main/java/org/elasticsearch/repositories/RepositoriesService.java @@ -77,7 +77,7 @@ import static java.util.Collections.unmodifiableMap; import static org.elasticsearch.core.Strings.format; -import static org.elasticsearch.repositories.RepositoryOperation.projectRepoString; +import static org.elasticsearch.repositories.ProjectRepo.projectRepoString; import static org.elasticsearch.snapshots.SearchableSnapshotsSettings.SEARCHABLE_SNAPSHOTS_REPOSITORY_NAME_SETTING_KEY; import static org.elasticsearch.snapshots.SearchableSnapshotsSettings.SEARCHABLE_SNAPSHOTS_REPOSITORY_UUID_SETTING_KEY; diff --git a/server/src/main/java/org/elasticsearch/repositories/Repository.java b/server/src/main/java/org/elasticsearch/repositories/Repository.java index 74c5f3e0e9dc0..e98c3ffb16532 100644 --- a/server/src/main/java/org/elasticsearch/repositories/Repository.java +++ b/server/src/main/java/org/elasticsearch/repositories/Repository.java @@ -87,6 +87,13 @@ default Repository create( @Nullable ProjectId getProjectId(); + /** + * Get the project qualified repository + */ + default ProjectRepo getProjectRepo() { + return new ProjectRepo(getProjectId(), getMetadata().name()); + } + /** * Returns metadata about this repository. */ @@ -138,10 +145,11 @@ public void onFailure(Exception e) { /** * Returns global metadata associated with the snapshot. * - * @param snapshotId the snapshot id to load the global metadata from + * @param snapshotId the snapshot id to load the global metadata from + * @param fromProjectMetadata The metadata may need to be constructed by first reading the project metadata * @return the global metadata about the snapshot */ - Metadata getSnapshotGlobalMetadata(SnapshotId snapshotId); + Metadata getSnapshotGlobalMetadata(SnapshotId snapshotId, boolean fromProjectMetadata); /** * Returns the index metadata associated with the snapshot. diff --git a/server/src/main/java/org/elasticsearch/repositories/RepositoryOperation.java b/server/src/main/java/org/elasticsearch/repositories/RepositoryOperation.java index e6bdf5bc5efc8..a0660758d2b80 100644 --- a/server/src/main/java/org/elasticsearch/repositories/RepositoryOperation.java +++ b/server/src/main/java/org/elasticsearch/repositories/RepositoryOperation.java @@ -8,13 +8,7 @@ */ package org.elasticsearch.repositories; -import org.elasticsearch.cluster.DiffableUtils; import org.elasticsearch.cluster.metadata.ProjectId; -import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.io.stream.Writeable; - -import java.io.IOException; /** * Coordinates of an operation that modifies a repository, assuming that repository at a specific generation. @@ -36,46 +30,4 @@ public interface RepositoryOperation { */ long repositoryStateId(); - /** - * A project qualified repository - * @param projectId The project that the repository belongs to - * @param name Name of the repository - */ - record ProjectRepo(ProjectId projectId, String name) implements Writeable { - - public ProjectRepo(StreamInput in) throws IOException { - this(ProjectId.readFrom(in), in.readString()); - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - projectId.writeTo(out); - out.writeString(name); - } - - @Override - public String toString() { - return projectRepoString(projectId, name); - } - } - - static ProjectRepo projectRepo(ProjectId projectId, String repositoryName) { - return new ProjectRepo(projectId, repositoryName); - } - - static String projectRepoString(ProjectId projectId, String repositoryName) { - return "[" + projectId + "][" + repositoryName + "]"; - } - - DiffableUtils.KeySerializer PROJECT_REPO_SERIALIZER = new DiffableUtils.KeySerializer<>() { - @Override - public void writeKey(ProjectRepo key, StreamOutput out) throws IOException { - key.writeTo(out); - } - - @Override - public ProjectRepo readKey(StreamInput in) throws IOException { - return new ProjectRepo(in); - } - }; } diff --git a/server/src/main/java/org/elasticsearch/repositories/UnknownTypeRepository.java b/server/src/main/java/org/elasticsearch/repositories/UnknownTypeRepository.java index 13d81d2d0d484..9750666c8c8a9 100644 --- a/server/src/main/java/org/elasticsearch/repositories/UnknownTypeRepository.java +++ b/server/src/main/java/org/elasticsearch/repositories/UnknownTypeRepository.java @@ -74,7 +74,7 @@ public void getSnapshotInfo( } @Override - public Metadata getSnapshotGlobalMetadata(SnapshotId snapshotId) { + public Metadata getSnapshotGlobalMetadata(SnapshotId snapshotId, boolean fromProjectMetadata) { throw createUnknownTypeException(); } diff --git a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java index cb3fa6f575c8c..44314e924bd6a 100644 --- a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java +++ b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java @@ -371,6 +371,15 @@ public static String getRepositoryDataBlobName(long repositoryGeneration) { ChunkedToXContent::wrapAsToXContent ); + public static final ChecksumBlobStoreFormat PROJECT_METADATA_FORMAT = new ChecksumBlobStoreFormat<>( + "project-metadata", + METADATA_NAME_FORMAT, + (repoName, parser) -> ProjectMetadata.Builder.fromXContent(parser), + projectMetadata -> ChunkedToXContent.wrapAsToXContent( + params -> Iterators.concat(Iterators.single((builder, ignored) -> builder.field("id", projectMetadata.id()))) + ) + ); + public static final ChecksumBlobStoreFormat INDEX_METADATA_FORMAT = new ChecksumBlobStoreFormat<>( "index-metadata", METADATA_NAME_FORMAT, @@ -1282,7 +1291,7 @@ private void determineShardCount(ActionListener listener) { private void getOneShardCount(String indexMetaGeneration) { try { updateShardCount( - INDEX_METADATA_FORMAT.read(metadata.name(), indexContainer, indexMetaGeneration, namedXContentRegistry) + INDEX_METADATA_FORMAT.read(getProjectRepo(), indexContainer, indexMetaGeneration, namedXContentRegistry) .getNumberOfShards() ); } catch (Exception ex) { @@ -1749,6 +1758,8 @@ int sizeInBytes() { @Override public void finalizeSnapshot(final FinalizeSnapshotContext finalizeSnapshotContext) { assert ThreadPool.assertCurrentThreadPool(ThreadPool.Names.SNAPSHOT); + assert finalizeSnapshotContext.snapshotInfo().projectId().equals(getProjectId()) + : "project-id mismatch: " + finalizeSnapshotContext.snapshotInfo().projectId() + " != " + getProjectId(); final long repositoryStateId = finalizeSnapshotContext.repositoryStateId(); final SnapshotInfo snapshotInfo = finalizeSnapshotContext.snapshotInfo(); assert repositoryStateId > RepositoryData.UNKNOWN_REPO_GEN @@ -1813,17 +1824,19 @@ record RootBlobUpdateResult(RepositoryData oldRepositoryData, RepositoryData new // Write global metadata final Metadata clusterMetadata = finalizeSnapshotContext.clusterMetadata(); - executor.execute( - ActionRunnable.run( - allMetaListeners.acquire(), - () -> GLOBAL_METADATA_FORMAT.write(clusterMetadata, blobContainer(), snapshotId.getUUID(), compress) - ) - ); + final var projectMetadata = clusterMetadata.getProject(getProjectId()); + executor.execute(ActionRunnable.run(allMetaListeners.acquire(), () -> { + if (finalizeSnapshotContext.serializeProjectMetadata()) { + PROJECT_METADATA_FORMAT.write(projectMetadata, blobContainer(), snapshotId.getUUID(), compress); + } else { + GLOBAL_METADATA_FORMAT.write(clusterMetadata, blobContainer(), snapshotId.getUUID(), compress); + } + })); // Write the index metadata for each index in the snapshot for (IndexId index : indices) { executor.execute(ActionRunnable.run(allMetaListeners.acquire(), () -> { - final IndexMetadata indexMetaData = clusterMetadata.getProject(getProjectId()).index(index.getName()); + final IndexMetadata indexMetaData = projectMetadata.index(index.getName()); if (writeIndexGens) { final String identifiers = IndexMetaDataGenerations.buildUniqueIdentifier(indexMetaData); String metaUUID = existingRepositoryData.indexMetaDataGenerations().getIndexMetaBlobId(identifiers); @@ -2014,7 +2027,7 @@ private void getOneSnapshotInfo(BlockingQueue queue, GetSnapshotInfo Exception failure = null; SnapshotInfo snapshotInfo = null; try { - snapshotInfo = SNAPSHOT_FORMAT.read(metadata.name(), blobContainer(), snapshotId.getUUID(), namedXContentRegistry); + snapshotInfo = SNAPSHOT_FORMAT.read(getProjectRepo(), blobContainer(), snapshotId.getUUID(), namedXContentRegistry); } catch (NoSuchFileException ex) { failure = new SnapshotMissingException(metadata.name(), snapshotId, ex); } catch (IOException | NotXContentException ex) { @@ -2038,9 +2051,19 @@ private void getOneSnapshotInfo(BlockingQueue queue, GetSnapshotInfo } @Override - public Metadata getSnapshotGlobalMetadata(final SnapshotId snapshotId) { + public Metadata getSnapshotGlobalMetadata(final SnapshotId snapshotId, boolean fromProjectMetadata) { try { - return GLOBAL_METADATA_FORMAT.read(metadata.name(), blobContainer(), snapshotId.getUUID(), namedXContentRegistry); + if (fromProjectMetadata) { + final var projectMetadata = PROJECT_METADATA_FORMAT.read( + getProjectRepo(), + blobContainer(), + snapshotId.getUUID(), + namedXContentRegistry + ); + return Metadata.builder().put(projectMetadata).build(); + } else { + return GLOBAL_METADATA_FORMAT.read(getProjectRepo(), blobContainer(), snapshotId.getUUID(), namedXContentRegistry); + } } catch (NoSuchFileException ex) { throw new SnapshotMissingException(metadata.name(), snapshotId, ex); } catch (IOException ex) { @@ -2052,7 +2075,7 @@ public Metadata getSnapshotGlobalMetadata(final SnapshotId snapshotId) { public IndexMetadata getSnapshotIndexMetaData(RepositoryData repositoryData, SnapshotId snapshotId, IndexId index) throws IOException { try { return INDEX_METADATA_FORMAT.read( - metadata.name(), + getProjectRepo(), indexContainer(index), repositoryData.indexMetaDataGenerations().indexMetaBlobId(snapshotId, index), namedXContentRegistry @@ -3897,7 +3920,7 @@ private void writeShardIndexBlobAtomic( */ public BlobStoreIndexShardSnapshot loadShardSnapshot(BlobContainer shardContainer, SnapshotId snapshotId) { try { - return INDEX_SHARD_SNAPSHOT_FORMAT.read(metadata.name(), shardContainer, snapshotId.getUUID(), namedXContentRegistry); + return INDEX_SHARD_SNAPSHOT_FORMAT.read(getProjectRepo(), shardContainer, snapshotId.getUUID(), namedXContentRegistry); } catch (NoSuchFileException ex) { throw new SnapshotMissingException(metadata.name(), snapshotId, ex); } catch (IOException ex) { @@ -3952,7 +3975,7 @@ private Tuple buildBlobStoreIndex try { return new Tuple<>( INDEX_SHARD_SNAPSHOTS_FORMAT.read( - metadata.name(), + getProjectRepo(), shardContainer, generation.getGenerationUUID(), namedXContentRegistry @@ -4008,7 +4031,7 @@ private Tuple buildBlobStoreIndex && shardSnapshotBlobName.endsWith(METADATA_BLOB_NAME_SUFFIX) && shardSnapshotBlobName.length() == shardSnapshotBlobNameLength) { final var shardSnapshot = INDEX_SHARD_SNAPSHOT_FORMAT.read( - metadata.name(), + getProjectRepo(), shardContainer, shardSnapshotBlobName.substring(SNAPSHOT_PREFIX.length(), shardSnapshotBlobNameLengthBeforeExt), namedXContentRegistry @@ -4066,7 +4089,7 @@ private Tuple buildBlobStoreIndexShardSnapsh long latest = latestGeneration(blobs); if (latest >= 0) { final BlobStoreIndexShardSnapshots shardSnapshots = INDEX_SHARD_SNAPSHOTS_FORMAT.read( - metadata.name(), + getProjectRepo(), shardContainer, Long.toString(latest), namedXContentRegistry diff --git a/server/src/main/java/org/elasticsearch/repositories/blobstore/ChecksumBlobStoreFormat.java b/server/src/main/java/org/elasticsearch/repositories/blobstore/ChecksumBlobStoreFormat.java index bae5ae0fdadf0..60a5f6c09d3e4 100644 --- a/server/src/main/java/org/elasticsearch/repositories/blobstore/ChecksumBlobStoreFormat.java +++ b/server/src/main/java/org/elasticsearch/repositories/blobstore/ChecksumBlobStoreFormat.java @@ -30,6 +30,7 @@ import org.elasticsearch.common.xcontent.XContentParserUtils; import org.elasticsearch.core.Nullable; import org.elasticsearch.gateway.CorruptStateException; +import org.elasticsearch.repositories.ProjectRepo; import org.elasticsearch.xcontent.NamedXContentRegistry; import org.elasticsearch.xcontent.ToXContent; import org.elasticsearch.xcontent.XContentBuilder; @@ -69,9 +70,9 @@ public final class ChecksumBlobStoreFormat { private final String blobNameFormat; - private final CheckedBiFunction reader; + private final CheckedBiFunction reader; - private final CheckedBiFunction fallbackReader; + private final CheckedBiFunction fallbackReader; private final Function writer; @@ -85,8 +86,8 @@ public final class ChecksumBlobStoreFormat { public ChecksumBlobStoreFormat( String codec, String blobNameFormat, - CheckedBiFunction reader, - @Nullable CheckedBiFunction fallbackReader, + CheckedBiFunction reader, + @Nullable CheckedBiFunction fallbackReader, Function writer ) { this.reader = reader; @@ -105,7 +106,7 @@ public ChecksumBlobStoreFormat( public ChecksumBlobStoreFormat( String codec, String blobNameFormat, - CheckedBiFunction reader, + CheckedBiFunction reader, Function writer ) { this(codec, blobNameFormat, reader, null, writer); @@ -118,11 +119,11 @@ public ChecksumBlobStoreFormat( * @param name name to be translated into * @return parsed blob object */ - public T read(String repoName, BlobContainer blobContainer, String name, NamedXContentRegistry namedXContentRegistry) + public T read(ProjectRepo projectRepo, BlobContainer blobContainer, String name, NamedXContentRegistry namedXContentRegistry) throws IOException { String blobName = blobName(name); try (InputStream in = blobContainer.readBlob(OperationPurpose.SNAPSHOT_METADATA, blobName)) { - return deserialize(repoName, namedXContentRegistry, in); + return deserialize(projectRepo, namedXContentRegistry, in); } } @@ -130,7 +131,7 @@ public String blobName(String name) { return String.format(Locale.ROOT, blobNameFormat, name); } - public T deserialize(String repoName, NamedXContentRegistry namedXContentRegistry, InputStream input) throws IOException { + public T deserialize(ProjectRepo projectRepo, NamedXContentRegistry namedXContentRegistry, InputStream input) throws IOException { final DeserializeMetaBlobInputStream deserializeMetaBlobInputStream = new DeserializeMetaBlobInputStream(input); try { CodecUtil.checkHeader(new InputStreamDataInput(deserializeMetaBlobInputStream), codec, VERSION, VERSION); @@ -154,7 +155,7 @@ public T deserialize(String repoName, NamedXContentRegistry namedXContentRegistr XContentType.SMILE ) ) { - result = reader.apply(repoName, parser); + result = reader.apply(projectRepo, parser); XContentParserUtils.ensureExpectedToken(null, parser.nextToken(), parser); } catch (Exception e) { try ( @@ -165,7 +166,7 @@ public T deserialize(String repoName, NamedXContentRegistry namedXContentRegistr XContentType.SMILE ) ) { - result = fallbackReader.apply(repoName, parser); + result = fallbackReader.apply(projectRepo, parser); XContentParserUtils.ensureExpectedToken(null, parser.nextToken(), parser); } } @@ -174,7 +175,7 @@ public T deserialize(String repoName, NamedXContentRegistry namedXContentRegistr XContentParser parser = XContentType.SMILE.xContent() .createParser(namedXContentRegistry, LoggingDeprecationHandler.INSTANCE, wrappedStream) ) { - result = reader.apply(repoName, parser); + result = reader.apply(projectRepo, parser); XContentParserUtils.ensureExpectedToken(null, parser.nextToken(), parser); } deserializeMetaBlobInputStream.verifyFooter(); diff --git a/server/src/main/java/org/elasticsearch/snapshots/InFlightShardSnapshotStates.java b/server/src/main/java/org/elasticsearch/snapshots/InFlightShardSnapshotStates.java index 89ebf13b58c95..94fc6dfad06d8 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/InFlightShardSnapshotStates.java +++ b/server/src/main/java/org/elasticsearch/snapshots/InFlightShardSnapshotStates.java @@ -12,7 +12,7 @@ import org.elasticsearch.cluster.SnapshotsInProgress; import org.elasticsearch.core.Nullable; import org.elasticsearch.repositories.IndexId; -import org.elasticsearch.repositories.RepositoryOperation.ProjectRepo; +import org.elasticsearch.repositories.ProjectRepo; import org.elasticsearch.repositories.RepositoryShardId; import org.elasticsearch.repositories.ShardGeneration; import org.elasticsearch.repositories.ShardGenerations; diff --git a/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java b/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java index f57db79d4625d..1680bfa3a59a0 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java @@ -359,7 +359,7 @@ private void startRestore( Metadata globalMetadata = null; final Metadata.Builder metadataBuilder; if (request.includeGlobalState()) { - globalMetadata = repository.getSnapshotGlobalMetadata(snapshotId); + globalMetadata = repository.getSnapshotGlobalMetadata(snapshotId, false); metadataBuilder = Metadata.builder(globalMetadata); } else { metadataBuilder = Metadata.builder(); @@ -652,7 +652,7 @@ private static Tuple, Map> getD dataStreamAliases = Map.of(); } else { if (globalMetadata == null) { - globalMetadata = repository.getSnapshotGlobalMetadata(snapshotId); + globalMetadata = repository.getSnapshotGlobalMetadata(snapshotId, false); } final Map dataStreamsInSnapshot = globalMetadata.getProject().dataStreams(); allDataStreams = Maps.newMapWithExpectedSize(requestedDataStreams.size()); diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java index 2bc75665da2a5..7d34373494150 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java @@ -22,6 +22,7 @@ import org.elasticsearch.core.Nullable; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.IndexVersion; +import org.elasticsearch.repositories.ProjectRepo; import org.elasticsearch.repositories.RepositoryShardId; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.xcontent.ConstructingObjectParser; @@ -704,7 +705,7 @@ public XContentBuilder toXContent(final XContentBuilder builder, final ToXConten * handle x-content written with the external version as external x-content * is only for display purposes and does not need to be parsed. */ - public static SnapshotInfo fromXContentInternal(final String repoName, final XContentParser parser) throws IOException { + public static SnapshotInfo fromXContentInternal(final ProjectRepo projectRepo, final XContentParser parser) throws IOException { String name = null; String uuid = null; IndexVersion version = IndexVersion.current(); @@ -799,7 +800,7 @@ public static SnapshotInfo fromXContentInternal(final String repoName, final XCo uuid = name; } return new SnapshotInfo( - new Snapshot(repoName, new SnapshotId(name, uuid)), + new Snapshot(projectRepo.projectId(), projectRepo.name(), new SnapshotId(name, uuid)), indices, dataStreams, featureStates, diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java index 396287af20208..f7901607fe724 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java @@ -77,7 +77,6 @@ import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException; import org.elasticsearch.common.util.concurrent.ListenableFuture; -import org.elasticsearch.core.FixForMultiProject; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.Predicates; import org.elasticsearch.core.SuppressForbidden; @@ -92,12 +91,12 @@ import org.elasticsearch.repositories.FinalizeSnapshotContext; import org.elasticsearch.repositories.FinalizeSnapshotContext.UpdatedShardGenerations; import org.elasticsearch.repositories.IndexId; +import org.elasticsearch.repositories.ProjectRepo; import org.elasticsearch.repositories.RepositoriesService; import org.elasticsearch.repositories.Repository; import org.elasticsearch.repositories.RepositoryData; import org.elasticsearch.repositories.RepositoryException; import org.elasticsearch.repositories.RepositoryMissingException; -import org.elasticsearch.repositories.RepositoryOperation.ProjectRepo; import org.elasticsearch.repositories.RepositoryShardId; import org.elasticsearch.repositories.ShardGeneration; import org.elasticsearch.repositories.ShardGenerations; @@ -137,6 +136,7 @@ import static org.elasticsearch.cluster.SnapshotsInProgress.completed; import static org.elasticsearch.common.Strings.arrayToCommaDelimitedString; import static org.elasticsearch.core.Strings.format; +import static org.elasticsearch.repositories.ProjectRepo.projectRepoString; /** * Service responsible for creating snapshots. This service runs all the steps executed on the master node during snapshot creation and @@ -198,6 +198,8 @@ public final class SnapshotsService extends AbstractLifecycleComponent implement private final SystemIndices systemIndices; + private final boolean supportsMultipleProjects; + private final MasterServiceTaskQueue masterServiceTaskQueue; private final ShardSnapshotUpdateCompletionHandler shardSnapshotUpdateCompletionHandler; @@ -225,7 +227,8 @@ public SnapshotsService( RepositoriesService repositoriesService, TransportService transportService, ActionFilters actionFilters, - SystemIndices systemIndices + SystemIndices systemIndices, + boolean supportsMultipleProjects ) { this.clusterService = clusterService; this.rerouteService = rerouteService; @@ -244,6 +247,7 @@ public SnapshotsService( .addSettingsUpdateConsumer(MAX_CONCURRENT_SNAPSHOT_OPERATIONS_SETTING, i -> maxConcurrentOperations = i); } this.systemIndices = systemIndices; + this.supportsMultipleProjects = supportsMultipleProjects; this.masterServiceTaskQueue = clusterService.createTaskQueue("snapshots-service", Priority.NORMAL, new SnapshotTaskExecutor()); this.updateNodeIdsToRemoveQueue = clusterService.createTaskQueue( @@ -434,7 +438,7 @@ public void clusterStateProcessed(ClusterState oldState, final ClusterState newS /** * Checks the cluster state for any in-progress repository cleanup tasks ({@link RepositoryCleanupInProgress}). - * Repository cleanup is intentionally cluster wide exclusive + * Note that repository cleanup is intentionally cluster wide exclusive. */ private static void ensureNoCleanupInProgress( final ClusterState currentState, @@ -787,7 +791,7 @@ private static UpdatedShardGenerations buildGenerations(SnapshotsInProgress.Entr RepositoryShardId key = entry.getKey(); ShardSnapshotStatus value = entry.getValue(); final Index index = snapshot.indexByName(key.indexName()); - if (metadata.findIndex(index).isEmpty()) { + if (metadata.getProject(snapshot.projectId()).hasIndex(index) == false) { assert snapshot.partial() : "Index [" + index + "] was deleted during a snapshot but snapshot was not partial."; if (deletedBuilder == null) { deletedBuilder = ShardGenerations.builder(); @@ -801,11 +805,13 @@ private static UpdatedShardGenerations buildGenerations(SnapshotsInProgress.Entr return new UpdatedShardGenerations(builder.build(), deletedBuilder == null ? ShardGenerations.EMPTY : deletedBuilder.build()); } - private static Metadata metadataForSnapshot(SnapshotsInProgress.Entry snapshot, Metadata metadata, ProjectId projectId) { + private Metadata metadataForSnapshot(SnapshotsInProgress.Entry snapshot, Metadata metadata, ProjectId projectId) { final ProjectMetadata snapshotProject = projectForSnapshot(snapshot, metadata.getProject(projectId)); final Metadata.Builder builder; - if (snapshot.includeGlobalState() == false) { - // Remove global state from the cluster state + if (snapshot.includeGlobalState() == false || supportsMultipleProjects) { + // Remove global state from the cluster state when + // 1. The request explicitly demands that + // 2. The snapshot is for a project in a multi-project cluster. Such a snapshot must not include cluster level info builder = Metadata.builder(); } else { builder = Metadata.builder(metadata); @@ -871,7 +877,7 @@ public static List currentSnapshots( return Collections.emptyList(); } if ("_all".equals(repository)) { - return snapshotsInProgress.asStream().toList(); + return snapshotsInProgress.asStream(projectId).toList(); } if (snapshots.isEmpty()) { return snapshotsInProgress.forRepo(projectId, repository); @@ -1169,7 +1175,8 @@ public ClusterState execute(ClusterState currentState) { final ClusterState res = readyDeletions( updatedSnapshots != snapshotsInProgress ? ClusterState.builder(currentState).putCustom(SnapshotsInProgress.TYPE, updatedSnapshots).build() - : currentState + : currentState, + null ).v1(); for (SnapshotDeletionsInProgress.Entry delete : SnapshotDeletionsInProgress.get(res).getEntries()) { if (delete.state() == SnapshotDeletionsInProgress.State.STARTED) { @@ -1205,7 +1212,7 @@ public void clusterStateProcessed(ClusterState oldState, ClusterState newState) } } } - startExecutableClones(SnapshotsInProgress.get(newState), null); + startExecutableClones(SnapshotsInProgress.get(newState)); // run newly ready deletes for (SnapshotDeletionsInProgress.Entry entry : deletionsToExecute) { if (tryEnterRepoLoop(entry.projectId(), entry.repository())) { @@ -1469,7 +1476,16 @@ private void leaveRepoLoop(ProjectId projectId, String repository) { } private void finalizeSnapshotEntry(Snapshot snapshot, Metadata metadata, RepositoryData repositoryData) { - threadPool.executor(ThreadPool.Names.SNAPSHOT).execute(new SnapshotFinalization(snapshot, metadata, repositoryData)); + final ProjectId projectId = snapshot.getProjectId(); + final Metadata effectiveMetadata; + if (supportsMultipleProjects) { + assert ProjectId.DEFAULT.equals(projectId) == false : "should not snapshot for default project when multi-project is enabled"; + effectiveMetadata = Metadata.builder().put(metadata.getProject(projectId)).build(); + } else { + assert ProjectId.DEFAULT.equals(projectId) : "expect snapshots for only the default project when multi-project is disabled"; + effectiveMetadata = metadata; + } + threadPool.executor(ThreadPool.Names.SNAPSHOT).execute(new SnapshotFinalization(snapshot, effectiveMetadata, repositoryData)); } /** @@ -1482,7 +1498,6 @@ private void finalizeSnapshotEntry(Snapshot snapshot, Metadata metadata, Reposit private class SnapshotFinalization extends AbstractRunnable { private final Snapshot snapshot; - @FixForMultiProject(description = "replace with ProjectMetadata") private final Metadata metadata; private final RepositoryData repositoryData; @@ -1528,7 +1543,7 @@ protected void doRun() { final Repository repo = repositoriesService.repository(projectId, repository); if (entry.isClone()) { threadPool.executor(ThreadPool.Names.SNAPSHOT).execute(ActionRunnable.supply(metadataListener, () -> { - final Metadata existingMetadata = repo.getSnapshotGlobalMetadata(entry.source()); + final Metadata existingMetadata = repo.getSnapshotGlobalMetadata(entry.source(), supportsMultipleProjects); final ProjectMetadata existingProject = existingMetadata.getProject(projectId); final ProjectMetadata.Builder projBuilder = ProjectMetadata.builder(existingProject); final Set existingIndices = new HashSet<>(); @@ -1610,6 +1625,7 @@ protected void doRun() { final ListenableFuture>> snapshotListeners = new ListenableFuture<>(); repo.finalizeSnapshot( new FinalizeSnapshotContext( + supportsMultipleProjects, updatedShardGenerations, repositoryData.getGenId(), metaForSnapshot, @@ -1781,7 +1797,7 @@ private void runReadyDeletions(RepositoryData repositoryData, ProjectId projectI @Override public ClusterState execute(ClusterState currentState) { - assert readyDeletions(currentState).v1() == currentState + assert readyDeletions(currentState, projectId).v1() == currentState : "Deletes should have been set to ready by finished snapshot deletes and finalizations"; for (SnapshotDeletionsInProgress.Entry entry : SnapshotDeletionsInProgress.get(currentState).getEntries()) { if (entry.projectId().equals(projectId) @@ -1818,12 +1834,16 @@ public void clusterStateProcessed(ClusterState oldState, ClusterState newState) * {@link SnapshotDeletionsInProgress.State#STARTED} or waiting entries in state {@link SnapshotDeletionsInProgress.State#WAITING} * that were moved to {@link SnapshotDeletionsInProgress.State#STARTED} in the returned updated cluster state. * + * @param projectId the project for repositories where deletions should be prepared. {@code null} means all projects * @param currentState current cluster state * @return tuple of an updated cluster state and currently executable snapshot delete operations */ - private static Tuple> readyDeletions(ClusterState currentState) { + private static Tuple> readyDeletions( + ClusterState currentState, + @Nullable ProjectId projectId + ) { final SnapshotDeletionsInProgress deletions = SnapshotDeletionsInProgress.get(currentState); - if (deletions.hasDeletionsInProgress() == false) { + if (deletions.hasDeletionsInProgress() == false || (projectId != null && deletions.hasDeletionsInProgress(projectId) == false)) { return Tuple.tuple(currentState, List.of()); } final SnapshotsInProgress snapshotsInProgress = currentState.custom(SnapshotsInProgress.TYPE); @@ -1833,6 +1853,11 @@ private static Tuple> read final ArrayList readyDeletions = new ArrayList<>(); final List newDeletes = new ArrayList<>(); for (SnapshotDeletionsInProgress.Entry entry : deletions.getEntries()) { + if (projectId != null && projectId.equals(entry.projectId()) == false) { + // Not the target project, keep the entry as is + newDeletes.add(entry); + continue; + } final var projectRepo = new ProjectRepo(entry.projectId(), entry.repository()); if (repositoriesSeen.add(projectRepo) && entry.state() == SnapshotDeletionsInProgress.State.WAITING @@ -1877,7 +1902,9 @@ public static ClusterState stateWithoutSnapshot( ClusterState result = state; int indexOfEntry = -1; // Find the in-progress snapshot entry that matches {@code snapshot}. - final List entryList = inProgressSnapshots.forRepo(snapshot.getProjectId(), snapshot.getRepository()); + final ProjectId projectId = snapshot.getProjectId(); + final String repository = snapshot.getRepository(); + final List entryList = inProgressSnapshots.forRepo(projectId, repository); for (int i = 0; i < entryList.size(); i++) { SnapshotsInProgress.Entry entry = entryList.get(i); if (entry.snapshot().equals(snapshot)) { @@ -1976,15 +2003,11 @@ public static ClusterState stateWithoutSnapshot( result = ClusterState.builder(state) .putCustom( SnapshotsInProgress.TYPE, - inProgressSnapshots.createCopyWithUpdatedEntriesForRepo( - snapshot.getProjectId(), - snapshot.getRepository(), - updatedEntries - ) + inProgressSnapshots.createCopyWithUpdatedEntriesForRepo(projectId, repository, updatedEntries) ) .build(); } - return readyDeletions(result).v1(); + return readyDeletions(result, projectId).v1(); } private static void addSnapshotEntry( @@ -2218,9 +2241,8 @@ public ClusterState execute(ClusterState currentState) { return currentState; } - final Set activeCloneSources = snapshotsInProgress.asStream() + final Set activeCloneSources = snapshotsInProgress.asStream(projectId) .filter(SnapshotsInProgress.Entry::isClone) - .filter(entry -> entry.projectId().equals(projectId)) .map(SnapshotsInProgress.Entry::source) .collect(Collectors.toSet()); for (SnapshotId snapshotId : snapshotIds) { @@ -2340,7 +2362,11 @@ public void onFailure(Exception e) { @Override public void clusterStateProcessed(ClusterState oldState, ClusterState newState) { logger.info( - () -> format("deleting snapshots [%s] from repository [%s]", arrayToCommaDelimitedString(snapshotNames), repositoryName) + () -> format( + "deleting snapshots [%s] from repository %s", + arrayToCommaDelimitedString(snapshotNames), + projectRepoString(projectId, repositoryName) + ) ); if (completedNoCleanup.isEmpty() == false) { @@ -2666,7 +2692,8 @@ protected void handleListeners(List> deleteListeners) { final var collector = new Strings.BoundedDelimitedStringCollector(sb, ",", 1024); snapshotIds.forEach(collector::appendItem); collector.finish(); - sb.append("] deleted"); + sb.append("] deleted in repository ["); + sb.append(projectRepoString(deleteEntry.projectId(), deleteEntry.repository())); return sb; }); doneFuture.onResponse(null); @@ -2785,7 +2812,8 @@ public ClusterState execute(ClusterState currentState) { } final SnapshotDeletionsInProgress newDeletions = filterDeletions(updatedDeletions); final Tuple> res = readyDeletions( - updateWithSnapshots(currentState, updatedSnapshotsInProgress(currentState, newDeletions), newDeletions) + updateWithSnapshots(currentState, updatedSnapshotsInProgress(currentState, newDeletions), newDeletions), + deleteEntry.projectId() ); readyDeletions = res.v2(); return res.v1(); @@ -2834,7 +2862,7 @@ public final void clusterStateProcessed(ClusterState oldState, ClusterState newS } // TODO: be more efficient here, we could collect newly ready shard clones as we compute them and then directly start them // instead of looping over all possible clones to execute - startExecutableClones(SnapshotsInProgress.get(newState), null); + startExecutableClones(SnapshotsInProgress.get(newState), deleteEntry.projectId()); } /** @@ -3162,13 +3190,10 @@ private static ShardSnapshotStatus initShardSnapshotStatus( * Returns the data streams that are currently being snapshotted (with partial == false) and that are contained in the * indices-to-check set. */ - @FixForMultiProject public static Set snapshottingDataStreams(final ProjectState projectState, final Set dataStreamsToCheck) { - // TODO multi-project: this will behave incorrectly when there are data streams with equal names in different projects that are - // being snapshotted at the same time. Map dataStreams = projectState.metadata().dataStreams(); return SnapshotsInProgress.get(projectState.cluster()) - .asStream() + .asStream(projectState.projectId()) .filter(e -> e.partial() == false) .flatMap(e -> e.dataStreams().stream()) .filter(ds -> dataStreams.containsKey(ds) && dataStreamsToCheck.contains(ds)) @@ -3180,7 +3205,8 @@ public static Set snapshottingDataStreams(final ProjectState projectStat */ public static Set snapshottingIndices(final ProjectState projectState, final Set indicesToCheck) { final Set indices = new HashSet<>(); - for (List snapshotsInRepo : SnapshotsInProgress.get(projectState.cluster()).entriesByRepo()) { + for (List snapshotsInRepo : SnapshotsInProgress.get(projectState.cluster()) + .entriesByRepo(projectState.projectId())) { for (final SnapshotsInProgress.Entry entry : snapshotsInRepo) { if (entry.partial() == false && entry.isClone() == false) { for (String indexName : entry.indices().keySet()) { @@ -3826,16 +3852,31 @@ private void innerUpdateSnapshotState( masterServiceTaskQueue.submitTask("update snapshot state", update, null); } - private void startExecutableClones(SnapshotsInProgress snapshotsInProgress, @Nullable ProjectRepo projectRepo) { - if (projectRepo == null) { - for (List entries : snapshotsInProgress.entriesByRepo()) { - startExecutableClones(entries); - } - } else { - startExecutableClones(snapshotsInProgress.forRepo(projectRepo)); + /** + * Maybe kick off new shard clone operations for all repositories from all projects + */ + private void startExecutableClones(SnapshotsInProgress snapshotsInProgress) { + for (List entries : snapshotsInProgress.entriesByRepo()) { + startExecutableClones(entries); + } + } + + /** + * Maybe kick off new shard clone operations for all repositories of the specified project + */ + private void startExecutableClones(SnapshotsInProgress snapshotsInProgress, ProjectId projectId) { + for (List entries : snapshotsInProgress.entriesByRepo(projectId)) { + startExecutableClones(entries); } } + /** + * Maybe kick off new shard clone operations for the single specified project repository + */ + private void startExecutableClones(SnapshotsInProgress snapshotsInProgress, ProjectRepo projectRepo) { + startExecutableClones(snapshotsInProgress.forRepo(Objects.requireNonNull(projectRepo))); + } + private void startExecutableClones(List entries) { for (SnapshotsInProgress.Entry entry : entries) { if (entry.isClone() && entry.state() == SnapshotsInProgress.State.STARTED) { @@ -3955,7 +3996,10 @@ public ClusterState execute(ClusterState currentState) { @Override public void onFailure(Exception e) { - logger.info(() -> "Failed to remove all snapshot tasks for repo [" + repository + "] from cluster state", e); + logger.info( + () -> "Failed to remove all snapshot tasks for repo [" + projectRepoString(projectId, repository) + "] from cluster state", + e + ); failAllListenersOnMasterFailOver(e); } @@ -4370,7 +4414,7 @@ private SnapshotsInProgress createSnapshot( request.partial(), indexIds, CollectionUtils.concatLists( - indexNameExpressionResolver.dataStreamNames(currentState, request.indicesOptions(), request.indices()), + indexNameExpressionResolver.dataStreamNames(projectState.metadata(), request.indicesOptions(), request.indices()), systemDataStreamNames ), threadPool.absoluteTimeInMillis(), diff --git a/server/src/test/java/org/elasticsearch/repositories/InvalidRepositoryTests.java b/server/src/test/java/org/elasticsearch/repositories/InvalidRepositoryTests.java index 7a82ec02c956d..8135b0ce3b799 100644 --- a/server/src/test/java/org/elasticsearch/repositories/InvalidRepositoryTests.java +++ b/server/src/test/java/org/elasticsearch/repositories/InvalidRepositoryTests.java @@ -32,7 +32,7 @@ public void testShouldThrowWhenGettingMetadata() { assertThat(repository.getProjectId(), equalTo(projectId)); final var expectedException = expectThrows( RepositoryException.class, - () -> repository.getSnapshotGlobalMetadata(new SnapshotId("name", "uuid")) + () -> repository.getSnapshotGlobalMetadata(new SnapshotId("name", "uuid"), false) ); assertThat(expectedException.getMessage(), equalTo("[name] repository type [type] failed to create on current node")); assertThat(expectedException.getCause(), isA(RepositoryException.class)); diff --git a/server/src/test/java/org/elasticsearch/repositories/RepositoriesServiceTests.java b/server/src/test/java/org/elasticsearch/repositories/RepositoriesServiceTests.java index 96d601f9091ff..da6d6ff78a989 100644 --- a/server/src/test/java/org/elasticsearch/repositories/RepositoriesServiceTests.java +++ b/server/src/test/java/org/elasticsearch/repositories/RepositoriesServiceTests.java @@ -670,7 +670,7 @@ public void getSnapshotInfo( } @Override - public Metadata getSnapshotGlobalMetadata(SnapshotId snapshotId) { + public Metadata getSnapshotGlobalMetadata(SnapshotId snapshotId, boolean fromProjectMetadata) { return null; } diff --git a/server/src/test/java/org/elasticsearch/repositories/UnknownTypeRepositoryTests.java b/server/src/test/java/org/elasticsearch/repositories/UnknownTypeRepositoryTests.java index b7225546fd82e..2a11cdba7099c 100644 --- a/server/src/test/java/org/elasticsearch/repositories/UnknownTypeRepositoryTests.java +++ b/server/src/test/java/org/elasticsearch/repositories/UnknownTypeRepositoryTests.java @@ -24,7 +24,7 @@ public class UnknownTypeRepositoryTests extends ESTestCase { public void testShouldThrowWhenGettingMetadata() { assertThat(repository.getProjectId(), equalTo(projectId)); - expectThrows(RepositoryException.class, () -> repository.getSnapshotGlobalMetadata(new SnapshotId("name", "uuid"))); + expectThrows(RepositoryException.class, () -> repository.getSnapshotGlobalMetadata(new SnapshotId("name", "uuid"), false)); } public void testShouldNotThrowWhenApplyingLifecycleChanges() { diff --git a/server/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryRestoreTests.java b/server/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryRestoreTests.java index 6c20831969738..12742fb9b56b7 100644 --- a/server/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryRestoreTests.java +++ b/server/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryRestoreTests.java @@ -180,6 +180,7 @@ public void testSnapshotWithConflictingName() throws Exception { final RepositoryData ignoredRepositoryData = safeAwait( listener -> repository.finalizeSnapshot( new FinalizeSnapshotContext( + false, snapshotShardGenerations, RepositoryData.EMPTY_REPO_GEN, Metadata.builder().put(shard.indexSettings().getIndexMetadata(), false).build(), diff --git a/server/src/test/java/org/elasticsearch/snapshots/BlobStoreFormatTests.java b/server/src/test/java/org/elasticsearch/snapshots/BlobStoreFormatTests.java index b779f626753c3..1ffa3e9365801 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/BlobStoreFormatTests.java +++ b/server/src/test/java/org/elasticsearch/snapshots/BlobStoreFormatTests.java @@ -11,6 +11,7 @@ import org.elasticsearch.ElasticsearchCorruptionException; import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.common.blobstore.BlobContainer; import org.elasticsearch.common.blobstore.BlobPath; import org.elasticsearch.common.blobstore.BlobStore; @@ -19,6 +20,7 @@ import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.xcontent.XContentParserUtils; import org.elasticsearch.core.Streams; +import org.elasticsearch.repositories.ProjectRepo; import org.elasticsearch.repositories.blobstore.ChecksumBlobStoreFormat; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xcontent.ToXContent; @@ -95,8 +97,9 @@ public void testBlobStoreOperations() throws IOException { checksumSMILE.write(new BlobObj(compressedText), blobContainer, "check-smile-comp", true); // Assert that all checksum blobs can be read - assertEquals(normalText, checksumSMILE.read("repo", blobContainer, "check-smile", xContentRegistry()).getText()); - assertEquals(compressedText, checksumSMILE.read("repo", blobContainer, "check-smile-comp", xContentRegistry()).getText()); + final var projectRepo = new ProjectRepo(ProjectId.DEFAULT, "repo"); + assertEquals(normalText, checksumSMILE.read(projectRepo, blobContainer, "check-smile", xContentRegistry()).getText()); + assertEquals(compressedText, checksumSMILE.read(projectRepo, blobContainer, "check-smile-comp", xContentRegistry()).getText()); } public void testCompressionIsApplied() throws IOException { @@ -133,10 +136,11 @@ public void testBlobCorruption() throws IOException { Function.identity() ); checksumFormat.write(blobObj, blobContainer, "test-path", randomBoolean()); - assertEquals(checksumFormat.read("repo", blobContainer, "test-path", xContentRegistry()).getText(), testString); + final var projectRepo = new ProjectRepo(ProjectId.DEFAULT, "repo"); + assertEquals(checksumFormat.read(projectRepo, blobContainer, "test-path", xContentRegistry()).getText(), testString); randomCorruption(blobContainer, "test-path"); try { - checksumFormat.read("repo", blobContainer, "test-path", xContentRegistry()); + checksumFormat.read(projectRepo, blobContainer, "test-path", xContentRegistry()); fail("Should have failed due to corruption"); } catch (ElasticsearchCorruptionException | EOFException ex) { // expected exceptions from random byte corruption diff --git a/server/src/test/java/org/elasticsearch/snapshots/SnapshotInfoBlobSerializationTests.java b/server/src/test/java/org/elasticsearch/snapshots/SnapshotInfoBlobSerializationTests.java index c664e2a108a29..6dac8e49209c3 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/SnapshotInfoBlobSerializationTests.java +++ b/server/src/test/java/org/elasticsearch/snapshots/SnapshotInfoBlobSerializationTests.java @@ -11,6 +11,7 @@ import org.elasticsearch.TransportVersion; import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.repositories.ProjectRepo; import org.elasticsearch.repositories.blobstore.BlobStoreRepository; import org.elasticsearch.test.AbstractWireTestCase; import org.elasticsearch.xcontent.NamedXContentRegistry; @@ -34,7 +35,7 @@ protected SnapshotInfo copyInstance(SnapshotInfo instance, TransportVersion vers final BytesStreamOutput out = new BytesStreamOutput(); BlobStoreRepository.SNAPSHOT_FORMAT.serialize(instance, "test", randomBoolean(), out); return BlobStoreRepository.SNAPSHOT_FORMAT.deserialize( - instance.repository(), + new ProjectRepo(instance.projectId(), instance.repository()), NamedXContentRegistry.EMPTY, out.bytes().streamInput() ); diff --git a/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java b/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java index d3f0c1aa2bb42..ee99195766111 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java +++ b/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java @@ -2421,7 +2421,8 @@ public RecyclerBytesStreamOutput newNetworkBytesStream() { repositoriesService, transportService, actionFilters, - EmptySystemIndices.INSTANCE + EmptySystemIndices.INSTANCE, + false ); nodeEnv = new NodeEnvironment(settings, environment); final NamedXContentRegistry namedXContentRegistry = new NamedXContentRegistry(Collections.emptyList()); diff --git a/server/src/test/java/org/elasticsearch/snapshots/SnapshotsInProgressSerializationTests.java b/server/src/test/java/org/elasticsearch/snapshots/SnapshotsInProgressSerializationTests.java index 4e4c5adf5d758..79d880c5ef5e1 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/SnapshotsInProgressSerializationTests.java +++ b/server/src/test/java/org/elasticsearch/snapshots/SnapshotsInProgressSerializationTests.java @@ -35,7 +35,7 @@ import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.repositories.IndexId; -import org.elasticsearch.repositories.RepositoryOperation.ProjectRepo; +import org.elasticsearch.repositories.ProjectRepo; import org.elasticsearch.repositories.ShardGeneration; import org.elasticsearch.repositories.ShardSnapshotResult; import org.elasticsearch.test.AbstractChunkedSerializingTestCase; diff --git a/test/framework/src/main/java/org/elasticsearch/index/shard/RestoreOnlyRepository.java b/test/framework/src/main/java/org/elasticsearch/index/shard/RestoreOnlyRepository.java index 67dd587258f23..a3e8f6e84600f 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/shard/RestoreOnlyRepository.java +++ b/test/framework/src/main/java/org/elasticsearch/index/shard/RestoreOnlyRepository.java @@ -82,7 +82,7 @@ public void getSnapshotInfo( } @Override - public Metadata getSnapshotGlobalMetadata(SnapshotId snapshotId) { + public Metadata getSnapshotGlobalMetadata(SnapshotId snapshotId, boolean fromProjectMetadata) { return null; } diff --git a/test/framework/src/main/java/org/elasticsearch/snapshots/AbstractSnapshotIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/snapshots/AbstractSnapshotIntegTestCase.java index 63e77cfdc4523..ead163eaf26a4 100644 --- a/test/framework/src/main/java/org/elasticsearch/snapshots/AbstractSnapshotIntegTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/snapshots/AbstractSnapshotIntegTestCase.java @@ -22,6 +22,7 @@ import org.elasticsearch.cluster.ClusterStateUpdateTask; import org.elasticsearch.cluster.SnapshotDeletionsInProgress; import org.elasticsearch.cluster.SnapshotsInProgress; +import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.cluster.metadata.RepositoriesMetadata; import org.elasticsearch.cluster.metadata.RepositoryMetadata; import org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider; @@ -39,6 +40,7 @@ import org.elasticsearch.plugins.Plugin; import org.elasticsearch.repositories.FinalizeSnapshotContext; import org.elasticsearch.repositories.FinalizeSnapshotContext.UpdatedShardGenerations; +import org.elasticsearch.repositories.ProjectRepo; import org.elasticsearch.repositories.RepositoriesService; import org.elasticsearch.repositories.Repository; import org.elasticsearch.repositories.RepositoryData; @@ -423,7 +425,7 @@ protected String initWithSnapshotVersion(String repoName, Path repoPath, IndexVe .replace(IndexVersion.current().toString(), version.toString()) ) ) { - downgradedSnapshotInfo = SnapshotInfo.fromXContentInternal(repoName, parser); + downgradedSnapshotInfo = SnapshotInfo.fromXContentInternal(new ProjectRepo(ProjectId.DEFAULT, repoName), parser); } final BlobStoreRepository blobStoreRepository = getRepositoryOnMaster(repoName); BlobStoreRepository.SNAPSHOT_FORMAT.write( @@ -545,6 +547,7 @@ protected void addBwCFailedSnapshot(String repoName, String snapshotName, Map listener) -> repo.finalizeSnapshot( new FinalizeSnapshotContext( + false, UpdatedShardGenerations.EMPTY, getRepositoryData(repoName).getGenId(), state.metadata(), diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRepository.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRepository.java index bd2c925d52347..469f4c4d81998 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRepository.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRepository.java @@ -281,7 +281,7 @@ private Respons } @Override - public Metadata getSnapshotGlobalMetadata(SnapshotId snapshotId) { + public Metadata getSnapshotGlobalMetadata(SnapshotId snapshotId, boolean fromProjectMetadata) { assert SNAPSHOT_ID.equals(snapshotId) : "RemoteClusterRepository only supports " + SNAPSHOT_ID + " as the SnapshotId"; var remoteClient = getRemoteClusterClient(); ClusterStateResponse clusterState = executeRecoveryAction( diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/sourceonly/SourceOnlySnapshotRepository.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/sourceonly/SourceOnlySnapshotRepository.java index e6f3c3fa54277..c845df8501e45 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/sourceonly/SourceOnlySnapshotRepository.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/sourceonly/SourceOnlySnapshotRepository.java @@ -100,6 +100,7 @@ public void finalizeSnapshot(FinalizeSnapshotContext finalizeSnapshotContext) { // required engine, that the index is read-only and the mapping to a default mapping super.finalizeSnapshot( new FinalizeSnapshotContext( + finalizeSnapshotContext.serializeProjectMetadata(), finalizeSnapshotContext.updatedShardGenerations(), finalizeSnapshotContext.repositoryStateId(), metadataToSnapshot( diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/AsyncRetryDuringSnapshotActionStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/AsyncRetryDuringSnapshotActionStep.java index 94e44e7f9118f..702e3a88dc434 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/AsyncRetryDuringSnapshotActionStep.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/AsyncRetryDuringSnapshotActionStep.java @@ -145,7 +145,7 @@ public void onTimeout(TimeValue timeout) { // The index has since been deleted, mission accomplished! return true; } - for (List snapshots : SnapshotsInProgress.get(state).entriesByRepo()) { + for (List snapshots : SnapshotsInProgress.get(state).entriesByRepo(projectId)) { for (SnapshotsInProgress.Entry snapshot : snapshots) { if (snapshot.indices().containsKey(indexName)) { // There is a snapshot running with this index name diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/sourceonly/SourceOnlySnapshotShardTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/sourceonly/SourceOnlySnapshotShardTests.java index fbb7ddfb6024c..655547a3f26ea 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/sourceonly/SourceOnlySnapshotShardTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/sourceonly/SourceOnlySnapshotShardTests.java @@ -375,6 +375,7 @@ public void testRestoreMinimal() throws IOException { .build(); repository.finalizeSnapshot( new FinalizeSnapshotContext( + false, new UpdatedShardGenerations(shardGenerations, ShardGenerations.EMPTY), ESBlobStoreRepositoryIntegTestCase.getRepositoryData(repository).getGenId(), Metadata.builder().put(shard.indexSettings().getIndexMetadata(), false).build(), diff --git a/x-pack/plugin/slm/src/internalClusterTest/java/org/elasticsearch/xpack/slm/SLMStatDisruptionIT.java b/x-pack/plugin/slm/src/internalClusterTest/java/org/elasticsearch/xpack/slm/SLMStatDisruptionIT.java index d3eff9eb8585e..5837354cd5bb2 100644 --- a/x-pack/plugin/slm/src/internalClusterTest/java/org/elasticsearch/xpack/slm/SLMStatDisruptionIT.java +++ b/x-pack/plugin/slm/src/internalClusterTest/java/org/elasticsearch/xpack/slm/SLMStatDisruptionIT.java @@ -251,6 +251,7 @@ protected TestRestartBeforeListenersRepo( @Override public void finalizeSnapshot(FinalizeSnapshotContext fsc) { var newFinalizeContext = new FinalizeSnapshotContext( + false, fsc.updatedShardGenerations(), fsc.repositoryStateId(), fsc.clusterMetadata(), diff --git a/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/SnapshotLifecycleTask.java b/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/SnapshotLifecycleTask.java index eb7c9c1b8794e..df124a0f7ba36 100644 --- a/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/SnapshotLifecycleTask.java +++ b/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/SnapshotLifecycleTask.java @@ -20,6 +20,7 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Strings; import org.elasticsearch.common.scheduler.SchedulerEngine; +import org.elasticsearch.core.FixForMultiProject; import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.core.TimeValue; import org.elasticsearch.snapshots.RegisteredPolicySnapshots; @@ -218,7 +219,9 @@ public static String exceptionToString(Exception ex) { static Set currentlyRunningSnapshots(ClusterState clusterState) { final SnapshotsInProgress snapshots = clusterState.custom(SnapshotsInProgress.TYPE, SnapshotsInProgress.EMPTY); final Set currentlyRunning = new HashSet<>(); - for (final List entriesForRepo : snapshots.entriesByRepo()) { + @FixForMultiProject(description = "replace with snapshots.entriesByRepo(ProjectId) when SLM is project aware") + final Iterable> entriesByRepo = snapshots.entriesByRepo(); + for (final List entriesForRepo : entriesByRepo) { for (SnapshotsInProgress.Entry entry : entriesForRepo) { currentlyRunning.add(entry.snapshot().getSnapshotId()); } diff --git a/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/TransportGetSnapshotLifecycleAction.java b/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/TransportGetSnapshotLifecycleAction.java index 5392e182ed20f..7dd702238f755 100644 --- a/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/TransportGetSnapshotLifecycleAction.java +++ b/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/TransportGetSnapshotLifecycleAction.java @@ -86,7 +86,7 @@ protected void masterOperation( } } else { final Map inProgress = new HashMap<>(); - for (List entriesForRepo : SnapshotsInProgress.get(state).entriesByRepo()) { + for (List entriesForRepo : SnapshotsInProgress.get(state).entriesByRepo(projectMetadata.id())) { for (SnapshotsInProgress.Entry entry : entriesForRepo) { Map meta = entry.userMetadata(); if (meta == null diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/testkit/integrity/RepositoryVerifyIntegrityIT.java b/x-pack/plugin/snapshot-repo-test-kit/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/testkit/integrity/RepositoryVerifyIntegrityIT.java index 298dcd9ed94c5..e8c23d0741712 100644 --- a/x-pack/plugin/snapshot-repo-test-kit/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/testkit/integrity/RepositoryVerifyIntegrityIT.java +++ b/x-pack/plugin/snapshot-repo-test-kit/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/testkit/integrity/RepositoryVerifyIntegrityIT.java @@ -14,6 +14,7 @@ import org.elasticsearch.action.support.SubscribableListener; import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; +import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeValue; @@ -27,6 +28,7 @@ import org.elasticsearch.index.snapshots.blobstore.SnapshotFiles; import org.elasticsearch.index.store.StoreFileMetadata; import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.repositories.ProjectRepo; import org.elasticsearch.repositories.RepositoriesService; import org.elasticsearch.repositories.RepositoryData; import org.elasticsearch.repositories.ShardGenerations; @@ -468,7 +470,11 @@ public void testBadSnapshotInfo() throws IOException { final SnapshotInfo snapshotInfo; try (var inputStream = Files.newInputStream(snapshotInfoBlob)) { - snapshotInfo = SNAPSHOT_FORMAT.deserialize(testContext.repositoryName(), xContentRegistry(), inputStream); + snapshotInfo = SNAPSHOT_FORMAT.deserialize( + new ProjectRepo(ProjectId.DEFAULT, testContext.repositoryName()), + xContentRegistry(), + inputStream + ); } final var newIndices = new ArrayList<>(snapshotInfo.indices()); @@ -727,7 +733,7 @@ private void runInconsistentShardGenerationBlobTest( final BlobStoreIndexShardSnapshots blobStoreIndexShardSnapshots; try (var inputStream = Files.newInputStream(shardGenerationBlob)) { blobStoreIndexShardSnapshots = INDEX_SHARD_SNAPSHOTS_FORMAT.deserialize( - testContext.repositoryName(), + new ProjectRepo(ProjectId.DEFAULT, testContext.repositoryName()), xContentRegistry(), inputStream ); diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/integrity/RepositoryIntegrityVerifier.java b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/integrity/RepositoryIntegrityVerifier.java index 5fa3282cd57ae..cadf4463b4605 100644 --- a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/integrity/RepositoryIntegrityVerifier.java +++ b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/integrity/RepositoryIntegrityVerifier.java @@ -315,7 +315,7 @@ void verifySnapshotInfo(SnapshotInfo snapshotInfo, ActionListener listener private void verifySnapshotGlobalMetadata(ActionListener listener) { metadataTaskRunner.run(ActionRunnable.wrap(listener, l -> { try { - blobStoreRepository.getSnapshotGlobalMetadata(snapshotId); + blobStoreRepository.getSnapshotGlobalMetadata(snapshotId, false); // no checks here, loading it is enough l.onResponse(null); } catch (Exception e) { From 7655d7cd226450c4564688666ca2db95b9e467b8 Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Wed, 25 Jun 2025 21:41:06 +1000 Subject: [PATCH 03/12] unmute --- .../core-rest-tests-with-multiple-projects/build.gradle | 9 --------- 1 file changed, 9 deletions(-) diff --git a/x-pack/qa/multi-project/core-rest-tests-with-multiple-projects/build.gradle b/x-pack/qa/multi-project/core-rest-tests-with-multiple-projects/build.gradle index f064c97ff296e..1a666d9ddd0cd 100644 --- a/x-pack/qa/multi-project/core-rest-tests-with-multiple-projects/build.gradle +++ b/x-pack/qa/multi-project/core-rest-tests-with-multiple-projects/build.gradle @@ -47,8 +47,6 @@ tasks.named("yamlRestTest").configure { ArrayList blacklist = [ /* These tests don't work on multi-project yet - we need to go through each of them and make them work */ '^cat.recovery/*/*', - '^cat.repositories/*/*', - '^cat.snapshots/*/*', '^cluster.desired_balance/10_basic/*', '^cluster.stats/10_basic/snapshot stats reported in get cluster stats', '^data_stream/40_supported_apis/Verify shard stores api', // uses _shard_stores API @@ -59,14 +57,7 @@ tasks.named("yamlRestTest").configure { '^indices.resolve_cluster/*/*/*', '^indices.shard_stores/*/*', '^migration/*/*', - '^nodes.stats/70_repository_throttling_stats/Repository throttling stats (some repositories exist)', - '^snapshot.clone/*/*', - '^snapshot.create/*/*', - '^snapshot.delete/*/*', - '^snapshot.get/*/*', - '^snapshot.get_repository/*/*', '^snapshot.restore/*/*', - '^snapshot.status/*/*', '^synonyms/*/*', '^tsdb/30_snapshot/*', '^update_by_query/80_scripting/Update all docs with one deletion and one noop using a stored script', // scripting is not project aware yet From f669e03c0fb583c4e599b40d74aa489c3dfe8e2e Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Wed, 25 Jun 2025 21:53:53 +1000 Subject: [PATCH 04/12] tweak --- .../TransportSnapshotsStatusAction.java | 66 +++++++++---------- .../snapshots/SnapshotsService.java | 2 - 2 files changed, 33 insertions(+), 35 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportSnapshotsStatusAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportSnapshotsStatusAction.java index a05a183d7f7e2..61925bd31d156 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportSnapshotsStatusAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportSnapshotsStatusAction.java @@ -362,39 +362,39 @@ private void loadRepositoryData( delegate.onResponse(new SnapshotsStatusResponse(Collections.unmodifiableList(builder))); } else { final List threadSafeBuilder = Collections.synchronizedList(builder); - repositoriesService.repository(projectId, repositoryName) - .getSnapshotInfo(snapshotIdsToLoad, true, task::isCancelled, snapshotInfo -> { - List shardStatusBuilder = new ArrayList<>(); - final Map shardStatuses; - shardStatuses = snapshotShards(projectId, repositoryName, repositoryData, task, snapshotInfo); - // an exception here stops further fetches of snapshotInfo since context is fail-fast - for (final var shardStatus : shardStatuses.entrySet()) { - IndexShardSnapshotStatus.Copy lastSnapshotStatus = shardStatus.getValue(); - shardStatusBuilder.add(new SnapshotIndexShardStatus(shardStatus.getKey(), lastSnapshotStatus)); - } - final SnapshotsInProgress.State state = switch (snapshotInfo.state()) { - case FAILED -> SnapshotsInProgress.State.FAILED; - case SUCCESS, PARTIAL -> - // Both of these means the snapshot has completed. - SnapshotsInProgress.State.SUCCESS; - default -> throw new IllegalArgumentException("Unexpected snapshot state " + snapshotInfo.state()); - }; - final long startTime = snapshotInfo.startTime(); - final long endTime = snapshotInfo.endTime(); - assert endTime >= startTime || (endTime == 0L && snapshotInfo.state().completed() == false) - : "Inconsistent timestamps found in SnapshotInfo [" + snapshotInfo + "]"; - threadSafeBuilder.add( - new SnapshotStatus( - new Snapshot(projectId, repositoryName, snapshotInfo.snapshotId()), - state, - Collections.unmodifiableList(shardStatusBuilder), - snapshotInfo.includeGlobalState(), - startTime, - // Use current time to calculate overall runtime for in-progress snapshots that have endTime == 0 - (endTime == 0 ? threadPool.absoluteTimeInMillis() : endTime) - startTime - ) - ); - }, delegate.map(v -> new SnapshotsStatusResponse(List.copyOf(threadSafeBuilder)))); + final Repository repository = repositoriesService.repository(projectId, repositoryName); + repository.getSnapshotInfo(snapshotIdsToLoad, true, task::isCancelled, snapshotInfo -> { + List shardStatusBuilder = new ArrayList<>(); + final Map shardStatuses; + shardStatuses = snapshotShards(projectId, repositoryName, repositoryData, task, snapshotInfo); + // an exception here stops further fetches of snapshotInfo since context is fail-fast + for (final var shardStatus : shardStatuses.entrySet()) { + IndexShardSnapshotStatus.Copy lastSnapshotStatus = shardStatus.getValue(); + shardStatusBuilder.add(new SnapshotIndexShardStatus(shardStatus.getKey(), lastSnapshotStatus)); + } + final SnapshotsInProgress.State state = switch (snapshotInfo.state()) { + case FAILED -> SnapshotsInProgress.State.FAILED; + case SUCCESS, PARTIAL -> + // Both of these means the snapshot has completed. + SnapshotsInProgress.State.SUCCESS; + default -> throw new IllegalArgumentException("Unexpected snapshot state " + snapshotInfo.state()); + }; + final long startTime = snapshotInfo.startTime(); + final long endTime = snapshotInfo.endTime(); + assert endTime >= startTime || (endTime == 0L && snapshotInfo.state().completed() == false) + : "Inconsistent timestamps found in SnapshotInfo [" + snapshotInfo + "]"; + threadSafeBuilder.add( + new SnapshotStatus( + new Snapshot(projectId, repositoryName, snapshotInfo.snapshotId()), + state, + Collections.unmodifiableList(shardStatusBuilder), + snapshotInfo.includeGlobalState(), + startTime, + // Use current time to calculate overall runtime for in-progress snapshots that have endTime == 0 + (endTime == 0 ? threadPool.absoluteTimeInMillis() : endTime) - startTime + ) + ); + }, delegate.map(v -> new SnapshotsStatusResponse(List.copyOf(threadSafeBuilder)))); } })); } diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java index f7901607fe724..689834459bb3d 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java @@ -1479,10 +1479,8 @@ private void finalizeSnapshotEntry(Snapshot snapshot, Metadata metadata, Reposit final ProjectId projectId = snapshot.getProjectId(); final Metadata effectiveMetadata; if (supportsMultipleProjects) { - assert ProjectId.DEFAULT.equals(projectId) == false : "should not snapshot for default project when multi-project is enabled"; effectiveMetadata = Metadata.builder().put(metadata.getProject(projectId)).build(); } else { - assert ProjectId.DEFAULT.equals(projectId) : "expect snapshots for only the default project when multi-project is disabled"; effectiveMetadata = metadata; } threadPool.executor(ThreadPool.Names.SNAPSHOT).execute(new SnapshotFinalization(snapshot, effectiveMetadata, repositoryData)); From a7454d6dda03a1e35be8f50120b288804d141354 Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Thu, 26 Jun 2025 00:14:57 +1000 Subject: [PATCH 05/12] fix license header --- .../repositories/ProjectRepo.java | 21 ++++++------------- 1 file changed, 6 insertions(+), 15 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/repositories/ProjectRepo.java b/server/src/main/java/org/elasticsearch/repositories/ProjectRepo.java index 034ba6e876d04..49884d6905c1b 100644 --- a/server/src/main/java/org/elasticsearch/repositories/ProjectRepo.java +++ b/server/src/main/java/org/elasticsearch/repositories/ProjectRepo.java @@ -1,20 +1,11 @@ /* - * ELASTICSEARCH CONFIDENTIAL - * __________________ - * - * Copyright Elasticsearch B.V. All rights reserved. - * - * NOTICE: All information contained herein is, and remains - * the property of Elasticsearch B.V. and its suppliers, if any. - * The intellectual and technical concepts contained herein - * are proprietary to Elasticsearch B.V. and its suppliers and - * may be covered by U.S. and Foreign Patents, patents in - * process, and are protected by trade secret or copyright - * law. Dissemination of this information or reproduction of - * this material is strictly forbidden unless prior written - * permission is obtained from Elasticsearch B.V. + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ - package org.elasticsearch.repositories; import org.elasticsearch.cluster.DiffableUtils; From d9248b4e32661209160f119610719fff88e4f057 Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Thu, 26 Jun 2025 00:40:13 +1000 Subject: [PATCH 06/12] fix test and improve logging --- .../TransportDeleteDataStreamActionTests.java | 10 +- .../blobstore/BlobStoreRepository.java | 132 +++++++++--------- .../snapshots/SnapshotsService.java | 13 +- 3 files changed, 81 insertions(+), 74 deletions(-) diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/TransportDeleteDataStreamActionTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/TransportDeleteDataStreamActionTests.java index 35623e8c1a216..d736340b9d8bc 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/TransportDeleteDataStreamActionTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/TransportDeleteDataStreamActionTests.java @@ -15,6 +15,7 @@ import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.metadata.DataStreamTestHelper; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.common.Strings; import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.settings.Settings; @@ -138,8 +139,9 @@ public void testDeleteSnapshottingDataStream() { List.of(new Tuple<>(dataStreamName, 2), new Tuple<>(dataStreamName2, 2)), otherIndices ); - SnapshotsInProgress snapshotsInProgress = SnapshotsInProgress.EMPTY.withAddedEntry(createEntry(dataStreamName, "repo1", false)) - .withAddedEntry(createEntry(dataStreamName2, "repo2", true)); + SnapshotsInProgress snapshotsInProgress = SnapshotsInProgress.EMPTY.withAddedEntry( + createEntry(dataStreamName, projectId, "repo1", false) + ).withAddedEntry(createEntry(dataStreamName2, projectId, "repo2", true)); ClusterState snapshotCs = ClusterState.builder(cs).putCustom(SnapshotsInProgress.TYPE, snapshotsInProgress).build(); DeleteDataStreamAction.Request req = new DeleteDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] { dataStreamName }); @@ -157,9 +159,9 @@ public void testDeleteSnapshottingDataStream() { ); } - private SnapshotsInProgress.Entry createEntry(String dataStreamName, String repo, boolean partial) { + private SnapshotsInProgress.Entry createEntry(String dataStreamName, ProjectId projectId, String repo, boolean partial) { return SnapshotsInProgress.Entry.snapshot( - new Snapshot(repo, new SnapshotId("", "")), + new Snapshot(projectId, repo, new SnapshotId("", "")), false, partial, SnapshotsInProgress.State.SUCCESS, diff --git a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java index 44314e924bd6a..97a611f5b0da2 100644 --- a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java +++ b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java @@ -179,6 +179,7 @@ import static org.elasticsearch.index.snapshots.blobstore.BlobStoreIndexShardSnapshot.FileInfo; import static org.elasticsearch.index.snapshots.blobstore.BlobStoreIndexShardSnapshot.FileInfo.canonicalName; import static org.elasticsearch.indices.recovery.RecoverySettings.INDICES_RECOVERY_MAX_BYTES_PER_SEC_SETTING; +import static org.elasticsearch.repositories.ProjectRepo.projectRepoString; /** * BlobStore - based implementation of Snapshot Repository @@ -1522,8 +1523,8 @@ private void cleanupUnlinkedRootAndIndicesBlobs(RepositoryData newRepositoryData } catch (Exception e) { logger.warn( () -> format( - "[%s] The following blobs are no longer part of any snapshot [%s] but failed to remove them", - metadata.name(), + "%s The following blobs are no longer part of any snapshot [%s] but failed to remove them", + toStringShort(), staleRootBlobs ), e @@ -1551,8 +1552,8 @@ private void cleanupUnlinkedRootAndIndicesBlobs(RepositoryData newRepositoryData logger.debug("[{}] Cleaned up stale index [{}]", metadata.name(), indexId); } catch (IOException e) { logger.warn(() -> format(""" - [%s] index %s is no longer part of any snapshot in the repository, \ - but failed to clean up its index folder""", metadata.name(), indexId), e); + %s index %s is no longer part of any snapshot in the repository, \ + but failed to clean up its index folder""", toStringShort(), indexId), e); } })); } @@ -1625,7 +1626,7 @@ private void logStaleRootLevelBlobs( .collect(Collectors.toSet()); final List blobsToLog = blobsToDelete.stream().filter(b -> blobNamesToIgnore.contains(b) == false).toList(); if (blobsToLog.isEmpty() == false) { - logger.info("[{}] Found stale root level blobs {}. Cleaning them up", metadata.name(), blobsToLog); + logger.info("{} Found stale root level blobs {}. Cleaning them up", toStringShort(), blobsToLog); } } } @@ -2153,9 +2154,9 @@ private RateLimiter getRateLimiter( if (warnIfOverRecovery && effectiveRecoverySpeed.getBytes() > 0) { if (maxConfiguredBytesPerSec.getBytes() > effectiveRecoverySpeed.getBytes()) { logger.warn( - "repository [{}] has a rate limit [{}={}] per second which is above the effective recovery rate limit " + "repository {} has a rate limit [{}={}] per second which is above the effective recovery rate limit " + "[{}={}] per second, thus the repository rate limit will be superseded by the recovery rate limit", - metadata.name(), + toStringShort(), settingKey, maxConfiguredBytesPerSec, INDICES_RECOVERY_MAX_BYTES_PER_SEC_SETTING.getKey(), @@ -2362,7 +2363,7 @@ public void onResponse(RepositoryData repositoryData) { @Override public void onFailure(Exception e) { logger.warn( - () -> format("[%s] Exception when initializing repository generation in cluster state", metadata.name()), + () -> format("%s Exception when initializing repository generation in cluster state", toStringShort()), e ); acquireAndClearRepoDataInitialized().onFailure(e); @@ -2611,56 +2612,53 @@ private static String previousWriterMessage(@Nullable Tuple previo private void markRepoCorrupted(long corruptedGeneration, Exception originalException, ActionListener listener) { assert corruptedGeneration != RepositoryData.UNKNOWN_REPO_GEN; assert bestEffortConsistency == false; - logger.warn(() -> "Marking repository [" + metadata.name() + "] as corrupted", originalException); - submitUnbatchedTask( - "mark repository corrupted [" + metadata.name() + "][" + corruptedGeneration + "]", - new ClusterStateUpdateTask() { - @Override - public ClusterState execute(ClusterState currentState) { - final var project = currentState.metadata().getProject(projectId); - final RepositoriesMetadata state = RepositoriesMetadata.get(project); - final RepositoryMetadata repoState = state.repository(metadata.name()); - if (repoState.generation() != corruptedGeneration) { - throw new IllegalStateException( - "Tried to mark repo generation [" - + corruptedGeneration - + "] as corrupted but its state concurrently changed to [" - + repoState - + "]" - ); - } - return ClusterState.builder(currentState) - .putProjectMetadata( - ProjectMetadata.builder(project) - .putCustom( - RepositoriesMetadata.TYPE, - state.withUpdatedGeneration( - metadata.name(), - RepositoryData.CORRUPTED_REPO_GEN, - repoState.pendingGeneration() - ) - ) - ) - .build(); - } - - @Override - public void onFailure(Exception e) { - listener.onFailure( - new RepositoryException( - metadata.name(), - "Failed marking repository state as corrupted", - ExceptionsHelper.useOrSuppress(e, originalException) - ) + logger.warn(() -> "Marking repository " + toStringShort() + " as corrupted", originalException); + submitUnbatchedTask("mark repository corrupted " + toStringShort() + "[" + corruptedGeneration + "]", new ClusterStateUpdateTask() { + @Override + public ClusterState execute(ClusterState currentState) { + final var project = currentState.metadata().getProject(projectId); + final RepositoriesMetadata state = RepositoriesMetadata.get(project); + final RepositoryMetadata repoState = state.repository(metadata.name()); + if (repoState.generation() != corruptedGeneration) { + throw new IllegalStateException( + "Tried to mark repo generation [" + + corruptedGeneration + + "] as corrupted but its state concurrently changed to [" + + repoState + + "]" ); } + return ClusterState.builder(currentState) + .putProjectMetadata( + ProjectMetadata.builder(project) + .putCustom( + RepositoriesMetadata.TYPE, + state.withUpdatedGeneration( + metadata.name(), + RepositoryData.CORRUPTED_REPO_GEN, + repoState.pendingGeneration() + ) + ) + ) + .build(); + } - @Override - public void clusterStateProcessed(ClusterState oldState, ClusterState newState) { - listener.onResponse(null); - } + @Override + public void onFailure(Exception e) { + listener.onFailure( + new RepositoryException( + metadata.name(), + "Failed marking repository state as corrupted", + ExceptionsHelper.useOrSuppress(e, originalException) + ) + ); } - ); + + @Override + public void clusterStateProcessed(ClusterState oldState, ClusterState newState) { + listener.onResponse(null); + } + }); } private RepositoryData getRepositoryData(long indexGen) { @@ -2786,9 +2784,9 @@ public ClusterState execute(ClusterState currentState) { final boolean uninitializedMeta = meta.generation() == RepositoryData.UNKNOWN_REPO_GEN || bestEffortConsistency; if (uninitializedMeta == false && meta.pendingGeneration() != genInState) { logger.info( - "Trying to write new repository data over unfinished write, repo [{}] is at " + "Trying to write new repository data over unfinished write, repo {} is at " + "safe generation [{}] and pending generation [{}]", - meta.name(), + toStringShort(), genInState, meta.pendingGeneration() ); @@ -2920,9 +2918,9 @@ public void onFailure(Exception e) { assert newRepositoryData.getUuid().equals(RepositoryData.MISSING_UUID) == false; logger.info( Strings.format( - "Generated new repository UUID [%s] for repository [%s] in generation [%d]", + "Generated new repository UUID [%s] for repository %s in generation [%d]", newRepositoryData.getUuid(), - metadata.name(), + toStringShort(), newGen ) ); @@ -3211,7 +3209,7 @@ private long latestGeneration(Collection rootBlobs) { } catch (NumberFormatException nfe) { // the index- blob wasn't of the format index-N where N is a number, // no idea what this blob is but it doesn't belong in the repository! - logger.warn("[{}] Unknown blob in the repository: {}", metadata.name(), blobName); + logger.warn("[{}] Unknown blob in the repository: {}", toStringShort(), blobName); } } return latest; @@ -3890,7 +3888,11 @@ public void verify(String seed, DiscoveryNode localNode) { @Override public String toString() { - return "BlobStoreRepository[" + "[" + metadata.name() + "], [" + blobStore.get() + ']' + ']'; + return "BlobStoreRepository[" + toStringShort() + ", [" + blobStore.get() + ']' + ']'; + } + + private String toStringShort() { + return projectRepoString(projectId, metadata.name()); } /** @@ -4011,10 +4013,10 @@ private Tuple buildBlobStoreIndex // keeping hold of its data blobs. try { final var message = Strings.format( - "index %s shard generation [%s] in [%s][%s] not found - falling back to reading all shard snapshots", + "index %s shard generation [%s] in %s[%s] not found - falling back to reading all shard snapshots", indexId, generation, - metadata.name(), + toStringShort(), shardContainer.path() ); logger.error(message, noSuchFileException); @@ -4055,17 +4057,17 @@ private Tuple buildBlobStoreIndex } } logger.error( - "read shard snapshots [{}] due to missing shard generation [{}] for index {} in [{}][{}]", + "read shard snapshots [{}] due to missing shard generation [{}] for index {} in {}[{}]", messageBuilder, generation, indexId, - metadata.name(), + toStringShort(), shardContainer.path() ); return new Tuple<>(blobStoreIndexShardSnapshots, generation); } catch (Exception fallbackException) { logger.error( - Strings.format("failed while reading all shard snapshots from [%s][%s]", metadata.name(), shardContainer.path()), + Strings.format("failed while reading all shard snapshots from %s[%s]", toStringShort(), shardContainer.path()), fallbackException ); noSuchFileException.addSuppressed(fallbackException); diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java index 689834459bb3d..381d500843f83 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java @@ -2671,7 +2671,7 @@ public void onFailure(Exception e) { final var collector = new Strings.BoundedDelimitedStringCollector(sb, ",", 1024); deleteEntry.snapshots().forEach(s -> collector.appendItem(s.getName())); collector.finish(); - sb.append("] from repository [").append(deleteEntry.repository()).append("]"); + sb.append("] from repository ").append(projectRepoString(deleteEntry.projectId(), deleteEntry.repository())); return sb; }, e); submitUnbatchedTask( @@ -4004,8 +4004,11 @@ public void onFailure(Exception e) { @Override public void clusterStateProcessed(ClusterState oldState, ClusterState newState) { logger.warn( - () -> format("Removed all snapshot tasks for repository [%s] from cluster state, now failing listeners", repository), - failure + () -> format( + "Removed all snapshot tasks for repository %s from cluster state, now failing listeners", + projectRepoString(projectId, repository), + failure + ) ); final List readyToResolveListeners = new ArrayList<>(); synchronized (currentlyFinalizing) { @@ -4147,8 +4150,8 @@ private static void logSnapshotFailure(String operation, Snapshot snapshot, Exce // suppress stack trace at INFO unless extra verbosity is configured logger.info( format( - "[%s][%s] failed to %s snapshot: %s", - snapshot.getRepository(), + "%s[%s] failed to %s snapshot: %s", + projectRepoString(snapshot.getProjectId(), snapshot.getRepository()), snapshot.getSnapshotId().getName(), operation, e.getMessage() From 132401f6ded144e07f1b9c7ff08f2ff1ee401bc9 Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Thu, 26 Jun 2025 09:30:25 +1000 Subject: [PATCH 07/12] fix tests --- .../SnapshotInProgressAllocationDecider.java | 7 ++++--- .../repositories/RepositoriesService.java | 4 ++-- .../blobstore/BlobStoreRepository.java | 3 ++- .../elasticsearch/snapshots/SnapshotsService.java | 14 +++++++------- .../metadata/MetadataDataStreamsServiceTests.java | 4 ++-- .../metadata/MetadataDeleteIndexServiceTests.java | 15 +++++++-------- .../metadata/MetadataIndexStateServiceTests.java | 6 +++++- .../blobstore/BlobStoreRepositoryTests.java | 6 +++--- 8 files changed, 32 insertions(+), 27 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/SnapshotInProgressAllocationDecider.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/SnapshotInProgressAllocationDecider.java index d204dfe8a3841..03ae7c6f1d32c 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/SnapshotInProgressAllocationDecider.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/SnapshotInProgressAllocationDecider.java @@ -10,11 +10,11 @@ package org.elasticsearch.cluster.routing.allocation.decider; import org.elasticsearch.cluster.SnapshotsInProgress; -import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.cluster.metadata.SingleNodeShutdownMetadata; import org.elasticsearch.cluster.routing.RoutingNode; import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.cluster.routing.allocation.RoutingAllocation; +import org.elasticsearch.core.FixForMultiProject; import java.util.Objects; @@ -74,8 +74,9 @@ private static Decision canMove(ShardRouting shardRouting, RoutingAllocation all return YES_NOT_SNAPSHOTTED; } - final ProjectId projectId = allocation.metadata().projectFor(shardRouting.index()).id(); - for (final var entriesByRepo : snapshotsInProgress.entriesByRepo(projectId)) { + @FixForMultiProject(description = "replace with entriesByRepo(ProjectId), see also ES-12195") + final var entriesByRepoIterable = snapshotsInProgress.entriesByRepo(); + for (final var entriesByRepo : entriesByRepoIterable) { for (final var entry : entriesByRepo) { if (entry.isClone()) { // clones do not run on data nodes diff --git a/server/src/main/java/org/elasticsearch/repositories/RepositoriesService.java b/server/src/main/java/org/elasticsearch/repositories/RepositoriesService.java index f5d8cad85a290..f11232771d48f 100644 --- a/server/src/main/java/org/elasticsearch/repositories/RepositoriesService.java +++ b/server/src/main/java/org/elasticsearch/repositories/RepositoriesService.java @@ -455,8 +455,8 @@ public static void updateRepositoryUuidInMetadata( logger.info( Strings.format( - "Registering repository [%s] with repository UUID [%s] and generation [%d]", - repositoryName, + "Registering repository %s with repository UUID [%s] and generation [%d]", + projectRepoString(projectId, repositoryName), repositoryData.getUuid(), repositoryData.getGenId() ) diff --git a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java index 97a611f5b0da2..1bcdcc823b2a4 100644 --- a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java +++ b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java @@ -3891,7 +3891,8 @@ public String toString() { return "BlobStoreRepository[" + toStringShort() + ", [" + blobStore.get() + ']' + ']'; } - private String toStringShort() { + // Package private for testing + String toStringShort() { return projectRepoString(projectId, metadata.name()); } diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java index 381d500843f83..e91dec18f166e 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java @@ -198,7 +198,7 @@ public final class SnapshotsService extends AbstractLifecycleComponent implement private final SystemIndices systemIndices; - private final boolean supportsMultipleProjects; + private final boolean serializeProjectMetadata; private final MasterServiceTaskQueue masterServiceTaskQueue; @@ -228,7 +228,7 @@ public SnapshotsService( TransportService transportService, ActionFilters actionFilters, SystemIndices systemIndices, - boolean supportsMultipleProjects + boolean serializeProjectMetadata ) { this.clusterService = clusterService; this.rerouteService = rerouteService; @@ -247,7 +247,7 @@ public SnapshotsService( .addSettingsUpdateConsumer(MAX_CONCURRENT_SNAPSHOT_OPERATIONS_SETTING, i -> maxConcurrentOperations = i); } this.systemIndices = systemIndices; - this.supportsMultipleProjects = supportsMultipleProjects; + this.serializeProjectMetadata = serializeProjectMetadata; this.masterServiceTaskQueue = clusterService.createTaskQueue("snapshots-service", Priority.NORMAL, new SnapshotTaskExecutor()); this.updateNodeIdsToRemoveQueue = clusterService.createTaskQueue( @@ -808,7 +808,7 @@ private static UpdatedShardGenerations buildGenerations(SnapshotsInProgress.Entr private Metadata metadataForSnapshot(SnapshotsInProgress.Entry snapshot, Metadata metadata, ProjectId projectId) { final ProjectMetadata snapshotProject = projectForSnapshot(snapshot, metadata.getProject(projectId)); final Metadata.Builder builder; - if (snapshot.includeGlobalState() == false || supportsMultipleProjects) { + if (snapshot.includeGlobalState() == false || serializeProjectMetadata) { // Remove global state from the cluster state when // 1. The request explicitly demands that // 2. The snapshot is for a project in a multi-project cluster. Such a snapshot must not include cluster level info @@ -1478,7 +1478,7 @@ private void leaveRepoLoop(ProjectId projectId, String repository) { private void finalizeSnapshotEntry(Snapshot snapshot, Metadata metadata, RepositoryData repositoryData) { final ProjectId projectId = snapshot.getProjectId(); final Metadata effectiveMetadata; - if (supportsMultipleProjects) { + if (serializeProjectMetadata) { effectiveMetadata = Metadata.builder().put(metadata.getProject(projectId)).build(); } else { effectiveMetadata = metadata; @@ -1541,7 +1541,7 @@ protected void doRun() { final Repository repo = repositoriesService.repository(projectId, repository); if (entry.isClone()) { threadPool.executor(ThreadPool.Names.SNAPSHOT).execute(ActionRunnable.supply(metadataListener, () -> { - final Metadata existingMetadata = repo.getSnapshotGlobalMetadata(entry.source(), supportsMultipleProjects); + final Metadata existingMetadata = repo.getSnapshotGlobalMetadata(entry.source(), serializeProjectMetadata); final ProjectMetadata existingProject = existingMetadata.getProject(projectId); final ProjectMetadata.Builder projBuilder = ProjectMetadata.builder(existingProject); final Set existingIndices = new HashSet<>(); @@ -1623,7 +1623,7 @@ protected void doRun() { final ListenableFuture>> snapshotListeners = new ListenableFuture<>(); repo.finalizeSnapshot( new FinalizeSnapshotContext( - supportsMultipleProjects, + serializeProjectMetadata, updatedShardGenerations, repositoryData.getGenId(), metaForSnapshot, diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataDataStreamsServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataDataStreamsServiceTests.java index 936831f15baf5..7ac4399cc0fae 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataDataStreamsServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataDataStreamsServiceTests.java @@ -559,7 +559,8 @@ public void testDeleteMissing() { public void testDeleteSnapshotting() { String dataStreamName = randomAlphaOfLength(5); - Snapshot snapshot = new Snapshot("doesn't matter", new SnapshotId("snapshot name", "snapshot uuid")); + var projectId = randomProjectIdOrDefault(); + Snapshot snapshot = new Snapshot(projectId, "doesn't matter", new SnapshotId("snapshot name", "snapshot uuid")); SnapshotsInProgress snaps = SnapshotsInProgress.EMPTY.withAddedEntry( SnapshotsInProgress.Entry.snapshot( snapshot, @@ -578,7 +579,6 @@ public void testDeleteSnapshotting() { ) ); final DataStream dataStream = DataStreamTestHelper.randomInstance(dataStreamName); - var projectId = randomProjectIdOrDefault(); ProjectState state = ClusterState.builder(ClusterName.DEFAULT) .putCustom(SnapshotsInProgress.TYPE, snaps) .putProjectMetadata(ProjectMetadata.builder(projectId).put(dataStream)) diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataDeleteIndexServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataDeleteIndexServiceTests.java index 2259ccb26621c..1020d9b2a14d7 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataDeleteIndexServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataDeleteIndexServiceTests.java @@ -88,7 +88,8 @@ public void testDeleteMissing() { public void testDeleteSnapshotting() { String indexName = randomAlphaOfLength(5); - Snapshot snapshot = new Snapshot("doesn't matter", new SnapshotId("snapshot name", "snapshot uuid")); + final ProjectId projectId = randomProjectIdOrDefault(); + Snapshot snapshot = new Snapshot(projectId, "doesn't matter", new SnapshotId("snapshot name", "snapshot uuid")); SnapshotsInProgress snaps = SnapshotsInProgress.EMPTY.withAddedEntry( SnapshotsInProgress.Entry.snapshot( snapshot, @@ -107,7 +108,7 @@ public void testDeleteSnapshotting() { ) ); final Index index = new Index(indexName, randomUUID()); - ClusterState state = ClusterState.builder(clusterState(index)).putCustom(SnapshotsInProgress.TYPE, snaps).build(); + ClusterState state = ClusterState.builder(clusterState(projectId, index)).putCustom(SnapshotsInProgress.TYPE, snaps).build(); Exception e = expectThrows( SnapshotInProgressException.class, () -> MetadataDeleteIndexService.deleteIndices(state, Set.of(index), Settings.EMPTY) @@ -125,9 +126,8 @@ public void testDeleteUnassigned() throws Exception { // Create an unassigned index String indexName = randomAlphaOfLength(5); Index index = new Index(indexName, randomUUID()); - ClusterState before = clusterState(index); - - final var projectId = before.metadata().projectFor(index).id(); + final ProjectId projectId = randomProjectIdOrDefault(); + ClusterState before = clusterState(projectId, index); // Mock the built reroute when(allocationService.reroute(any(ClusterState.class), anyString(), any())).then(i -> i.getArguments()[0]); @@ -433,11 +433,10 @@ public void testDeleteIndicesFromMultipleProjects() { assertThat(after.metadata().projects(), aMapWithSize(numProjects)); } - private ClusterState clusterState(Index index) { + private ClusterState clusterState(ProjectId projectId, Index index) { final IndexMetadata indexMetadata = IndexMetadata.builder(index.getName()) .settings(indexSettings(IndexVersionUtils.randomVersion(), index.getUUID(), 1, 1)) .build(); - final ProjectId projectId = randomProjectIdOrDefault(); final Metadata.Builder metadataBuilder = Metadata.builder().put(ProjectMetadata.builder(projectId).put(indexMetadata, false)); if (randomBoolean()) { @@ -454,7 +453,7 @@ private ClusterState clusterState(Index index) { return ClusterState.builder(ClusterName.DEFAULT) .metadata(metadata) .routingTable(GlobalRoutingTableTestHelper.buildRoutingTable(metadata, RoutingTable.Builder::addAsNew)) - .blocks(ClusterBlocks.builder().addBlocks(indexMetadata)) + .blocks(ClusterBlocks.builder().addBlocks(projectId, indexMetadata)) .build(); } } diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexStateServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexStateServiceTests.java index 88603fd3e6315..cf9dd323db6ac 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexStateServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexStateServiceTests.java @@ -452,7 +452,11 @@ private static ClusterState addSnapshotIndex( ); } - final Snapshot snapshot = new Snapshot(randomAlphaOfLength(10), new SnapshotId(randomAlphaOfLength(5), randomAlphaOfLength(5))); + final Snapshot snapshot = new Snapshot( + projectId, + randomAlphaOfLength(10), + new SnapshotId(randomAlphaOfLength(5), randomAlphaOfLength(5)) + ); final SnapshotsInProgress.Entry entry = SnapshotsInProgress.Entry.snapshot( snapshot, randomBoolean(), diff --git a/server/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryTests.java b/server/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryTests.java index eb0b545712982..802e5a86afa35 100644 --- a/server/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryTests.java +++ b/server/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryTests.java @@ -700,7 +700,7 @@ public void testUuidCreationLogging() { "new repo uuid message", BlobStoreRepository.class.getCanonicalName(), Level.INFO, - Strings.format("Generated new repository UUID [*] for repository [%s] in generation [*]", repoName) + Strings.format("Generated new repository UUID [*] for repository %s in generation [*]", repo.toStringShort()) ) ); @@ -729,7 +729,7 @@ public void testUuidCreationLogging() { "existing repo uuid message", RepositoriesService.class.getCanonicalName(), Level.INFO, - Strings.format("Registering repository [%s] with repository UUID *", repoName) + Strings.format("Registering repository %s with repository UUID *", repo.toStringShort()) ) ); @@ -785,7 +785,7 @@ public void testUuidCreationLogging() { "existing repo uuid message", RepositoriesService.class.getCanonicalName(), Level.INFO, - Strings.format("Registering repository [%s] with repository UUID *", repoName) + Strings.format("Registering repository %s with repository UUID *", repo.toStringShort()) ) ); }, From 0981ad4c369c70ce543fb23322ec5872cbbbb51e Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Thu, 26 Jun 2025 10:58:21 +1000 Subject: [PATCH 08/12] fix more tests --- .../snapshots/CorruptedBlobStoreRepositoryIT.java | 5 +++-- .../org/elasticsearch/snapshots/SnapshotThrottlingIT.java | 4 ++-- .../org/elasticsearch/snapshots/SnapshotsServiceIT.java | 8 ++++---- .../org/elasticsearch/snapshots/SnapshotsService.java | 2 +- 4 files changed, 10 insertions(+), 9 deletions(-) diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/CorruptedBlobStoreRepositoryIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/CorruptedBlobStoreRepositoryIT.java index badc7f47b1a0a..dcc4a49df8bd5 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/CorruptedBlobStoreRepositoryIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/CorruptedBlobStoreRepositoryIT.java @@ -820,7 +820,8 @@ public void testSnapshotWithMissingShardLevelIndexFile() throws Exception { "fallback message", "org.elasticsearch.repositories.blobstore.BlobStoreRepository", Level.ERROR, - "index [test-idx-1/*] shard generation [*] in [test-repo][*] not found - falling back to reading all shard snapshots" + "index [test-idx-1/*] shard generation [*] in [default][test-repo][*] not found " + + "- falling back to reading all shard snapshots" ) ); mockLog.addExpectation( @@ -828,7 +829,7 @@ public void testSnapshotWithMissingShardLevelIndexFile() throws Exception { "shard blobs list", "org.elasticsearch.repositories.blobstore.BlobStoreRepository", Level.ERROR, - "read shard snapshots [*] due to missing shard generation [*] for index [test-idx-1/*] in [test-repo][*]" + "read shard snapshots [*] due to missing shard generation [*] for index [test-idx-1/*] in [default][test-repo][*]" ) ); if (repairWithDelete) { diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotThrottlingIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotThrottlingIT.java index d664e31b91383..d5dba02f14855 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotThrottlingIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotThrottlingIT.java @@ -141,7 +141,7 @@ public void testWarningSpeedOverRecovery() throws Exception { "snapshot speed over recovery speed", "org.elasticsearch.repositories.blobstore.BlobStoreRepository", Level.WARN, - "repository [test-repo] has a rate limit [max_snapshot_bytes_per_sec=1gb] per second which is above " + "repository [default][test-repo] has a rate limit [max_snapshot_bytes_per_sec=1gb] per second which is above " + "the effective recovery rate limit [indices.recovery.max_bytes_per_sec=100mb] per second, thus the repository " + "rate limit will be superseded by the recovery rate limit" ); @@ -152,7 +152,7 @@ public void testWarningSpeedOverRecovery() throws Exception { "snapshot restore speed over recovery speed", "org.elasticsearch.repositories.blobstore.BlobStoreRepository", Level.WARN, - "repository [test-repo] has a rate limit [max_restore_bytes_per_sec=2gb] per second which is above " + "repository [default][test-repo] has a rate limit [max_restore_bytes_per_sec=2gb] per second which is above " + "the effective recovery rate limit [indices.recovery.max_bytes_per_sec=100mb] per second, thus the repository " + "rate limit will be superseded by the recovery rate limit" ); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotsServiceIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotsServiceIT.java index cf2e4c54f121b..5bd242322efaa 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotsServiceIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotsServiceIT.java @@ -44,7 +44,7 @@ public void testDeletingSnapshotsIsLoggedAfterClusterStateIsProcessed() throws E "[does-not-exist]", SnapshotsService.class.getName(), Level.INFO, - "deleting snapshots [does-not-exist] from repository [test-repo]" + "deleting snapshots [does-not-exist] from repository [default][test-repo]" ) ); @@ -53,7 +53,7 @@ public void testDeletingSnapshotsIsLoggedAfterClusterStateIsProcessed() throws E "[deleting test-snapshot]", SnapshotsService.class.getName(), Level.INFO, - "deleting snapshots [test-snapshot] from repository [test-repo]" + "deleting snapshots [test-snapshot] from repository [default][test-repo]" ) ); @@ -62,7 +62,7 @@ public void testDeletingSnapshotsIsLoggedAfterClusterStateIsProcessed() throws E "[test-snapshot deleted]", SnapshotsService.class.getName(), Level.INFO, - "snapshots [test-snapshot/*] deleted" + "snapshots [test-snapshot/*] deleted in repository [default][test-repo]" ) ); @@ -91,7 +91,7 @@ public void testSnapshotDeletionFailureShouldBeLogged() throws Exception { "[test-snapshot]", SnapshotsService.class.getName(), Level.WARN, - "failed to complete snapshot deletion for [test-snapshot] from repository [test-repo]" + "failed to complete snapshot deletion for [test-snapshot] from repository [default][test-repo]" ) ); diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java index e91dec18f166e..ca9ca187c594f 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java @@ -2690,7 +2690,7 @@ protected void handleListeners(List> deleteListeners) { final var collector = new Strings.BoundedDelimitedStringCollector(sb, ",", 1024); snapshotIds.forEach(collector::appendItem); collector.finish(); - sb.append("] deleted in repository ["); + sb.append("] deleted in repository "); sb.append(projectRepoString(deleteEntry.projectId(), deleteEntry.repository())); return sb; }); From 6109e9dcd8525401e35d71ef206fbad7910f0960 Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Thu, 26 Jun 2025 11:09:09 +1000 Subject: [PATCH 09/12] tweak --- .../java/org/elasticsearch/snapshots/SnapshotsService.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java index ca9ca187c594f..59602cef099b6 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java @@ -808,10 +808,8 @@ private static UpdatedShardGenerations buildGenerations(SnapshotsInProgress.Entr private Metadata metadataForSnapshot(SnapshotsInProgress.Entry snapshot, Metadata metadata, ProjectId projectId) { final ProjectMetadata snapshotProject = projectForSnapshot(snapshot, metadata.getProject(projectId)); final Metadata.Builder builder; - if (snapshot.includeGlobalState() == false || serializeProjectMetadata) { - // Remove global state from the cluster state when - // 1. The request explicitly demands that - // 2. The snapshot is for a project in a multi-project cluster. Such a snapshot must not include cluster level info + if (snapshot.includeGlobalState() == false) { + // Remove global state from the cluster state builder = Metadata.builder(); } else { builder = Metadata.builder(metadata); @@ -1479,6 +1477,7 @@ private void finalizeSnapshotEntry(Snapshot snapshot, Metadata metadata, Reposit final ProjectId projectId = snapshot.getProjectId(); final Metadata effectiveMetadata; if (serializeProjectMetadata) { + // If we are serializing ProjectMetadata (i.e. multi-project enabled), capture only the ProjectMetadata with an empty Metadata effectiveMetadata = Metadata.builder().put(metadata.getProject(projectId)).build(); } else { effectiveMetadata = metadata; From b846483002cfc41fefd83a2e16bd7cc72de8e555 Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Thu, 26 Jun 2025 13:10:33 +1000 Subject: [PATCH 10/12] tighten assertions --- .../org/elasticsearch/snapshots/SnapshotsService.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java index 59602cef099b6..6d06e2b644275 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java @@ -988,19 +988,19 @@ private boolean assertConsistentWithClusterState(ClusterState state) { private static boolean assertNoDanglingSnapshots(ClusterState state) { final SnapshotsInProgress snapshotsInProgress = SnapshotsInProgress.get(state); final SnapshotDeletionsInProgress snapshotDeletionsInProgress = SnapshotDeletionsInProgress.get(state); - final Set reposWithRunningDelete = snapshotDeletionsInProgress.getEntries() + final Set reposWithRunningDelete = snapshotDeletionsInProgress.getEntries() .stream() .filter(entry -> entry.state() == SnapshotDeletionsInProgress.State.STARTED) - .map(SnapshotDeletionsInProgress.Entry::repository) + .map(entry -> new ProjectRepo(entry.projectId(), entry.repository())) .collect(Collectors.toSet()); for (List repoEntry : snapshotsInProgress.entriesByRepo()) { final SnapshotsInProgress.Entry entry = repoEntry.get(0); for (ShardSnapshotStatus value : entry.shardSnapshotStatusByRepoShardId().values()) { if (value.equals(ShardSnapshotStatus.UNASSIGNED_QUEUED)) { - assert reposWithRunningDelete.contains(entry.repository()) + assert reposWithRunningDelete.contains(new ProjectRepo(entry.projectId(), entry.repository())) : "Found shard snapshot waiting to be assigned in [" + entry + "] but it is not blocked by any running delete"; } else if (value.isActive()) { - assert reposWithRunningDelete.contains(entry.repository()) == false + assert reposWithRunningDelete.contains(new ProjectRepo(entry.projectId(), entry.repository())) == false : "Found shard snapshot actively executing in [" + entry + "] when it should be blocked by a running delete [" From 89b4c0045ea6e739b523f902bec200e659df8af9 Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Fri, 27 Jun 2025 13:01:10 +1000 Subject: [PATCH 11/12] tweak --- .../TransportCleanupRepositoryAction.java | 2 +- .../get/TransportGetSnapshotsAction.java | 24 +++++++------------ .../cluster/SnapshotDeletionsInProgress.java | 5 +++- .../blobstore/BlobStoreRepository.java | 4 ++-- .../snapshots/SnapshotsService.java | 2 +- 5 files changed, 17 insertions(+), 20 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/cleanup/TransportCleanupRepositoryAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/cleanup/TransportCleanupRepositoryAction.java index 9dda15a21ce30..a7c452cd3e43c 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/cleanup/TransportCleanupRepositoryAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/cleanup/TransportCleanupRepositoryAction.java @@ -144,7 +144,7 @@ protected void masterOperation( @Override protected ClusterBlockException checkBlock(CleanupRepositoryRequest request, ClusterState state) { // Cluster is not affected but we look up repositories in metadata - return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_READ); + return state.blocks().globalBlockedException(projectResolver.getProjectId(), ClusterBlockLevel.METADATA_READ); } /** diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java index a98e364bb3d56..d542d619d77b3 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java @@ -503,21 +503,15 @@ private Iterator getAsyncSnapshotInfoIterator(Repository repo final var indicesLookup = getIndicesLookup(repositoryData); return Iterators.concat( // matching in-progress snapshots first - Iterators.map( - Iterators.filter( - snapshotsInProgress.forRepo(repository.getProjectId(), repository.getMetadata().name()).iterator(), - snapshotInProgress -> { - final var snapshotId = snapshotInProgress.snapshot().getSnapshotId(); - if (snapshotNamePredicate.test(snapshotId.getName(), true)) { - matchingInProgressSnapshots.add(snapshotId); - return true; - } else { - return false; - } - } - ), - this::forSnapshotInProgress - ), + Iterators.map(Iterators.filter(snapshotsInProgress.forRepo(repository.getProjectRepo()).iterator(), snapshotInProgress -> { + final var snapshotId = snapshotInProgress.snapshot().getSnapshotId(); + if (snapshotNamePredicate.test(snapshotId.getName(), true)) { + matchingInProgressSnapshots.add(snapshotId); + return true; + } else { + return false; + } + }), this::forSnapshotInProgress), repositoryData == null // Only returning in-progress snapshots: ? Collections.emptyIterator() diff --git a/server/src/main/java/org/elasticsearch/cluster/SnapshotDeletionsInProgress.java b/server/src/main/java/org/elasticsearch/cluster/SnapshotDeletionsInProgress.java index f16e03a7e9594..583c2b8b5deb1 100644 --- a/server/src/main/java/org/elasticsearch/cluster/SnapshotDeletionsInProgress.java +++ b/server/src/main/java/org/elasticsearch/cluster/SnapshotDeletionsInProgress.java @@ -135,8 +135,11 @@ public boolean hasDeletionsInProgress() { return entries.isEmpty() == false; } + /** + * Similar to {@link #hasDeletionsInProgress()} but checks in the scope of the given project. + */ public boolean hasDeletionsInProgress(ProjectId projectId) { - return entries.stream().filter(entry -> entry.projectId().equals(projectId)).findFirst().isPresent(); + return entries.stream().anyMatch(entry -> entry.projectId().equals(projectId)); } @Override diff --git a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java index 1bcdcc823b2a4..dea02908d215f 100644 --- a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java +++ b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java @@ -735,7 +735,7 @@ public void updateState(ClusterState state) { return; } if (bestEffortConsistency) { - long bestGenerationFromCS = bestGeneration(SnapshotsInProgress.get(state).forRepo(getProjectId(), this.metadata.name())); + long bestGenerationFromCS = bestGeneration(SnapshotsInProgress.get(state).forRepo(getProjectRepo())); // Don't use generation from the delete task if we already found a generation for an in progress snapshot. // In this case, the generation points at the generation the repo will be in after the snapshot finishes so it may not yet // exist @@ -3125,7 +3125,7 @@ private ClusterState updateRepositoryGenerationsIfNecessary(ClusterState state, boolean changedDeletions = false; final List deletionEntries = new ArrayList<>(); for (SnapshotDeletionsInProgress.Entry entry : SnapshotDeletionsInProgress.get(state).getEntries()) { - if (entry.repository().equals(repoName) && entry.repositoryStateId() == oldGen) { + if (entry.projectId().equals(getProjectId()) && entry.repository().equals(repoName) && entry.repositoryStateId() == oldGen) { deletionEntries.add(entry.withRepoGen(newGen)); changedDeletions = true; } else { diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java index 6d06e2b644275..68eb8bd8cf2cd 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java @@ -707,7 +707,7 @@ private void runReadyClone( } } - // This is limit is cluster wide across all projects + // This limit is intentionally cluster wide across all projects private void ensureBelowConcurrencyLimit( String repository, String name, From e1e6ef29db16c551c7e9bcafcc85006d608d8a7d Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Thu, 3 Jul 2025 12:30:49 +1000 Subject: [PATCH 12/12] use slash to separate project and repo name --- .../snapshots/CorruptedBlobStoreRepositoryIT.java | 4 ++-- .../org/elasticsearch/snapshots/SnapshotThrottlingIT.java | 4 ++-- .../org/elasticsearch/snapshots/SnapshotsServiceIT.java | 8 ++++---- .../java/org/elasticsearch/repositories/ProjectRepo.java | 2 +- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/CorruptedBlobStoreRepositoryIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/CorruptedBlobStoreRepositoryIT.java index dcc4a49df8bd5..7d6c3b2d2296a 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/CorruptedBlobStoreRepositoryIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/CorruptedBlobStoreRepositoryIT.java @@ -820,7 +820,7 @@ public void testSnapshotWithMissingShardLevelIndexFile() throws Exception { "fallback message", "org.elasticsearch.repositories.blobstore.BlobStoreRepository", Level.ERROR, - "index [test-idx-1/*] shard generation [*] in [default][test-repo][*] not found " + "index [test-idx-1/*] shard generation [*] in [default/test-repo][*] not found " + "- falling back to reading all shard snapshots" ) ); @@ -829,7 +829,7 @@ public void testSnapshotWithMissingShardLevelIndexFile() throws Exception { "shard blobs list", "org.elasticsearch.repositories.blobstore.BlobStoreRepository", Level.ERROR, - "read shard snapshots [*] due to missing shard generation [*] for index [test-idx-1/*] in [default][test-repo][*]" + "read shard snapshots [*] due to missing shard generation [*] for index [test-idx-1/*] in [default/test-repo][*]" ) ); if (repairWithDelete) { diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotThrottlingIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotThrottlingIT.java index d5dba02f14855..80ade63624675 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotThrottlingIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotThrottlingIT.java @@ -141,7 +141,7 @@ public void testWarningSpeedOverRecovery() throws Exception { "snapshot speed over recovery speed", "org.elasticsearch.repositories.blobstore.BlobStoreRepository", Level.WARN, - "repository [default][test-repo] has a rate limit [max_snapshot_bytes_per_sec=1gb] per second which is above " + "repository [default/test-repo] has a rate limit [max_snapshot_bytes_per_sec=1gb] per second which is above " + "the effective recovery rate limit [indices.recovery.max_bytes_per_sec=100mb] per second, thus the repository " + "rate limit will be superseded by the recovery rate limit" ); @@ -152,7 +152,7 @@ public void testWarningSpeedOverRecovery() throws Exception { "snapshot restore speed over recovery speed", "org.elasticsearch.repositories.blobstore.BlobStoreRepository", Level.WARN, - "repository [default][test-repo] has a rate limit [max_restore_bytes_per_sec=2gb] per second which is above " + "repository [default/test-repo] has a rate limit [max_restore_bytes_per_sec=2gb] per second which is above " + "the effective recovery rate limit [indices.recovery.max_bytes_per_sec=100mb] per second, thus the repository " + "rate limit will be superseded by the recovery rate limit" ); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotsServiceIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotsServiceIT.java index 5bd242322efaa..16b3a957f9113 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotsServiceIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotsServiceIT.java @@ -44,7 +44,7 @@ public void testDeletingSnapshotsIsLoggedAfterClusterStateIsProcessed() throws E "[does-not-exist]", SnapshotsService.class.getName(), Level.INFO, - "deleting snapshots [does-not-exist] from repository [default][test-repo]" + "deleting snapshots [does-not-exist] from repository [default/test-repo]" ) ); @@ -53,7 +53,7 @@ public void testDeletingSnapshotsIsLoggedAfterClusterStateIsProcessed() throws E "[deleting test-snapshot]", SnapshotsService.class.getName(), Level.INFO, - "deleting snapshots [test-snapshot] from repository [default][test-repo]" + "deleting snapshots [test-snapshot] from repository [default/test-repo]" ) ); @@ -62,7 +62,7 @@ public void testDeletingSnapshotsIsLoggedAfterClusterStateIsProcessed() throws E "[test-snapshot deleted]", SnapshotsService.class.getName(), Level.INFO, - "snapshots [test-snapshot/*] deleted in repository [default][test-repo]" + "snapshots [test-snapshot/*] deleted in repository [default/test-repo]" ) ); @@ -91,7 +91,7 @@ public void testSnapshotDeletionFailureShouldBeLogged() throws Exception { "[test-snapshot]", SnapshotsService.class.getName(), Level.WARN, - "failed to complete snapshot deletion for [test-snapshot] from repository [default][test-repo]" + "failed to complete snapshot deletion for [test-snapshot] from repository [default/test-repo]" ) ); diff --git a/server/src/main/java/org/elasticsearch/repositories/ProjectRepo.java b/server/src/main/java/org/elasticsearch/repositories/ProjectRepo.java index 49884d6905c1b..18d4578b14dde 100644 --- a/server/src/main/java/org/elasticsearch/repositories/ProjectRepo.java +++ b/server/src/main/java/org/elasticsearch/repositories/ProjectRepo.java @@ -52,6 +52,6 @@ public String toString() { } public static String projectRepoString(ProjectId projectId, String repositoryName) { - return "[" + projectId + "][" + repositoryName + "]"; + return "[" + projectId + "/" + repositoryName + "]"; } }