diff --git a/changelog/unreleased/solr-18011-locking-update.yml b/changelog/unreleased/solr-18011-locking-update.yml new file mode 100644 index 00000000000..a19faee91d9 --- /dev/null +++ b/changelog/unreleased/solr-18011-locking-update.yml @@ -0,0 +1,10 @@ +# See https://github.com/apache/solr/blob/main/dev-docs/changelog.adoc +title: Allow locked Admin APIs to call other locked AdminAPIs without deadlocking +type: changed # added, changed, fixed, deprecated, removed, dependency_update, security, other +authors: + - name: Houston Putman + nick: HoustonPutman + url: https://home.apache.org/phonebook.html?uid=houston +links: + - name: SOLR-18011 + url: https://issues.apache.org/jira/browse/SOLR-18011 diff --git a/solr/api/src/java/org/apache/solr/client/api/endpoint/AddReplicaPropertyApi.java b/solr/api/src/java/org/apache/solr/client/api/endpoint/AddReplicaPropertyApi.java index 8e4d53e521a..997613e292f 100644 --- a/solr/api/src/java/org/apache/solr/client/api/endpoint/AddReplicaPropertyApi.java +++ b/solr/api/src/java/org/apache/solr/client/api/endpoint/AddReplicaPropertyApi.java @@ -24,7 +24,7 @@ import jakarta.ws.rs.Path; import jakarta.ws.rs.PathParam; import org.apache.solr.client.api.model.AddReplicaPropertyRequestBody; -import org.apache.solr.client.api.model.SolrJerseyResponse; +import org.apache.solr.client.api.model.SubResponseAccumulatingJerseyResponse; @Path("/collections/{collName}/shards/{shardName}/replicas/{replicaName}/properties/{propName}") public interface AddReplicaPropertyApi { @@ -33,7 +33,7 @@ public interface AddReplicaPropertyApi { @Operation( summary = "Adds a property to the specified replica", tags = {"replica-properties"}) - public SolrJerseyResponse addReplicaProperty( + public SubResponseAccumulatingJerseyResponse addReplicaProperty( @Parameter( description = "The name of the collection the replica belongs to.", required = true) diff --git a/solr/api/src/java/org/apache/solr/client/api/endpoint/AliasPropertyApis.java b/solr/api/src/java/org/apache/solr/client/api/endpoint/AliasPropertyApis.java index 2c1f2ae8267..1710e2e0e48 100644 --- a/solr/api/src/java/org/apache/solr/client/api/endpoint/AliasPropertyApis.java +++ b/solr/api/src/java/org/apache/solr/client/api/endpoint/AliasPropertyApis.java @@ -26,7 +26,7 @@ import jakarta.ws.rs.PathParam; import org.apache.solr.client.api.model.GetAliasPropertyResponse; import org.apache.solr.client.api.model.GetAllAliasPropertiesResponse; -import org.apache.solr.client.api.model.SolrJerseyResponse; +import org.apache.solr.client.api.model.SubResponseAccumulatingJerseyResponse; import org.apache.solr.client.api.model.UpdateAliasPropertiesRequestBody; import org.apache.solr.client.api.model.UpdateAliasPropertyRequestBody; @@ -56,7 +56,7 @@ GetAliasPropertyResponse getAliasProperty( @Operation( summary = "Update properties for a collection alias.", tags = {"alias-properties"}) - SolrJerseyResponse updateAliasProperties( + SubResponseAccumulatingJerseyResponse updateAliasProperties( @Parameter(description = "Alias Name") @PathParam("aliasName") String aliasName, @RequestBody(description = "Properties that need to be updated", required = true) UpdateAliasPropertiesRequestBody requestBody) @@ -67,7 +67,7 @@ SolrJerseyResponse updateAliasProperties( @Operation( summary = "Update a specific property for a collection alias.", tags = {"alias-properties"}) - SolrJerseyResponse createOrUpdateAliasProperty( + SubResponseAccumulatingJerseyResponse createOrUpdateAliasProperty( @Parameter(description = "Alias Name") @PathParam("aliasName") String aliasName, @Parameter(description = "Property Name") @PathParam("propName") String propName, @RequestBody(description = "Property value that needs to be updated", required = true) @@ -79,7 +79,7 @@ SolrJerseyResponse createOrUpdateAliasProperty( @Operation( summary = "Delete a specific property for a collection alias.", tags = {"alias-properties"}) - SolrJerseyResponse deleteAliasProperty( + SubResponseAccumulatingJerseyResponse deleteAliasProperty( @Parameter(description = "Alias Name") @PathParam("aliasName") String aliasName, @Parameter(description = "Property Name") @PathParam("propName") String propName) throws Exception; diff --git a/solr/api/src/java/org/apache/solr/client/api/endpoint/BalanceReplicasApi.java b/solr/api/src/java/org/apache/solr/client/api/endpoint/BalanceReplicasApi.java index 9cbfa5bf3a8..3fd863d1e0a 100644 --- a/solr/api/src/java/org/apache/solr/client/api/endpoint/BalanceReplicasApi.java +++ b/solr/api/src/java/org/apache/solr/client/api/endpoint/BalanceReplicasApi.java @@ -21,7 +21,7 @@ import jakarta.ws.rs.POST; import jakarta.ws.rs.Path; import org.apache.solr.client.api.model.BalanceReplicasRequestBody; -import org.apache.solr.client.api.model.SolrJerseyResponse; +import org.apache.solr.client.api.model.SubResponseAccumulatingJerseyResponse; @Path("cluster/replicas/balance") public interface BalanceReplicasApi { @@ -29,7 +29,7 @@ public interface BalanceReplicasApi { @Operation( summary = "Balance Replicas across the given set of Nodes.", tags = {"cluster"}) - SolrJerseyResponse balanceReplicas( + SubResponseAccumulatingJerseyResponse balanceReplicas( @RequestBody(description = "Contains user provided parameters") BalanceReplicasRequestBody requestBody) throws Exception; diff --git a/solr/api/src/java/org/apache/solr/client/api/endpoint/CreateAliasApi.java b/solr/api/src/java/org/apache/solr/client/api/endpoint/CreateAliasApi.java index 78b9b4376c0..b5bfc5afca1 100644 --- a/solr/api/src/java/org/apache/solr/client/api/endpoint/CreateAliasApi.java +++ b/solr/api/src/java/org/apache/solr/client/api/endpoint/CreateAliasApi.java @@ -20,7 +20,7 @@ import jakarta.ws.rs.POST; import jakarta.ws.rs.Path; import org.apache.solr.client.api.model.CreateAliasRequestBody; -import org.apache.solr.client.api.model.SolrJerseyResponse; +import org.apache.solr.client.api.model.SubResponseAccumulatingJerseyResponse; @Path("/aliases") public interface CreateAliasApi { @@ -28,5 +28,6 @@ public interface CreateAliasApi { @Operation( summary = "Create a traditional or 'routed' alias", tags = {"aliases"}) - SolrJerseyResponse createAlias(CreateAliasRequestBody requestBody) throws Exception; + SubResponseAccumulatingJerseyResponse createAlias(CreateAliasRequestBody requestBody) + throws Exception; } diff --git a/solr/api/src/java/org/apache/solr/client/api/endpoint/DeleteAliasApi.java b/solr/api/src/java/org/apache/solr/client/api/endpoint/DeleteAliasApi.java index badaea3c117..50ec9e67b0b 100644 --- a/solr/api/src/java/org/apache/solr/client/api/endpoint/DeleteAliasApi.java +++ b/solr/api/src/java/org/apache/solr/client/api/endpoint/DeleteAliasApi.java @@ -23,7 +23,7 @@ import jakarta.ws.rs.Path; import jakarta.ws.rs.PathParam; import jakarta.ws.rs.QueryParam; -import org.apache.solr.client.api.model.SolrJerseyResponse; +import org.apache.solr.client.api.model.SubResponseAccumulatingJerseyResponse; @Path("/aliases/{aliasName}") public interface DeleteAliasApi { @@ -32,7 +32,7 @@ public interface DeleteAliasApi { @Operation( summary = "Deletes an alias by its name", tags = {"aliases"}) - SolrJerseyResponse deleteAlias( + SubResponseAccumulatingJerseyResponse deleteAlias( @Parameter(description = "The name of the alias to delete", required = true) @PathParam("aliasName") String aliasName, diff --git a/solr/api/src/java/org/apache/solr/client/api/endpoint/DeleteNodeApi.java b/solr/api/src/java/org/apache/solr/client/api/endpoint/DeleteNodeApi.java index 64b5978afd2..b42491b335f 100644 --- a/solr/api/src/java/org/apache/solr/client/api/endpoint/DeleteNodeApi.java +++ b/solr/api/src/java/org/apache/solr/client/api/endpoint/DeleteNodeApi.java @@ -24,7 +24,7 @@ import jakarta.ws.rs.Path; import jakarta.ws.rs.PathParam; import org.apache.solr.client.api.model.DeleteNodeRequestBody; -import org.apache.solr.client.api.model.SolrJerseyResponse; +import org.apache.solr.client.api.model.SubResponseAccumulatingJerseyResponse; @Path("cluster/nodes/{nodeName}/clear/") public interface DeleteNodeApi { @@ -33,7 +33,7 @@ public interface DeleteNodeApi { @Operation( summary = "Delete all replicas off of the specified SolrCloud node", tags = {"node"}) - SolrJerseyResponse deleteNode( + SubResponseAccumulatingJerseyResponse deleteNode( @Parameter( description = "The name of the node to be cleared. Usually of the form 'host:1234_solr'.", diff --git a/solr/api/src/java/org/apache/solr/client/api/endpoint/InstallShardDataApi.java b/solr/api/src/java/org/apache/solr/client/api/endpoint/InstallShardDataApi.java index f38c2bd00ee..e91c57054bf 100644 --- a/solr/api/src/java/org/apache/solr/client/api/endpoint/InstallShardDataApi.java +++ b/solr/api/src/java/org/apache/solr/client/api/endpoint/InstallShardDataApi.java @@ -21,7 +21,7 @@ import jakarta.ws.rs.Path; import jakarta.ws.rs.PathParam; import org.apache.solr.client.api.model.InstallShardDataRequestBody; -import org.apache.solr.client.api.model.SolrJerseyResponse; +import org.apache.solr.client.api.model.SubResponseAccumulatingJerseyResponse; /** V2 API definition allowing users to import offline-constructed index into a shard. */ @Path("/collections/{collName}/shards/{shardName}/install") @@ -30,7 +30,7 @@ public interface InstallShardDataApi { @Operation( summary = "Install offline index into an existing shard", tags = {"shards"}) - SolrJerseyResponse installShardData( + SubResponseAccumulatingJerseyResponse installShardData( @PathParam("collName") String collName, @PathParam("shardName") String shardName, InstallShardDataRequestBody requestBody) diff --git a/solr/api/src/java/org/apache/solr/client/api/endpoint/MigrateReplicasApi.java b/solr/api/src/java/org/apache/solr/client/api/endpoint/MigrateReplicasApi.java index 566be87b76b..10e918ba9ef 100644 --- a/solr/api/src/java/org/apache/solr/client/api/endpoint/MigrateReplicasApi.java +++ b/solr/api/src/java/org/apache/solr/client/api/endpoint/MigrateReplicasApi.java @@ -21,7 +21,7 @@ import jakarta.ws.rs.POST; import jakarta.ws.rs.Path; import org.apache.solr.client.api.model.MigrateReplicasRequestBody; -import org.apache.solr.client.api.model.SolrJerseyResponse; +import org.apache.solr.client.api.model.SubResponseAccumulatingJerseyResponse; /** V2 API definition for migrating replicas from a set of nodes to another set of nodes. */ @Path("cluster/replicas/migrate") @@ -30,7 +30,7 @@ public interface MigrateReplicasApi { @Operation( summary = "Migrate Replicas from a given set of nodes.", tags = {"cluster"}) - SolrJerseyResponse migrateReplicas( + SubResponseAccumulatingJerseyResponse migrateReplicas( @RequestBody(description = "Contains user provided parameters", required = true) MigrateReplicasRequestBody requestBody) throws Exception; diff --git a/solr/api/src/java/org/apache/solr/client/api/endpoint/ReplaceNodeApi.java b/solr/api/src/java/org/apache/solr/client/api/endpoint/ReplaceNodeApi.java index cee4032a196..cb3c577783b 100644 --- a/solr/api/src/java/org/apache/solr/client/api/endpoint/ReplaceNodeApi.java +++ b/solr/api/src/java/org/apache/solr/client/api/endpoint/ReplaceNodeApi.java @@ -23,7 +23,7 @@ import jakarta.ws.rs.Path; import jakarta.ws.rs.PathParam; import org.apache.solr.client.api.model.ReplaceNodeRequestBody; -import org.apache.solr.client.api.model.SolrJerseyResponse; +import org.apache.solr.client.api.model.SubResponseAccumulatingJerseyResponse; /** * V2 API definition for recreating replicas in one node (the source) on another node(s) (the @@ -35,7 +35,7 @@ public interface ReplaceNodeApi { @Operation( summary = "'Replace' a specified node by moving all replicas elsewhere", tags = {"node"}) - SolrJerseyResponse replaceNode( + SubResponseAccumulatingJerseyResponse replaceNode( @Parameter(description = "The name of the node to be replaced.", required = true) @PathParam("sourceNodeName") String sourceNodeName, diff --git a/solr/api/src/java/org/apache/solr/client/api/model/CreateCollectionSnapshotResponse.java b/solr/api/src/java/org/apache/solr/client/api/model/CreateCollectionSnapshotResponse.java index 16944706498..0392e0ed1ce 100644 --- a/solr/api/src/java/org/apache/solr/client/api/model/CreateCollectionSnapshotResponse.java +++ b/solr/api/src/java/org/apache/solr/client/api/model/CreateCollectionSnapshotResponse.java @@ -19,7 +19,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import io.swagger.v3.oas.annotations.media.Schema; -public class CreateCollectionSnapshotResponse extends AsyncJerseyResponse { +public class CreateCollectionSnapshotResponse extends SubResponseAccumulatingJerseyResponse { @Schema(description = "The name of the collection.") public String collection; diff --git a/solr/api/src/java/org/apache/solr/client/api/model/DeleteCollectionSnapshotResponse.java b/solr/api/src/java/org/apache/solr/client/api/model/DeleteCollectionSnapshotResponse.java index 905b0937f1d..d822e2ae089 100644 --- a/solr/api/src/java/org/apache/solr/client/api/model/DeleteCollectionSnapshotResponse.java +++ b/solr/api/src/java/org/apache/solr/client/api/model/DeleteCollectionSnapshotResponse.java @@ -23,7 +23,7 @@ import io.swagger.v3.oas.annotations.media.Schema; /** The Response for {@link org.apache.solr.client.api.endpoint.CollectionSnapshotApis.Delete} */ -public class DeleteCollectionSnapshotResponse extends AsyncJerseyResponse { +public class DeleteCollectionSnapshotResponse extends SubResponseAccumulatingJerseyResponse { @Schema(description = "The name of the collection.") @JsonProperty(COLLECTION) public String collection; diff --git a/solr/api/src/java/org/apache/solr/client/api/model/InstallShardDataRequestBody.java b/solr/api/src/java/org/apache/solr/client/api/model/InstallShardDataRequestBody.java index 31bec8eb434..05b27f1dcab 100644 --- a/solr/api/src/java/org/apache/solr/client/api/model/InstallShardDataRequestBody.java +++ b/solr/api/src/java/org/apache/solr/client/api/model/InstallShardDataRequestBody.java @@ -24,5 +24,9 @@ public class InstallShardDataRequestBody { @JsonProperty public String repository; + @JsonProperty public String name; + + @JsonProperty public String shardBackupId; + @JsonProperty public String async; } diff --git a/solr/core/src/java/org/apache/solr/api/V2HttpCall.java b/solr/core/src/java/org/apache/solr/api/V2HttpCall.java index 51d14b65993..8246bcfa39c 100644 --- a/solr/core/src/java/org/apache/solr/api/V2HttpCall.java +++ b/solr/core/src/java/org/apache/solr/api/V2HttpCall.java @@ -18,6 +18,7 @@ package org.apache.solr.api; import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP; +import static org.apache.solr.common.params.CollectionAdminParams.CALLING_LOCK_IDS_HEADER; import static org.apache.solr.servlet.SolrDispatchFilter.Action.ADMIN; import static org.apache.solr.servlet.SolrDispatchFilter.Action.ADMIN_OR_REMOTEPROXY; import static org.apache.solr.servlet.SolrDispatchFilter.Action.PROCESS; @@ -216,6 +217,11 @@ private void initAdminRequest(String path) throws Exception { solrReq.getContext().put(CoreContainer.class.getName(), cores); requestType = AuthorizationContext.RequestType.ADMIN; action = ADMIN; + + String callingLockIds = req.getHeader(CALLING_LOCK_IDS_HEADER); + if (callingLockIds != null && !callingLockIds.isBlank()) { + solrReq.getContext().put(CALLING_LOCK_IDS_HEADER, callingLockIds); + } } protected void parseRequest() throws Exception { diff --git a/solr/core/src/java/org/apache/solr/cloud/DistributedCollectionLockFactory.java b/solr/core/src/java/org/apache/solr/cloud/DistributedCollectionLockFactory.java index e49aba6c3f5..ec6fa179aeb 100644 --- a/solr/core/src/java/org/apache/solr/cloud/DistributedCollectionLockFactory.java +++ b/solr/core/src/java/org/apache/solr/cloud/DistributedCollectionLockFactory.java @@ -17,6 +17,7 @@ package org.apache.solr.cloud; +import java.util.List; import org.apache.solr.cloud.api.collections.CollectionApiLockFactory; import org.apache.solr.common.params.CollectionParams; @@ -62,5 +63,6 @@ DistributedLock createLock( CollectionParams.LockLevel level, String collName, String shardId, - String replicaName); + String replicaName, + List callingLockIds); } diff --git a/solr/core/src/java/org/apache/solr/cloud/DistributedLock.java b/solr/core/src/java/org/apache/solr/cloud/DistributedLock.java index 1929766e86e..c26c5d499f0 100644 --- a/solr/core/src/java/org/apache/solr/cloud/DistributedLock.java +++ b/solr/core/src/java/org/apache/solr/cloud/DistributedLock.java @@ -24,4 +24,8 @@ public interface DistributedLock { void release(); boolean isAcquired(); + + String getLockId(); + + boolean isMirroringLock(); } diff --git a/solr/core/src/java/org/apache/solr/cloud/DistributedMultiLock.java b/solr/core/src/java/org/apache/solr/cloud/DistributedMultiLock.java index 9979c144e84..ef74b9862c8 100644 --- a/solr/core/src/java/org/apache/solr/cloud/DistributedMultiLock.java +++ b/solr/core/src/java/org/apache/solr/cloud/DistributedMultiLock.java @@ -20,6 +20,7 @@ import com.google.common.annotations.VisibleForTesting; import java.lang.invoke.MethodHandles; import java.util.List; +import java.util.stream.Collectors; import org.apache.solr.common.SolrException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -46,7 +47,12 @@ public void waitUntilAcquired() { for (DistributedLock lock : locks) { log.debug("DistributedMultiLock.waitUntilAcquired. About to wait on lock {}", lock); lock.waitUntilAcquired(); - log.debug("DistributedMultiLock.waitUntilAcquired. Acquired lock {}", lock); + if (lock.isMirroringLock()) { + log.debug( + "DistributedMultiLock.waitUntilAcquired. Mirroring already-acquired lock {}", lock); + } else { + log.debug("DistributedMultiLock.waitUntilAcquired. Acquired lock {}", lock); + } } } @@ -70,6 +76,10 @@ public boolean isAcquired() { return true; } + public String getLockId() { + return locks.stream().map(DistributedLock::getLockId).collect(Collectors.joining(",")); + } + @VisibleForTesting public int getCountInternalLocks() { return locks.size(); diff --git a/solr/core/src/java/org/apache/solr/cloud/LockTree.java b/solr/core/src/java/org/apache/solr/cloud/LockTree.java index e8d96d4f2cd..7940b07ef7d 100644 --- a/solr/core/src/java/org/apache/solr/cloud/LockTree.java +++ b/solr/core/src/java/org/apache/solr/cloud/LockTree.java @@ -22,6 +22,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.UUID; import org.apache.solr.cloud.OverseerMessageHandler.Lock; import org.apache.solr.common.params.CollectionParams; import org.apache.solr.common.params.CollectionParams.LockLevel; @@ -38,20 +39,35 @@ public class LockTree { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); private final Node root = new Node(null, LockLevel.CLUSTER, null); + public final Map allLocks = new HashMap<>(); + private class LockImpl implements Lock { final Node node; + final String id; LockImpl(Node node) { this.node = node; + this.id = UUID.randomUUID().toString(); } @Override public void unlock() { synchronized (LockTree.this) { node.unlock(this); + allLocks.remove(id); } } + @Override + public String id() { + return id; + } + + @Override + public boolean validateSubpath(int lockLevel, List path) { + return node.validateSubpath(lockLevel, path); + } + @Override public String toString() { return StrUtils.join(node.constructPath(new ArrayDeque<>()), '/'); @@ -71,12 +87,33 @@ public String toString() { public class Session { private SessionNode root = new SessionNode(LockLevel.CLUSTER); - public Lock lock(CollectionParams.CollectionAction action, List path) { + public Lock lock( + CollectionParams.CollectionAction action, List path, List callingLockIds) { if (action.lockLevel == LockLevel.NONE) return FREELOCK; + log.debug("Calling lock level: {}", callingLockIds); + Node startingNode = LockTree.this.root; + SessionNode startingSession = root; + + // If a callingLockId was passed in, validate it with the current lock path, and only start + // locking below the calling lock + Lock callingLock = callingLockIds != null ? allLocks.get(callingLockIds.getLast()) : null; + boolean ignoreCallingLock = false; + if (callingLock != null && callingLock.validateSubpath(action.lockLevel.getHeight(), path)) { + startingNode = ((LockImpl) callingLock).node; + startingSession = startingSession.find(startingNode.level.getHeight(), path); + if (startingSession == null) { + startingSession = root; + } + ignoreCallingLock = true; + } synchronized (LockTree.this) { - if (root.isBusy(action.lockLevel, path)) return null; - Lock lockObject = LockTree.this.root.lock(action.lockLevel, path); - if (lockObject == null) root.markBusy(action.lockLevel, path); + if (startingSession.isBusy(action.lockLevel, path)) return null; + Lock lockObject = startingNode.lock(action.lockLevel, path, ignoreCallingLock); + if (lockObject == null) { + startingSession.markBusy(action.lockLevel, path); + } else { + allLocks.put(lockObject.id(), lockObject); + } return lockObject; } } @@ -125,6 +162,18 @@ boolean isBusy(LockLevel lockLevel, List path) { return false; } } + + SessionNode find(int lockLevel, List path) { + if (level.getHeight() == lockLevel) { + return this; + } else if (level.getHeight() < lockLevel + && kids != null + && kids.containsKey(path.get(level.getHeight()))) { + return kids.get(path.get(level.getHeight())).find(lockLevel, path); + } else { + return null; + } + } } public Session getSession() { @@ -158,8 +207,9 @@ void unlock(LockImpl lockObject) { } } - Lock lock(LockLevel lockLevel, List path) { - if (myLock != null) return null; // I'm already locked. no need to go any further + Lock lock(LockLevel lockLevel, List path, boolean ignoreCurrentLock) { + if (myLock != null && !ignoreCurrentLock) + return null; // I'm already locked. no need to go any further if (lockLevel == level) { // lock is supposed to be acquired at this level // If I am locked or any of my children or grandchildren are locked @@ -171,10 +221,16 @@ Lock lock(LockLevel lockLevel, List path) { Node child = children.get(childName); if (child == null) children.put(childName, child = new Node(childName, level.getChild(), this)); - return child.lock(lockLevel, path); + return child.lock(lockLevel, path, false); } } + boolean validateSubpath(int lockLevel, List path) { + return level.getHeight() < lockLevel + && (level.getHeight() == 0 || name.equals(path.get(level.getHeight() - 1))) + && (mom == null || mom.validateSubpath(lockLevel, path)); + } + ArrayDeque constructPath(ArrayDeque collect) { if (name != null) collect.addFirst(name); if (mom != null) mom.constructPath(collect); @@ -182,5 +238,20 @@ ArrayDeque constructPath(ArrayDeque collect) { } } - static final Lock FREELOCK = () -> {}; + static final String FREELOCK_ID = "-1"; + static final Lock FREELOCK = + new Lock() { + @Override + public void unlock() {} + + @Override + public String id() { + return FREELOCK_ID; + } + + @Override + public boolean validateSubpath(int lockLevel, List path) { + return false; + } + }; } diff --git a/solr/core/src/java/org/apache/solr/cloud/OverseerConfigSetMessageHandler.java b/solr/core/src/java/org/apache/solr/cloud/OverseerConfigSetMessageHandler.java index 0bf454a0642..849626a30b4 100644 --- a/solr/core/src/java/org/apache/solr/cloud/OverseerConfigSetMessageHandler.java +++ b/solr/core/src/java/org/apache/solr/cloud/OverseerConfigSetMessageHandler.java @@ -21,6 +21,7 @@ import java.lang.invoke.MethodHandles; import java.util.HashSet; +import java.util.List; import java.util.Set; import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrException.ErrorCode; @@ -61,7 +62,7 @@ public OverseerConfigSetMessageHandler(ZkStateReader zkStateReader, CoreContaine } @Override - public OverseerSolrResponse processMessage(ZkNodeProps message, String operation) { + public OverseerSolrResponse processMessage(ZkNodeProps message, String operation, Lock lock) { NamedList results = new NamedList<>(); try { if (!operation.startsWith(CONFIGSETS_ACTION_PREFIX)) { @@ -117,7 +118,22 @@ public Lock lockTask(ZkNodeProps message, long ignored) { String configSetName = getTaskKey(message); if (canExecute(configSetName, message)) { markExclusiveTask(configSetName, message); - return () -> unmarkExclusiveTask(configSetName, message); + return new Lock() { + @Override + public void unlock() { + unmarkExclusiveTask(configSetName, message); + } + + @Override + public String id() { + return configSetName; + } + + @Override + public boolean validateSubpath(int lockLevel, List path) { + return false; + } + }; } return null; } diff --git a/solr/core/src/java/org/apache/solr/cloud/OverseerMessageHandler.java b/solr/core/src/java/org/apache/solr/cloud/OverseerMessageHandler.java index 3e369b90731..4bc13e2fe6a 100644 --- a/solr/core/src/java/org/apache/solr/cloud/OverseerMessageHandler.java +++ b/solr/core/src/java/org/apache/solr/cloud/OverseerMessageHandler.java @@ -16,6 +16,7 @@ */ package org.apache.solr.cloud; +import java.util.List; import org.apache.solr.common.cloud.ZkNodeProps; /** Interface for processing messages received by an {@link OverseerTaskProcessor} */ @@ -26,7 +27,7 @@ public interface OverseerMessageHandler { * @param operation the operation to process * @return response */ - OverseerSolrResponse processMessage(ZkNodeProps message, String operation); + OverseerSolrResponse processMessage(ZkNodeProps message, String operation, Lock lock); /** * @return the name of the OverseerMessageHandler @@ -41,6 +42,10 @@ public interface OverseerMessageHandler { interface Lock { void unlock(); + + String id(); + + boolean validateSubpath(int lockLevel, List path); } /** diff --git a/solr/core/src/java/org/apache/solr/cloud/OverseerTaskProcessor.java b/solr/core/src/java/org/apache/solr/cloud/OverseerTaskProcessor.java index f1fe3c696f9..b44c4f51ce2 100644 --- a/solr/core/src/java/org/apache/solr/cloud/OverseerTaskProcessor.java +++ b/solr/core/src/java/org/apache/solr/cloud/OverseerTaskProcessor.java @@ -562,7 +562,7 @@ public void run() { if (log.isDebugEnabled()) { log.debug("Runner processing {}", head.getId()); } - response = messageHandler.processMessage(message, operation); + response = messageHandler.processMessage(message, operation, lock); } finally { timerContext.stop(); updateStats(statsName); diff --git a/solr/core/src/java/org/apache/solr/cloud/ZkDistributedCollectionLockFactory.java b/solr/core/src/java/org/apache/solr/cloud/ZkDistributedCollectionLockFactory.java index fe77b18c3f2..2e3e5c4b8d7 100644 --- a/solr/core/src/java/org/apache/solr/cloud/ZkDistributedCollectionLockFactory.java +++ b/solr/core/src/java/org/apache/solr/cloud/ZkDistributedCollectionLockFactory.java @@ -17,6 +17,7 @@ package org.apache.solr.cloud; +import java.util.List; import java.util.Objects; import org.apache.solr.cloud.api.collections.DistributedCollectionConfigSetCommandRunner; import org.apache.solr.common.SolrException; @@ -44,7 +45,8 @@ public DistributedLock createLock( CollectionParams.LockLevel level, String collName, String shardId, - String replicaName) { + String replicaName, + List callingLockIds) { Objects.requireNonNull(collName, "collName can't be null"); if (level != CollectionParams.LockLevel.COLLECTION) { Objects.requireNonNull( @@ -56,7 +58,8 @@ public DistributedLock createLock( } String lockPath = getLockPath(level, collName, shardId, replicaName); - return doCreateLock(isWriteLock, lockPath); + + return doCreateLock(isWriteLock, lockPath, callingLockIds); } /** diff --git a/solr/core/src/java/org/apache/solr/cloud/ZkDistributedConfigSetLockFactory.java b/solr/core/src/java/org/apache/solr/cloud/ZkDistributedConfigSetLockFactory.java index 884703a8829..c21d8bdeb78 100644 --- a/solr/core/src/java/org/apache/solr/cloud/ZkDistributedConfigSetLockFactory.java +++ b/solr/core/src/java/org/apache/solr/cloud/ZkDistributedConfigSetLockFactory.java @@ -17,6 +17,7 @@ package org.apache.solr.cloud; +import java.util.Collections; import java.util.Objects; import org.apache.solr.common.cloud.SolrZkClient; @@ -40,7 +41,7 @@ public DistributedLock createLock(boolean isWriteLock, String configSetName) { Objects.requireNonNull(configSetName, "configSetName can't be null"); String lockPath = getLockPath(configSetName); - return doCreateLock(isWriteLock, lockPath); + return doCreateLock(isWriteLock, lockPath, Collections.emptyList()); } /** diff --git a/solr/core/src/java/org/apache/solr/cloud/ZkDistributedLock.java b/solr/core/src/java/org/apache/solr/cloud/ZkDistributedLock.java index 4b594508d2a..8517661dba7 100644 --- a/solr/core/src/java/org/apache/solr/cloud/ZkDistributedLock.java +++ b/solr/core/src/java/org/apache/solr/cloud/ZkDistributedLock.java @@ -41,16 +41,20 @@ abstract class ZkDistributedLock implements DistributedLock { static final char LOCK_PREFIX_SUFFIX = '_'; /** Prefix of EPHEMERAL read lock node names */ - static final String READ_LOCK_PREFIX = "R" + LOCK_PREFIX_SUFFIX; + static final char READ_LOCK_PREFIX_CHAR = 'R'; + + static final String READ_LOCK_PREFIX = "" + READ_LOCK_PREFIX_CHAR + LOCK_PREFIX_SUFFIX; /** Prefix of EPHEMERAL write lock node names */ - static final String WRITE_LOCK_PREFIX = "W" + LOCK_PREFIX_SUFFIX; + static final char WRITE_LOCK_PREFIX_CHAR = 'W'; + + static final String WRITE_LOCK_PREFIX = "" + WRITE_LOCK_PREFIX_CHAR + LOCK_PREFIX_SUFFIX; /** Read lock. */ static class Read extends ZkDistributedLock { - protected Read(SolrZkClient zkClient, String lockPath) + protected Read(SolrZkClient zkClient, String lockPath, String mirroredLockId) throws KeeperException, InterruptedException { - super(zkClient, lockPath, READ_LOCK_PREFIX); + super(zkClient, lockPath, READ_LOCK_PREFIX, mirroredLockId); } @Override @@ -59,13 +63,18 @@ boolean isBlockedByNodeType(String otherLockName) { // Lower numbered read locks are ok, they can coexist. return otherLockName.startsWith(WRITE_LOCK_PREFIX); } + + @Override + boolean canMirrorLock(String lockId) { + return true; + } } /** Write lock. */ static class Write extends ZkDistributedLock { - protected Write(SolrZkClient zkClient, String lockPath) + protected Write(SolrZkClient zkClient, String lockPath, String mirroredLockId) throws KeeperException, InterruptedException { - super(zkClient, lockPath, WRITE_LOCK_PREFIX); + super(zkClient, lockPath, WRITE_LOCK_PREFIX, mirroredLockId); } @Override @@ -73,6 +82,17 @@ boolean isBlockedByNodeType(String otherLockName) { // A write lock is blocked by another read or write lock with a lower sequence number return true; } + + @Override + boolean canMirrorLock(String lockId) { + // Only another Write lock can be mirrored + int lockTypeSuffixIndex = lockId.indexOf(LOCK_PREFIX_SUFFIX) - 1; + if (lockTypeSuffixIndex < 0) { + return false; + } else { + return lockId.charAt(lockTypeSuffixIndex) == WRITE_LOCK_PREFIX_CHAR; + } + } } private final SolrZkClient zkClient; @@ -80,22 +100,35 @@ boolean isBlockedByNodeType(String otherLockName) { private final String lockNode; protected final long sequence; protected volatile boolean released = false; + protected final boolean mirrored; - protected ZkDistributedLock(SolrZkClient zkClient, String lockDir, String lockNodePrefix) + protected ZkDistributedLock( + SolrZkClient zkClient, String lockDir, String lockNodePrefix, String mirroredLockId) throws KeeperException, InterruptedException { this.zkClient = zkClient; this.lockDir = lockDir; // Create the SEQUENTIAL EPHEMERAL node. We enter the locking rat race here. We MUST eventually // call release() or we block others. - lockNode = - zkClient.create( - lockDir - + DistributedCollectionConfigSetCommandRunner.ZK_PATH_SEPARATOR - + lockNodePrefix, - null, - CreateMode.EPHEMERAL_SEQUENTIAL); + if (mirroredLockId == null || !mirroredLockId.startsWith(lockDir)) { + lockNode = + zkClient.create( + lockDir + + DistributedCollectionConfigSetCommandRunner.ZK_PATH_SEPARATOR + + lockNodePrefix, + null, + CreateMode.EPHEMERAL_SEQUENTIAL); + mirrored = false; + } else { + if (!canMirrorLock(mirroredLockId)) { + throw new SolrException( + SolrException.ErrorCode.SERVER_ERROR, + "Cannot mirror lock " + mirroredLockId + " with given lockPrefix: " + lockNodePrefix); + } + lockNode = mirroredLockId; + mirrored = true; + } sequence = getSequenceFromNodename(lockNode); } @@ -158,8 +191,10 @@ public void waitUntilAcquired() { @Override public void release() { try { - zkClient.delete(lockNode, -1); - released = true; + if (!mirrored) { + zkClient.delete(lockNode, -1); + released = true; + } } catch (KeeperException e) { throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e); } catch (InterruptedException e) { @@ -212,7 +247,11 @@ String nodeToWatch() throws KeeperException, InterruptedException { if (!foundSelf) { // If this basic assumption doesn't hold with Zookeeper, we're in deep trouble. And not only // here. - throw new SolrException(SERVER_ERROR, "Missing lock node " + lockNode); + if (mirrored) { + throw new SolrException(SERVER_ERROR, "Missing mirrored lock node " + lockNode); + } else { + throw new SolrException(SERVER_ERROR, "Missing lock node " + lockNode); + } } // Didn't return early on any other blocking lock, means we own it @@ -240,6 +279,18 @@ static long getSequenceFromNodename(String lockNode) { return Long.parseLong(lockNode.substring(lockNode.length() - SEQUENCE_LENGTH)); } + @Override + public String getLockId() { + return lockNode; + } + + @Override + public boolean isMirroringLock() { + return mirrored; + } + + abstract boolean canMirrorLock(String lockId); + @Override public String toString() { return lockNode; diff --git a/solr/core/src/java/org/apache/solr/cloud/ZkDistributedLockFactory.java b/solr/core/src/java/org/apache/solr/cloud/ZkDistributedLockFactory.java index 76696dc3942..f7935a3c011 100644 --- a/solr/core/src/java/org/apache/solr/cloud/ZkDistributedLockFactory.java +++ b/solr/core/src/java/org/apache/solr/cloud/ZkDistributedLockFactory.java @@ -17,6 +17,7 @@ package org.apache.solr.cloud; +import java.util.List; import org.apache.solr.cloud.api.collections.DistributedCollectionConfigSetCommandRunner; import org.apache.solr.common.SolrException; import org.apache.solr.common.cloud.SolrZkClient; @@ -33,16 +34,20 @@ abstract class ZkDistributedLockFactory { this.rootPath = rootPath; } - protected DistributedLock doCreateLock(boolean isWriteLock, String lockPath) { + protected DistributedLock doCreateLock( + boolean isWriteLock, String lockPath, List callingLockIds) { try { // TODO optimize by first attempting to create the ZkDistributedLock without calling // makeLockPath() and only call it if the lock creation fails. This will be less costly on // high contention (and slightly more on low contention) makeLockPath(lockPath); + // If a callingLock has the same lockPath as this lock, we just want to mirror that lock + String mirroredLockId = + callingLockIds.stream().filter(p -> p.startsWith(lockPath)).findFirst().orElse(null); return isWriteLock - ? new ZkDistributedLock.Write(zkClient, lockPath) - : new ZkDistributedLock.Read(zkClient, lockPath); + ? new ZkDistributedLock.Write(zkClient, lockPath, mirroredLockId) + : new ZkDistributedLock.Read(zkClient, lockPath, mirroredLockId); } catch (KeeperException e) { throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e); } catch (InterruptedException e) { diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/AddReplicaCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/AddReplicaCmd.java index a71a515c56d..a13bce718d8 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/AddReplicaCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/AddReplicaCmd.java @@ -25,7 +25,6 @@ import static org.apache.solr.common.params.CollectionAdminParams.COLL_CONF; import static org.apache.solr.common.params.CollectionAdminParams.FOLLOW_ALIASES; import static org.apache.solr.common.params.CollectionParams.CollectionAction.ADDREPLICA; -import static org.apache.solr.common.params.CommonAdminParams.ASYNC; import static org.apache.solr.common.params.CommonAdminParams.TIMEOUT; import static org.apache.solr.common.params.CommonAdminParams.WAIT_FOR_FINAL_STATE; @@ -76,13 +75,13 @@ public AddReplicaCmd(CollectionCommandContext ccc) { } @Override - public void call(ClusterState state, ZkNodeProps message, NamedList results) + public void call(AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList results) throws Exception { - addReplica(state, message, results, null); + addReplica(adminCmdContext, message, results, null); } List addReplica( - ClusterState clusterState, + AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList results, Runnable onComplete) @@ -103,7 +102,7 @@ List addReplica( collectionName = extCollectionName; } - DocCollection coll = clusterState.getCollection(collectionName); + DocCollection coll = adminCmdContext.getClusterState().getCollection(collectionName); if (coll == null) { throw new SolrException( SolrException.ErrorCode.BAD_REQUEST, "Collection: " + collectionName + " does not exist"); @@ -117,7 +116,6 @@ List addReplica( boolean waitForFinalState = message.getBool(WAIT_FOR_FINAL_STATE, false); boolean skipCreateReplicaInClusterState = message.getBool(SKIP_CREATE_REPLICA_IN_CLUSTER_STATE, false); - final String asyncId = message.getStr(ASYNC); String node = message.getStr(CoreAdminParams.NODE); String createNodeSetStr = message.getStr(CREATE_NODE_SET); @@ -152,7 +150,7 @@ List addReplica( List createReplicas = buildReplicaPositions( ccc.getSolrCloudManager(), - clusterState, + adminCmdContext.getClusterState(), collectionName, message, numReplicas, @@ -161,14 +159,17 @@ List addReplica( .map( replicaPosition -> assignReplicaDetails( - ccc.getSolrCloudManager(), clusterState, message, replicaPosition)) + ccc.getSolrCloudManager(), + adminCmdContext.getClusterState(), + message, + replicaPosition)) .collect(Collectors.toList()); ShardHandler shardHandler = ccc.newShardHandler(); ZkStateReader zkStateReader = ccc.getZkStateReader(); final ShardRequestTracker shardRequestTracker = - CollectionHandlingUtils.asyncRequestTracker(asyncId, ccc); + CollectionHandlingUtils.asyncRequestTracker(adminCmdContext, ccc); for (CreateReplica createReplica : createReplicas) { assert createReplica.coreName != null; ModifiableSolrParams params = diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/AdminCmdContext.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/AdminCmdContext.java new file mode 100644 index 00000000000..e0db692bcff --- /dev/null +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/AdminCmdContext.java @@ -0,0 +1,81 @@ +package org.apache.solr.cloud.api.collections; + +import org.apache.solr.common.cloud.ClusterState; +import org.apache.solr.common.params.CollectionParams; +import org.apache.solr.common.util.StrUtils; + +public class AdminCmdContext { + private final CollectionParams.CollectionAction action; + private final String asyncId; + private String lockId; + private String callingLockIds; + private String subRequestCallingLockIds; + private ClusterState clusterState; + + public AdminCmdContext(CollectionParams.CollectionAction action) { + this(action, null); + } + + public AdminCmdContext(CollectionParams.CollectionAction action, String asyncId) { + this.action = action; + this.asyncId = asyncId; + } + + public CollectionParams.CollectionAction getAction() { + return action; + } + + public String getAsyncId() { + return asyncId; + } + + public void setLockId(String lockId) { + this.lockId = lockId; + regenerateSubRequestCallingLockIds(); + } + + public String getLockId() { + return lockId; + } + + public void setCallingLockIds(String callingLockIds) { + this.callingLockIds = callingLockIds; + regenerateSubRequestCallingLockIds(); + } + + private void regenerateSubRequestCallingLockIds() { + subRequestCallingLockIds = callingLockIds; + if (StrUtils.isNotBlank(callingLockIds) && StrUtils.isNotBlank(lockId)) { + subRequestCallingLockIds += ","; + } + subRequestCallingLockIds += lockId; + } + + public String getCallingLockIds() { + return callingLockIds; + } + + public ClusterState getClusterState() { + return clusterState; + } + + public AdminCmdContext withClusterState(ClusterState clusterState) { + this.clusterState = clusterState; + return this; + } + + public String getSubRequestCallingLockIds() { + return subRequestCallingLockIds; + } + + public AdminCmdContext subRequestContext(CollectionParams.CollectionAction action) { + return subRequestContext(action, asyncId); + } + + public AdminCmdContext subRequestContext( + CollectionParams.CollectionAction action, String asyncId) { + AdminCmdContext nextContext = new AdminCmdContext(action, asyncId); + nextContext.setCallingLockIds(subRequestCallingLockIds); + return nextContext.withClusterState(clusterState); + } +} diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/AliasCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/AliasCmd.java index 4815a76b807..ceefa9e889b 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/AliasCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/AliasCmd.java @@ -25,9 +25,9 @@ import java.util.Map; import org.apache.solr.cloud.OverseerSolrResponse; import org.apache.solr.common.SolrException; -import org.apache.solr.common.cloud.ClusterState; import org.apache.solr.common.cloud.CollectionProperties; import org.apache.solr.common.cloud.ZkNodeProps; +import org.apache.solr.common.params.CollectionParams; import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.solr.common.util.NamedList; import org.apache.solr.handler.admin.CollectionsHandler; @@ -52,7 +52,7 @@ protected AliasCmd(CollectionCommandContext ccc) { * If the collection already exists then this is not an error. */ static NamedList createCollectionAndWait( - ClusterState clusterState, + AdminCmdContext adminCmdContext, String aliasName, Map aliasMetadata, String createCollName, @@ -83,7 +83,11 @@ static NamedList createCollectionAndWait( // CreateCollectionCmd. // note: there's doesn't seem to be any point in locking on the collection name, so we don't. // We currently should already have a lock on the alias name which should be sufficient. - new CreateCollectionCmd(ccc).call(clusterState, createMessage, results); + new CreateCollectionCmd(ccc) + .call( + adminCmdContext.subRequestContext(CollectionParams.CollectionAction.CREATE), + createMessage, + results); } catch (SolrException e) { // The collection might already exist, and that's okay -- we can adopt it. if (!e.getMessage().contains("collection already exists")) { diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/BackupCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/BackupCmd.java index 151cbeeee1a..60ec1cb7e6c 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/BackupCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/BackupCmd.java @@ -19,7 +19,6 @@ import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP; import static org.apache.solr.common.cloud.ZkStateReader.CORE_NAME_PROP; import static org.apache.solr.common.params.CollectionAdminParams.FOLLOW_ALIASES; -import static org.apache.solr.common.params.CommonAdminParams.ASYNC; import static org.apache.solr.common.params.CommonParams.NAME; import java.io.IOException; @@ -34,7 +33,6 @@ import org.apache.solr.cloud.api.collections.CollectionHandlingUtils.ShardRequestTracker; import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrException.ErrorCode; -import org.apache.solr.common.cloud.ClusterState; import org.apache.solr.common.cloud.DocCollection; import org.apache.solr.common.cloud.Replica; import org.apache.solr.common.cloud.Replica.State; @@ -71,7 +69,7 @@ public BackupCmd(CollectionCommandContext ccc) { @SuppressWarnings("unchecked") @Override - public void call(ClusterState state, ZkNodeProps message, NamedList results) + public void call(AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList results) throws Exception { String extCollectionName = message.getStr(COLLECTION_PROP); @@ -123,7 +121,13 @@ public void call(ClusterState state, ZkNodeProps message, NamedList resu if (incremental) { try { incrementalCopyIndexFiles( - backupUri, collectionName, message, results, backupProperties, backupMgr); + adminCmdContext, + backupUri, + collectionName, + message, + results, + backupProperties, + backupMgr); } catch (SolrException e) { log.error( "Error happened during incremental backup for collection: {}", @@ -135,7 +139,13 @@ public void call(ClusterState state, ZkNodeProps message, NamedList resu } } else { copyIndexFiles( - backupUri, collectionName, message, results, backupProperties, backupMgr); + adminCmdContext, + backupUri, + collectionName, + message, + results, + backupProperties, + backupMgr); } break; } @@ -288,6 +298,7 @@ private Replica selectReplicaWithSnapshot(CollectionSnapshotMetaData snapshotMet } private void incrementalCopyIndexFiles( + AdminCmdContext adminCmdContext, URI backupUri, String collectionName, ZkNodeProps request, @@ -296,7 +307,6 @@ private void incrementalCopyIndexFiles( BackupManager backupManager) throws IOException { String backupName = request.getStr(NAME); - String asyncId = request.getStr(ASYNC); String repoName = request.getStr(CoreAdminParams.BACKUP_REPOSITORY); ShardHandler shardHandler = ccc.newShardHandler(); @@ -308,7 +318,7 @@ private void incrementalCopyIndexFiles( Optional previousProps = backupManager.tryReadBackupProperties(); final ShardRequestTracker shardRequestTracker = - CollectionHandlingUtils.asyncRequestTracker(asyncId, ccc); + CollectionHandlingUtils.asyncRequestTracker(adminCmdContext, ccc); Collection slices = ccc.getZkStateReader().getClusterState().getCollection(collectionName).getActiveSlices(); @@ -434,6 +444,7 @@ private ModifiableSolrParams coreBackupParams( } private void copyIndexFiles( + AdminCmdContext adminCmdContext, URI backupPath, String collectionName, ZkNodeProps request, @@ -442,7 +453,6 @@ private void copyIndexFiles( BackupManager backupManager) throws Exception { String backupName = request.getStr(NAME); - String asyncId = request.getStr(ASYNC); String repoName = request.getStr(CoreAdminParams.BACKUP_REPOSITORY); ShardHandler shardHandler = ccc.newShardHandler(); @@ -484,7 +494,7 @@ private void copyIndexFiles( } final ShardRequestTracker shardRequestTracker = - CollectionHandlingUtils.asyncRequestTracker(asyncId, ccc); + CollectionHandlingUtils.asyncRequestTracker(adminCmdContext, ccc); Collection slices = ccc.getZkStateReader().getClusterState().getCollection(collectionName).getActiveSlices(); for (Slice slice : slices) { diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/BalanceReplicasCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/BalanceReplicasCmd.java index 7d84c7c6633..9d1acfb27b5 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/BalanceReplicasCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/BalanceReplicasCmd.java @@ -17,15 +17,12 @@ package org.apache.solr.cloud.api.collections; -import static org.apache.solr.common.params.CommonAdminParams.ASYNC; - import java.util.Collection; import java.util.Collections; import java.util.HashSet; import java.util.Map; import java.util.Set; import org.apache.solr.common.SolrException; -import org.apache.solr.common.cloud.ClusterState; import org.apache.solr.common.cloud.Replica; import org.apache.solr.common.cloud.ZkNodeProps; import org.apache.solr.common.params.CollectionParams; @@ -41,7 +38,7 @@ public BalanceReplicasCmd(CollectionCommandContext ccc) { @SuppressWarnings({"unchecked"}) @Override - public void call(ClusterState state, ZkNodeProps message, NamedList results) + public void call(AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList results) throws Exception { Set nodes; Object nodesRaw = message.get(CollectionParams.NODES); @@ -60,7 +57,6 @@ public void call(ClusterState state, ZkNodeProps message, NamedList resu + nodesRaw.getClass().getName()); } boolean waitForFinalState = message.getBool(CommonAdminParams.WAIT_FOR_FINAL_STATE, false); - String async = message.getStr(ASYNC); int timeout = message.getInt("timeout", 10 * 60); // 10 minutes boolean parallel = message.getBool("parallel", false); @@ -79,7 +75,7 @@ public void call(ClusterState state, ZkNodeProps message, NamedList resu boolean migrationSuccessful = ReplicaMigrationUtils.migrateReplicas( - ccc, replicaMovements, parallel, waitForFinalState, timeout, async, results); + ccc, adminCmdContext, replicaMovements, parallel, waitForFinalState, timeout, results); if (migrationSuccessful) { results.add( "success", diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/CollApiCmds.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/CollApiCmds.java index d5bb2d24568..e360012ea01 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/CollApiCmds.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/CollApiCmds.java @@ -65,7 +65,6 @@ import static org.apache.solr.common.params.CollectionParams.CollectionAction.REPLACENODE; import static org.apache.solr.common.params.CollectionParams.CollectionAction.RESTORE; import static org.apache.solr.common.params.CollectionParams.CollectionAction.SPLITSHARD; -import static org.apache.solr.common.params.CommonAdminParams.ASYNC; import static org.apache.solr.common.params.CommonParams.NAME; import io.opentelemetry.api.trace.Span; @@ -81,7 +80,6 @@ import org.apache.solr.cloud.Overseer; import org.apache.solr.cloud.OverseerNodePrioritizer; import org.apache.solr.common.SolrException; -import org.apache.solr.common.cloud.ClusterState; import org.apache.solr.common.cloud.Replica; import org.apache.solr.common.cloud.SolrZkClient; import org.apache.solr.common.cloud.ZkNodeProps; @@ -114,7 +112,8 @@ public class CollApiCmds { * classes whose names ends in {@code Cmd}. */ protected interface CollectionApiCommand { - void call(ClusterState state, ZkNodeProps message, NamedList results) throws Exception; + void call(AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList results) + throws Exception; } /** @@ -206,7 +205,8 @@ public TraceAwareCommand(CollectionApiCommand command) { } @Override - public void call(ClusterState state, ZkNodeProps message, NamedList results) + public void call( + AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList results) throws Exception { final Span localSpan; final Context localContext; @@ -217,7 +217,7 @@ public void call(ClusterState state, ZkNodeProps message, NamedList resu String collection = Optional.ofNullable(message.getStr(COLLECTION_PROP, message.getStr(NAME))) .orElse("unknown"); - boolean isAsync = message.containsKey(ASYNC); + boolean isAsync = adminCmdContext.getAsyncId() != null; localSpan = TraceUtils.startCollectionApiCommandSpan( command.getClass().getSimpleName(), collection, isAsync); @@ -226,7 +226,7 @@ public void call(ClusterState state, ZkNodeProps message, NamedList resu try (var scope = localContext.makeCurrent()) { assert scope != null; // prevent javac warning about scope being unused - command.call(state, message, results); + command.call(adminCmdContext, message, results); } finally { if (localSpan != null) { localSpan.end(); @@ -238,7 +238,8 @@ public void call(ClusterState state, ZkNodeProps message, NamedList resu public static class MockOperationCmd implements CollectionApiCommand { @Override @SuppressForbidden(reason = "Needs currentTimeMillis for mock requests") - public void call(ClusterState state, ZkNodeProps message, NamedList results) + public void call( + AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList results) throws InterruptedException { // only for test purposes Thread.sleep(message.getInt("sleep", 1)); @@ -260,20 +261,19 @@ public ReloadCollectionCmd(CollectionCommandContext ccc) { } @Override - public void call(ClusterState clusterState, ZkNodeProps message, NamedList results) { + public void call( + AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList results) { ModifiableSolrParams params = new ModifiableSolrParams(); params.set(CoreAdminParams.ACTION, CoreAdminParams.CoreAdminAction.RELOAD.toString()); - String asyncId = message.getStr(ASYNC); CollectionHandlingUtils.collectionCmd( + adminCmdContext, message, params, results, Replica.State.ACTIVE, - asyncId, Collections.emptySet(), - ccc, - clusterState); + ccc); } } @@ -285,7 +285,7 @@ public RebalanceLeadersCmd(CollectionCommandContext ccc) { } @Override - public void call(ClusterState clusterState, ZkNodeProps message, NamedList results) + public void call(AdminCmdContext clusterState, ZkNodeProps message, NamedList results) throws Exception { CollectionHandlingUtils.checkRequired( message, @@ -326,7 +326,7 @@ public AddReplicaPropCmd(CollectionCommandContext ccc) { } @Override - public void call(ClusterState clusterState, ZkNodeProps message, NamedList results) + public void call(AdminCmdContext clusterState, ZkNodeProps message, NamedList results) throws Exception { CollectionHandlingUtils.checkRequired( message, @@ -360,7 +360,7 @@ public DeleteReplicaPropCmd(CollectionCommandContext ccc) { } @Override - public void call(ClusterState clusterState, ZkNodeProps message, NamedList results) + public void call(AdminCmdContext clusterState, ZkNodeProps message, NamedList results) throws Exception { CollectionHandlingUtils.checkRequired( message, COLLECTION_PROP, SHARD_ID_PROP, REPLICA_PROP, PROPERTY_PROP); @@ -389,7 +389,7 @@ public BalanceShardsUniqueCmd(CollectionCommandContext ccc) { } @Override - public void call(ClusterState clusterState, ZkNodeProps message, NamedList results) + public void call(AdminCmdContext clusterState, ZkNodeProps message, NamedList results) throws Exception { if (StrUtils.isBlank(message.getStr(COLLECTION_PROP)) || StrUtils.isBlank(message.getStr(PROPERTY_PROP))) { @@ -425,9 +425,9 @@ public ModifyCollectionCmd(CollectionCommandContext ccc) { } @Override - public void call(ClusterState clusterState, ZkNodeProps message, NamedList results) + public void call( + AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList results) throws Exception { - final String collectionName = message.getStr(ZkStateReader.COLLECTION_PROP); // the rest of the processing is based on writing cluster state properties String configName = (String) message.getProperties().get(COLL_CONF); @@ -497,7 +497,7 @@ public void call(ClusterState clusterState, ZkNodeProps message, NamedList callingLockIdList; + if (adminCmdContext.getCallingLockIds() == null) { + callingLockIdList = Collections.emptyList(); + } else { + callingLockIdList = List.of(adminCmdContext.getCallingLockIds().split(",")); + } + // The first requested lock is a write one (on the target object for the action, depending on // lock level), then requesting read locks on "higher" levels (collection > shard > replica here // for the level. Note LockLevel "height" is other way around). @@ -113,10 +126,18 @@ DistributedMultiLock createCollectionApiLock( }; List locks = new ArrayList<>(iterationOrder.length); for (CollectionParams.LockLevel level : iterationOrder) { + // We do not want to re-lock from the parent lock level + // TODO: Fix + // if (calledFromLockLevel.isHigherOrEqual(level)) { + // continue; + // } // This comparison is based on the LockLevel height value that classifies replica > shard > // collection. if (lockLevel.isHigherOrEqual(level)) { - locks.add(lockFactory.createLock(requestWriteLock, level, collName, shardId, replicaName)); + DistributedLock lock = + lockFactory.createLock( + requestWriteLock, level, collName, shardId, replicaName, callingLockIdList); + locks.add(lock); requestWriteLock = false; } } diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionCommandContext.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionCommandContext.java index f167d4dc6b4..0271f7bbe00 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionCommandContext.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionCommandContext.java @@ -55,9 +55,13 @@ public interface CollectionCommandContext { CoreContainer getCoreContainer(); - default ShardRequestTracker asyncRequestTracker(String asyncId) { + default ShardRequestTracker asyncRequestTracker(AdminCmdContext adminCmdContext) { return new ShardRequestTracker( - asyncId, getAdminPath(), getZkStateReader(), newShardHandler().getShardHandlerFactory()); + adminCmdContext.getAsyncId(), + adminCmdContext.getSubRequestCallingLockIds(), + getAdminPath(), + getZkStateReader(), + newShardHandler().getShardHandlerFactory()); } /** admin path passed to Overseer is a constant, including in tests. */ diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionHandlingUtils.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionHandlingUtils.java index c44a52e2e7d..0b3d4bb930a 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionHandlingUtils.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionHandlingUtils.java @@ -17,6 +17,7 @@ package org.apache.solr.cloud.api.collections; +import static org.apache.solr.common.params.CollectionAdminParams.CALLING_LOCK_IDS_HEADER; import static org.apache.solr.common.params.CollectionParams.CollectionAction.DELETE; import static org.apache.solr.common.params.CommonAdminParams.ASYNC; import static org.apache.solr.common.params.CommonParams.NAME; @@ -307,13 +308,17 @@ static void addPropertyParams(ZkNodeProps message, Map map) { } static void cleanupCollection( - String collectionName, NamedList results, CollectionCommandContext ccc) + AdminCmdContext adminCmdContext, + String collectionName, + NamedList results, + CollectionCommandContext ccc) throws Exception { log.error("Cleaning up collection [{}].", collectionName); - Map props = - Map.of(Overseer.QUEUE_OPERATION, DELETE.toLower(), NAME, collectionName); new DeleteCollectionCmd(ccc) - .call(ccc.getZkStateReader().getClusterState(), new ZkNodeProps(props), results); + .call( + adminCmdContext.subRequestContext(DELETE), + new ZkNodeProps(NAME, collectionName), + results); } static Map waitToSeeReplicasInState( @@ -372,17 +377,6 @@ static void deleteBackup( new DeleteBackupCmd(ccc).keepNumberOfBackup(repository, backupPath, maxNumBackup, results); } - static List addReplica( - ClusterState clusterState, - ZkNodeProps message, - NamedList results, - Runnable onComplete, - CollectionCommandContext ccc) - throws Exception { - - return new AddReplicaCmd(ccc).addReplica(clusterState, message, results, onComplete); - } - static void validateConfigOrThrowSolrException( ConfigSetService configSetService, String configName) throws IOException { boolean isValid = configSetService.checkConfigExists(configName); @@ -399,24 +393,24 @@ static void validateConfigOrThrowSolrException( * @return List of replicas which is not live for receiving the request */ static List collectionCmd( + AdminCmdContext adminCmdContext, ZkNodeProps message, ModifiableSolrParams params, NamedList results, Replica.State stateMatcher, - String asyncId, Set okayExceptions, - CollectionCommandContext ccc, - ClusterState clusterState) { - log.info("Executing Collection Cmd={}, asyncId={}", params, asyncId); + CollectionCommandContext ccc) { + log.info("Executing Collection Cmd={}, asyncId={}", params, adminCmdContext.getAsyncId()); String collectionName = message.getStr(NAME); ShardHandler shardHandler = ccc.newShardHandler(); - DocCollection coll = clusterState.getCollection(collectionName); + DocCollection coll = ccc.getZkStateReader().getClusterState().getCollection(collectionName); List notLivesReplicas = new ArrayList<>(); final CollectionHandlingUtils.ShardRequestTracker shardRequestTracker = - asyncRequestTracker(asyncId, ccc); + asyncRequestTracker(adminCmdContext, ccc); for (Slice slice : coll.getSlices()) { notLivesReplicas.addAll( - shardRequestTracker.sliceCmd(clusterState, params, stateMatcher, slice, shardHandler)); + shardRequestTracker.sliceCmd( + ccc.getZkStateReader().getClusterState(), params, stateMatcher, slice, shardHandler)); } shardRequestTracker.processResponses(results, shardHandler, false, null, okayExceptions); @@ -574,14 +568,22 @@ private static NamedList waitForCoreAdminAsyncCallToComplete( } while (true); } - public static ShardRequestTracker syncRequestTracker(CollectionCommandContext ccc) { - return asyncRequestTracker(null, ccc); + public static ShardRequestTracker syncRequestTracker( + AdminCmdContext adminCmdContext, CollectionCommandContext ccc) { + return requestTracker(null, adminCmdContext.getSubRequestCallingLockIds(), ccc); } public static ShardRequestTracker asyncRequestTracker( - String asyncId, CollectionCommandContext ccc) { + AdminCmdContext adminCmdContext, CollectionCommandContext ccc) { + return requestTracker( + adminCmdContext.getAsyncId(), adminCmdContext.getSubRequestCallingLockIds(), ccc); + } + + protected static ShardRequestTracker requestTracker( + String asyncId, String lockIds, CollectionCommandContext ccc) { return new ShardRequestTracker( asyncId, + lockIds, ccc.getAdminPath(), ccc.getZkStateReader(), ccc.newShardHandler().getShardHandlerFactory()); @@ -589,6 +591,7 @@ public static ShardRequestTracker asyncRequestTracker( public static class ShardRequestTracker { private final String asyncId; + private final String lockIdList; private final String adminPath; private final ZkStateReader zkStateReader; private final ShardHandlerFactory shardHandlerFactory; @@ -596,10 +599,12 @@ public static class ShardRequestTracker { public ShardRequestTracker( String asyncId, + String lockIdList, String adminPath, ZkStateReader zkStateReader, ShardHandlerFactory shardHandlerFactory) { this.asyncId = asyncId; + this.lockIdList = lockIdList; this.adminPath = adminPath; this.zkStateReader = zkStateReader; this.shardHandlerFactory = shardHandlerFactory; @@ -662,6 +667,9 @@ public void sendShardRequest( sreq.actualShards = sreq.shards; sreq.nodeName = nodeName; sreq.params = params; + if (lockIdList != null && !lockIdList.isBlank()) { + sreq.headers = Map.of(CALLING_LOCK_IDS_HEADER, lockIdList); + } shardHandler.submit(sreq, replica, sreq.params); } diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateAliasCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateAliasCmd.java index f48a984aad1..53de754285f 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateAliasCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateAliasCmd.java @@ -50,7 +50,7 @@ public CreateAliasCmd(CollectionCommandContext ccc) { } @Override - public void call(ClusterState state, ZkNodeProps message, NamedList results) + public void call(AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList results) throws Exception { final String aliasName = message.getStr(CommonParams.NAME); ZkStateReader zkStateReader = ccc.getZkStateReader(); @@ -63,7 +63,7 @@ public void call(ClusterState state, ZkNodeProps message, NamedList resu if (!anyRoutingParams(message)) { callCreatePlainAlias(message, aliasName, zkStateReader); } else { - callCreateRoutedAlias(message, aliasName, zkStateReader, state); + callCreateRoutedAlias(adminCmdContext, message, aliasName, zkStateReader); } // Sleep a bit to allow ZooKeeper state propagation. @@ -112,7 +112,10 @@ private List parseCollectionsParameter(Object collections) { } private void callCreateRoutedAlias( - ZkNodeProps message, String aliasName, ZkStateReader zkStateReader, ClusterState state) + AdminCmdContext adminCmdContext, + ZkNodeProps message, + String aliasName, + ZkStateReader zkStateReader) throws Exception { // Validate we got a basic minimum if (!message.getProperties().keySet().containsAll(RoutedAlias.MINIMAL_REQUIRED_PARAMS)) { @@ -150,7 +153,11 @@ private void callCreateRoutedAlias( // Create the first collection. Prior validation ensures that this is not a standard alias collectionListStr = routedAlias.computeInitialCollectionName(); ensureAliasCollection( - aliasName, zkStateReader, state, routedAlias.getAliasMetadata(), collectionListStr); + adminCmdContext, + aliasName, + zkStateReader, + routedAlias.getAliasMetadata(), + collectionListStr); } else { List collectionList = aliases.resolveAliases(aliasName); collectionListStr = String.join(",", collectionList); @@ -163,14 +170,15 @@ private void callCreateRoutedAlias( } private void ensureAliasCollection( + AdminCmdContext adminCmdContext, String aliasName, ZkStateReader zkStateReader, - ClusterState state, Map aliasProperties, String initialCollectionName) throws Exception { // Create the collection - createCollectionAndWait(state, aliasName, aliasProperties, initialCollectionName, ccc); + createCollectionAndWait( + adminCmdContext, aliasName, aliasProperties, initialCollectionName, ccc); validateAllCollectionsExistAndNoDuplicates( Collections.singletonList(initialCollectionName), zkStateReader); } diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateCollectionCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateCollectionCmd.java index acd82c44203..b0f6754a15b 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateCollectionCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateCollectionCmd.java @@ -20,6 +20,7 @@ import static org.apache.solr.common.params.CollectionAdminParams.ALIAS; import static org.apache.solr.common.params.CollectionAdminParams.COLL_CONF; import static org.apache.solr.common.params.CollectionParams.CollectionAction.ADDREPLICA; +import static org.apache.solr.common.params.CollectionParams.CollectionAction.DELETE; import static org.apache.solr.common.params.CommonAdminParams.ASYNC; import static org.apache.solr.common.params.CommonAdminParams.WAIT_FOR_FINAL_STATE; import static org.apache.solr.common.params.CommonParams.NAME; @@ -101,11 +102,12 @@ public CreateCollectionCmd(CollectionCommandContext ccc) { } @Override - public void call(ClusterState clusterState, ZkNodeProps message, NamedList results) + public void call(AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList results) throws Exception { if (ccc.getZkStateReader().aliasesManager != null) { // not a mock ZkStateReader ccc.getZkStateReader().aliasesManager.update(); } + ClusterState clusterState = adminCmdContext.getClusterState(); final Aliases aliases = ccc.getZkStateReader().getAliases(); final String collectionName = message.getStr(NAME); final boolean waitForFinalState = message.getBool(WAIT_FOR_FINAL_STATE, false); @@ -151,9 +153,6 @@ public void call(ClusterState clusterState, ZkNodeProps message, NamedList(), ccc); + CollectionHandlingUtils.cleanupCollection( + adminCmdContext, collectionName, new NamedList<>(), ccc); log.info("Cleaned up artifacts for failed create collection for [{}]", collectionName); throw new SolrException( ErrorCode.BAD_REQUEST, diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateShardCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateShardCmd.java index b76250cfd8d..ee7b2242f8c 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateShardCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateShardCmd.java @@ -19,7 +19,6 @@ import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP; import static org.apache.solr.common.cloud.ZkStateReader.SHARD_ID_PROP; import static org.apache.solr.common.params.CollectionAdminParams.FOLLOW_ALIASES; -import static org.apache.solr.common.params.CommonAdminParams.ASYNC; import java.lang.invoke.MethodHandles; import java.util.Map; @@ -29,6 +28,7 @@ import org.apache.solr.common.cloud.DocCollection; import org.apache.solr.common.cloud.ReplicaCount; import org.apache.solr.common.cloud.ZkNodeProps; +import org.apache.solr.common.params.CollectionParams; import org.apache.solr.common.params.CommonAdminParams; import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.SimpleOrderedMap; @@ -45,7 +45,7 @@ public CreateShardCmd(CollectionCommandContext ccc) { } @Override - public void call(ClusterState clusterState, ZkNodeProps message, NamedList results) + public void call(AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList results) throws Exception { String extCollectionName = message.getStr(COLLECTION_PROP); String sliceName = message.getStr(SHARD_ID_PROP); @@ -64,6 +64,7 @@ public void call(ClusterState clusterState, ZkNodeProps message, NamedList addReplicasProps = Utils.makeMap( COLLECTION_PROP, @@ -110,14 +111,13 @@ public void call(ClusterState clusterState, ZkNodeProps message, NamedList addResult = new NamedList<>(); try { new AddReplicaCmd(ccc) .addReplica( - clusterState, + adminCmdContext + .subRequestContext(CollectionParams.CollectionAction.ADDREPLICA, async) + .withClusterState(clusterState), new ZkNodeProps(addReplicasProps), addResult, () -> { @@ -147,9 +147,13 @@ public void call(ClusterState clusterState, ZkNodeProps message, NamedList results) + public void call(AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList results) throws Exception { String extCollectionName = message.getStr(COLLECTION_PROP); boolean followAliases = message.getBool(FOLLOW_ALIASES, false); @@ -75,7 +73,6 @@ public void call(ClusterState state, ZkNodeProps message, NamedList resu } String commitName = message.getStr(CoreAdminParams.COMMIT_NAME); - String asyncId = message.getStr(ASYNC); SolrZkClient zkClient = ccc.getZkStateReader().getZkClient(); Date creationDate = new Date(); @@ -101,7 +98,7 @@ public void call(ClusterState state, ZkNodeProps message, NamedList resu ShardHandler shardHandler = ccc.newShardHandler(); final ShardRequestTracker shardRequestTracker = - CollectionHandlingUtils.asyncRequestTracker(asyncId, ccc); + CollectionHandlingUtils.asyncRequestTracker(adminCmdContext, ccc); for (Slice slice : ccc.getZkStateReader().getClusterState().getCollection(collectionName).getSlices()) { for (Replica replica : slice.getReplicas()) { diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteAliasCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteAliasCmd.java index 2c99dc36ef7..1d90b4e8797 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteAliasCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteAliasCmd.java @@ -19,7 +19,6 @@ import static org.apache.solr.common.params.CommonParams.NAME; -import org.apache.solr.common.cloud.ClusterState; import org.apache.solr.common.cloud.ZkNodeProps; import org.apache.solr.common.cloud.ZkStateReader; import org.apache.solr.common.util.NamedList; @@ -32,7 +31,7 @@ public DeleteAliasCmd(CollectionCommandContext ccc) { } @Override - public void call(ClusterState state, ZkNodeProps message, NamedList results) + public void call(AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList results) throws Exception { String aliasName = message.getStr(NAME); diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteBackupCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteBackupCmd.java index aa6c8f9ef5b..dfec0e3bb0e 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteBackupCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteBackupCmd.java @@ -36,7 +36,6 @@ import java.util.Set; import java.util.stream.Collectors; import org.apache.solr.common.SolrException; -import org.apache.solr.common.cloud.ClusterState; import org.apache.solr.common.cloud.ZkNodeProps; import org.apache.solr.common.params.CoreAdminParams; import org.apache.solr.common.util.NamedList; @@ -71,7 +70,7 @@ public DeleteBackupCmd(CollectionCommandContext ccc) { } @Override - public void call(ClusterState state, ZkNodeProps message, NamedList results) + public void call(AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList results) throws Exception { String backupLocation = message.getStr(CoreAdminParams.BACKUP_LOCATION); String backupName = message.getStr(NAME); diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteCollectionCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteCollectionCmd.java index 4298ee3f6d3..9c292863ece 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteCollectionCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteCollectionCmd.java @@ -19,7 +19,6 @@ import static org.apache.solr.common.params.CollectionAdminParams.FOLLOW_ALIASES; import static org.apache.solr.common.params.CollectionParams.CollectionAction.DELETE; -import static org.apache.solr.common.params.CommonAdminParams.ASYNC; import static org.apache.solr.common.params.CommonParams.NAME; import java.lang.invoke.MethodHandles; @@ -36,7 +35,6 @@ import org.apache.solr.common.NonExistentCoreException; import org.apache.solr.common.SolrException; import org.apache.solr.common.cloud.Aliases; -import org.apache.solr.common.cloud.ClusterState; import org.apache.solr.common.cloud.DocCollection; import org.apache.solr.common.cloud.Replica; import org.apache.solr.common.cloud.SolrZkClient; @@ -64,7 +62,7 @@ public DeleteCollectionCmd(CollectionCommandContext ccc) { } @Override - public void call(ClusterState state, ZkNodeProps message, NamedList results) + public void call(AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList results) throws Exception { Object o = message.get(MaintainRoutedAliasCmd.INVOKED_BY_ROUTED_ALIAS); if (o != null) { @@ -91,7 +89,7 @@ public void call(ClusterState state, ZkNodeProps message, NamedList resu } // verify the placement modifications caused by the deletion are allowed - DocCollection coll = state.getCollectionOrNull(collection); + DocCollection coll = zkStateReader.getClusterState().getCollectionOrNull(collection); if (coll != null) { Assign.AssignStrategy assignStrategy = Assign.createAssignStrategy(ccc.getCoreContainer()); assignStrategy.verifyDeleteCollection(ccc.getSolrCloudManager(), coll); @@ -119,13 +117,11 @@ public void call(ClusterState state, ZkNodeProps message, NamedList resu params.set(CoreAdminParams.DELETE_INSTANCE_DIR, true); params.set(CoreAdminParams.DELETE_DATA_DIR, true); - String asyncId = message.getStr(ASYNC); - ZkNodeProps internalMsg = message.plus(NAME, collection); List failedReplicas = CollectionHandlingUtils.collectionCmd( - internalMsg, params, results, null, asyncId, okayExceptions, ccc, state); + adminCmdContext, internalMsg, params, results, null, okayExceptions, ccc); for (Replica failedReplica : failedReplicas) { boolean isSharedFS = failedReplica.getBool(ZkStateReader.SHARED_STORAGE_PROP, false) diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteNodeCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteNodeCmd.java index b93980ff7da..656eb374709 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteNodeCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteNodeCmd.java @@ -17,8 +17,6 @@ package org.apache.solr.cloud.api.collections; -import static org.apache.solr.common.params.CommonAdminParams.ASYNC; - import java.util.ArrayList; import java.util.List; import org.apache.solr.common.cloud.ClusterState; @@ -36,12 +34,14 @@ public DeleteNodeCmd(CollectionCommandContext ccc) { } @Override - public void call(ClusterState state, ZkNodeProps message, NamedList results) + public void call(AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList results) throws Exception { CollectionHandlingUtils.checkRequired(message, "node"); String node = message.getStr("node"); - List sourceReplicas = ReplicaMigrationUtils.getReplicasOfNode(node, state); - List singleReplicas = verifyReplicaAvailability(sourceReplicas, state); + List sourceReplicas = + ReplicaMigrationUtils.getReplicasOfNode(node, adminCmdContext.getClusterState()); + List singleReplicas = + verifyReplicaAvailability(sourceReplicas, adminCmdContext.getClusterState()); if (!singleReplicas.isEmpty()) { results.add( "failure", @@ -50,8 +50,7 @@ public void call(ClusterState state, ZkNodeProps message, NamedList resu + ": " + singleReplicas); } else { - ReplicaMigrationUtils.cleanupReplicas( - results, state, sourceReplicas, ccc, message.getStr(ASYNC)); + ReplicaMigrationUtils.cleanupReplicas(results, adminCmdContext, sourceReplicas, ccc); } } diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteReplicaCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteReplicaCmd.java index 138dec2ef55..5246401db6e 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteReplicaCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteReplicaCmd.java @@ -21,7 +21,6 @@ import static org.apache.solr.common.cloud.ZkStateReader.SHARD_ID_PROP; import static org.apache.solr.common.params.CollectionAdminParams.COUNT_PROP; import static org.apache.solr.common.params.CollectionAdminParams.FOLLOW_ALIASES; -import static org.apache.solr.common.params.CommonAdminParams.ASYNC; import java.io.IOException; import java.lang.invoke.MethodHandles; @@ -36,7 +35,6 @@ import org.apache.solr.cloud.api.collections.CollApiCmds.CollectionApiCommand; import org.apache.solr.cloud.api.collections.CollectionHandlingUtils.ShardRequestTracker; import org.apache.solr.common.SolrException; -import org.apache.solr.common.cloud.ClusterState; import org.apache.solr.common.cloud.DocCollection; import org.apache.solr.common.cloud.Replica; import org.apache.solr.common.cloud.Slice; @@ -61,13 +59,13 @@ public DeleteReplicaCmd(CollectionCommandContext ccc) { } @Override - public void call(ClusterState clusterState, ZkNodeProps message, NamedList results) + public void call(AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList results) throws Exception { - deleteReplica(clusterState, message, results, null); + deleteReplica(adminCmdContext, message, results, null); } void deleteReplica( - ClusterState clusterState, + AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList results, Runnable onComplete) @@ -79,7 +77,7 @@ void deleteReplica( // If a count is specified the strategy needs be different if (message.getStr(COUNT_PROP) != null) { - deleteReplicaBasedOnCount(clusterState, message, results, onComplete, parallel); + deleteReplicaBasedOnCount(adminCmdContext, message, results, onComplete, parallel); return; } @@ -97,14 +95,15 @@ void deleteReplica( collectionName = extCollectionName; } - DocCollection coll = clusterState.getCollection(collectionName); + DocCollection coll = adminCmdContext.getClusterState().getCollection(collectionName); Slice slice = coll.getSlice(shard); if (slice == null) { throw new SolrException( SolrException.ErrorCode.BAD_REQUEST, "Invalid shard name : " + shard + " in collection : " + collectionName); } - deleteCore(coll, shard, replicaName, message, results, onComplete, parallel, true); + deleteCore( + adminCmdContext, coll, shard, replicaName, message, results, onComplete, parallel, true); } /** @@ -112,7 +111,7 @@ void deleteReplica( * deletes given num replicas across all shards for the given collection. */ void deleteReplicaBasedOnCount( - ClusterState clusterState, + AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList results, Runnable onComplete, @@ -122,7 +121,7 @@ void deleteReplicaBasedOnCount( int count = Integer.parseInt(message.getStr(COUNT_PROP)); String collectionName = message.getStr(COLLECTION_PROP); String shard = message.getStr(SHARD_ID_PROP); - DocCollection coll = clusterState.getCollection(collectionName); + DocCollection coll = adminCmdContext.getClusterState().getCollection(collectionName); Slice slice = null; // Validate if shard is passed. if (shard != null) { @@ -172,7 +171,8 @@ void deleteReplicaBasedOnCount( for (String replica : replicas) { log.debug("Deleting replica {} for shard {} based on count {}", replica, shardId, count); // don't verify with the placement plugin - we already did it - deleteCore(coll, shardId, replica, message, results, onComplete, parallel, false); + deleteCore( + adminCmdContext, coll, shardId, replica, message, results, onComplete, parallel, false); } results.add("shard_id", shardId); results.add("replicas_deleted", replicas); @@ -242,6 +242,7 @@ private void validateReplicaAvailability( } void deleteCore( + AdminCmdContext adminCmdContext, DocCollection coll, String shardId, String replicaName, @@ -295,7 +296,6 @@ void deleteCore( ShardHandler shardHandler = ccc.newShardHandler(); String core = replica.getStr(ZkStateReader.CORE_NAME_PROP); - String asyncId = message.getStr(ASYNC); ModifiableSolrParams params = new ModifiableSolrParams(); params.add(CoreAdminParams.ACTION, CoreAdminParams.CoreAdminAction.UNLOAD.toString()); @@ -311,7 +311,7 @@ void deleteCore( boolean isLive = ccc.getZkStateReader().getClusterState().getLiveNodes().contains(replica.getNodeName()); final ShardRequestTracker shardRequestTracker = - CollectionHandlingUtils.asyncRequestTracker(asyncId, ccc); + CollectionHandlingUtils.asyncRequestTracker(adminCmdContext, ccc); if (isLive) { shardRequestTracker.sendShardRequest(replica.getNodeName(), params, shardHandler); } diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteShardCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteShardCmd.java index d32b80d4b39..1234925c94c 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteShardCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteShardCmd.java @@ -20,8 +20,8 @@ import static org.apache.solr.common.cloud.ZkStateReader.NODE_NAME_PROP; import static org.apache.solr.common.cloud.ZkStateReader.SHARD_ID_PROP; import static org.apache.solr.common.params.CollectionAdminParams.FOLLOW_ALIASES; +import static org.apache.solr.common.params.CollectionParams.CollectionAction.DELETEREPLICA; import static org.apache.solr.common.params.CollectionParams.CollectionAction.DELETESHARD; -import static org.apache.solr.common.params.CommonAdminParams.ASYNC; import java.lang.invoke.MethodHandles; import java.util.ArrayList; @@ -35,7 +35,6 @@ import org.apache.solr.cloud.Overseer; import org.apache.solr.cloud.overseer.OverseerAction; import org.apache.solr.common.SolrException; -import org.apache.solr.common.cloud.ClusterState; import org.apache.solr.common.cloud.Replica; import org.apache.solr.common.cloud.Slice; import org.apache.solr.common.cloud.SolrZkClient; @@ -58,7 +57,7 @@ public DeleteShardCmd(CollectionCommandContext ccc) { } @Override - public void call(ClusterState clusterState, ZkNodeProps message, NamedList results) + public void call(AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList results) throws Exception { String extCollectionName = message.getStr(ZkStateReader.COLLECTION_PROP); String sliceId = message.getStr(ZkStateReader.SHARD_ID_PROP); @@ -73,7 +72,7 @@ public void call(ClusterState clusterState, ZkNodeProps message, NamedList replicas = getReplicasForSlice(collectionName, slice); CountDownLatch cleanupLatch = new CountDownLatch(replicas.size()); for (ZkNodeProps r : replicas) { - final ZkNodeProps replica = - r.plus(message.getProperties()).plus("parallel", "true").plus(ASYNC, asyncId); + final ZkNodeProps replica = r.plus(message.getProperties()).plus("parallel", "true"); if (log.isInfoEnabled()) { log.info( "Deleting replica for collection={} shard={} on node={}", @@ -144,7 +140,7 @@ public void call(ClusterState clusterState, ZkNodeProps message, NamedList { diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteSnapshotCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteSnapshotCmd.java index b3970210cb6..ffb93c48fc2 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteSnapshotCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteSnapshotCmd.java @@ -19,7 +19,6 @@ import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP; import static org.apache.solr.common.cloud.ZkStateReader.CORE_NAME_PROP; import static org.apache.solr.common.params.CollectionAdminParams.FOLLOW_ALIASES; -import static org.apache.solr.common.params.CommonAdminParams.ASYNC; import static org.apache.solr.common.params.CommonParams.NAME; import java.lang.invoke.MethodHandles; @@ -31,7 +30,6 @@ import org.apache.solr.cloud.api.collections.CollectionHandlingUtils.ShardRequestTracker; import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrException.ErrorCode; -import org.apache.solr.common.cloud.ClusterState; import org.apache.solr.common.cloud.Replica; import org.apache.solr.common.cloud.Replica.State; import org.apache.solr.common.cloud.Slice; @@ -60,7 +58,7 @@ public DeleteSnapshotCmd(CollectionCommandContext ccc) { } @Override - public void call(ClusterState state, ZkNodeProps message, NamedList results) + public void call(AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList results) throws Exception { String extCollectionName = message.getStr(COLLECTION_PROP); boolean followAliases = message.getBool(FOLLOW_ALIASES, false); @@ -71,7 +69,6 @@ public void call(ClusterState state, ZkNodeProps message, NamedList resu collectionName = extCollectionName; } String commitName = message.getStr(CoreAdminParams.COMMIT_NAME); - String asyncId = message.getStr(ASYNC); NamedList shardRequestResults = new NamedList<>(); ShardHandler shardHandler = ccc.newShardHandler(); SolrZkClient zkClient = ccc.getZkStateReader().getZkClient(); @@ -101,7 +98,7 @@ public void call(ClusterState state, ZkNodeProps message, NamedList resu } final ShardRequestTracker shardRequestTracker = - CollectionHandlingUtils.asyncRequestTracker(asyncId, ccc); + CollectionHandlingUtils.asyncRequestTracker(adminCmdContext, ccc); log.info( "Existing cores with snapshot for collection={} are {}", collectionName, existingCores); for (Slice slice : diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/DistributedCollectionConfigSetCommandRunner.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/DistributedCollectionConfigSetCommandRunner.java index 465cc4d6273..d0b3fbead5d 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/DistributedCollectionConfigSetCommandRunner.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/DistributedCollectionConfigSetCommandRunner.java @@ -22,7 +22,6 @@ import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP; import static org.apache.solr.common.cloud.ZkStateReader.REPLICA_PROP; import static org.apache.solr.common.cloud.ZkStateReader.SHARD_ID_PROP; -import static org.apache.solr.common.params.CommonAdminParams.ASYNC; import static org.apache.solr.common.params.CommonParams.NAME; import java.io.IOException; @@ -46,7 +45,6 @@ import org.apache.solr.common.SolrException; import org.apache.solr.common.cloud.SolrZkClient; import org.apache.solr.common.cloud.ZkNodeProps; -import org.apache.solr.common.params.CollectionParams; import org.apache.solr.common.params.ConfigSetParams; import org.apache.solr.common.params.CoreAdminParams; import org.apache.solr.common.util.ExecutorUtil; @@ -252,7 +250,7 @@ public void runConfigSetCommand( * */ public OverseerSolrResponse runCollectionCommand( - ZkNodeProps message, CollectionParams.CollectionAction action, long timeoutMs) { + AdminCmdContext adminCmdContext, ZkNodeProps message, long timeoutMs) { // We refuse new tasks, but will wait for already submitted ones (i.e. those that made it // through this method earlier). See stopAndWaitForPendingTasksToComplete() below if (shuttingDown) { @@ -261,35 +259,35 @@ public OverseerSolrResponse runCollectionCommand( "Solr is shutting down, no more Collection API tasks may be executed"); } - final String asyncId = message.getStr(ASYNC); - if (log.isInfoEnabled()) { log.info( - "Running Collection API locally for " + action.name() + " asyncId=" + asyncId); // nowarn + "Running Collection API locally for {} asyncId={}", + adminCmdContext.getAction().name(), + adminCmdContext.getAsyncId()); } // Following the call below returning true, we must eventually cancel or complete the task. // Happens either in the CollectionCommandRunner below or in the catch when the runner would not // execute. - if (!asyncTaskTracker.createNewAsyncJobTracker(asyncId)) { + if (!asyncTaskTracker.createNewAsyncJobTracker(adminCmdContext.getAsyncId())) { throw new SolrException( SolrException.ErrorCode.BAD_REQUEST, - "Task with the same requestid already exists. (" + asyncId + ")"); + "Task with the same requestid already exists. (" + adminCmdContext.getAsyncId() + ")"); } - CollectionCommandRunner commandRunner = new CollectionCommandRunner(message, action, asyncId); + CollectionCommandRunner commandRunner = new CollectionCommandRunner(adminCmdContext, message); final Future taskFuture; try { taskFuture = commandsExecutor.submit(commandRunner); } catch (RejectedExecutionException ree) { // The command will not run, need to cancel the async ID so it can be reused on a subsequent // attempt by the client - asyncTaskTracker.cancelAsyncId(asyncId); + asyncTaskTracker.cancelAsyncId(adminCmdContext.getAsyncId()); throw new SolrException( SolrException.ErrorCode.SERVICE_UNAVAILABLE, "Too many executing commands", ree); } - if (asyncId == null) { + if (adminCmdContext.getAsyncId() == null) { // Non async calls wait for a while in case the command completes. If they time out, there's // no way to track the job progress (improvement suggestion: decorrelate having a task ID from // the fact of waiting for the job to complete) @@ -297,18 +295,21 @@ public OverseerSolrResponse runCollectionCommand( return taskFuture.get(timeoutMs, TimeUnit.MILLISECONDS); } catch (TimeoutException te) { throw new SolrException( - SolrException.ErrorCode.SERVER_ERROR, action + " timed out after " + timeoutMs + "ms"); + SolrException.ErrorCode.SERVER_ERROR, + adminCmdContext.getAction() + " timed out after " + timeoutMs + "ms"); } catch (InterruptedException e) { Thread.currentThread().interrupt(); - throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, action + " interrupted", e); + throw new SolrException( + SolrException.ErrorCode.SERVER_ERROR, adminCmdContext.getAction() + " interrupted", e); } catch (Exception e) { - throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, action + " failed", e); + throw new SolrException( + SolrException.ErrorCode.SERVER_ERROR, adminCmdContext.getAction() + " failed", e); } } else { // Async calls do not wait for the command to finish but get instead back the async id (that // they just sent...) NamedList resp = new NamedList<>(); - resp.add(CoreAdminParams.REQUESTID, asyncId); + resp.add(CoreAdminParams.REQUESTID, adminCmdContext.getAsyncId()); return new OverseerSolrResponse(resp); } } @@ -357,15 +358,12 @@ public static String getCollectionName(ZkNodeProps message) { * similar to the one provided by Overseer based Collection API execution. */ private class CollectionCommandRunner implements Callable { + private final AdminCmdContext adminCmdContext; private final ZkNodeProps message; - private final CollectionParams.CollectionAction action; - private final String asyncId; - private CollectionCommandRunner( - ZkNodeProps message, CollectionParams.CollectionAction action, String asyncId) { + private CollectionCommandRunner(AdminCmdContext adminCmdContext, ZkNodeProps message) { + this.adminCmdContext = adminCmdContext; this.message = message; - this.action = action; - this.asyncId = asyncId; } /** @@ -398,36 +396,45 @@ public OverseerSolrResponse call() { new CollectionApiLockFactory( new ZkDistributedCollectionLockFactory( ccc.getZkStateReader().getZkClient(), ZK_COLLECTION_LOCKS)) - .createCollectionApiLock(action.lockLevel, collName, shardId, replicaName); + .createCollectionApiLock(adminCmdContext, collName, shardId, replicaName); try { - log.debug( - "CollectionCommandRunner about to acquire lock for action {} lock level {}. {}/{}/{}", - action, - action.lockLevel, - collName, - shardId, - replicaName); + if (log.isDebugEnabled()) { + log.debug( + "CollectionCommandRunner about to acquire lock for action {} lock level {}. {}/{}/{}", + adminCmdContext.getAction().name(), + adminCmdContext.getAction().lockLevel, + collName, + shardId, + replicaName); + } // Block this thread until all required locks are acquired. lock.waitUntilAcquired(); + adminCmdContext.setLockId(lock.getLockId()); // Got the lock so moving from submitted to running if we run for an async task (if // asyncId is null the asyncTaskTracker calls do nothing). - asyncTaskTracker.setTaskRunning(asyncId); + asyncTaskTracker.setTaskRunning(adminCmdContext.getAsyncId()); - log.debug( - "DistributedCollectionConfigSetCommandRunner.runCollectionCommand. Lock acquired. Calling: {}, {}", - action, - message); + if (log.isDebugEnabled()) { + log.debug( + "DistributedCollectionConfigSetCommandRunner.runCollectionCommand. Lock acquired. Calling: {}, {}", + adminCmdContext.getAction(), + message); + } - CollApiCmds.CollectionApiCommand command = commandMapper.getActionCommand(action); + CollApiCmds.CollectionApiCommand command = + commandMapper.getActionCommand(adminCmdContext.getAction()); if (command != null) { - command.call(ccc.getSolrCloudManager().getClusterState(), message, results); + command.call( + adminCmdContext.withClusterState(ccc.getSolrCloudManager().getClusterState()), + message, + results); } else { - asyncTaskTracker.cancelAsyncId(asyncId); + asyncTaskTracker.cancelAsyncId(adminCmdContext.getAsyncId()); // Seeing this is a bug, not bad user data - String message = "Bug: Unknown operation " + action; + String message = "Bug: Unknown operation " + adminCmdContext.getAction(); log.error(message); throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, message); } @@ -439,21 +446,25 @@ public OverseerSolrResponse call() { // hierarchy do no harm, and there shouldn't be too many of those. lock.release(); } catch (SolrException se) { - log.error( - "Error when releasing collection locks for operation " + action, se); // nowarn + if (log.isErrorEnabled()) { + log.error( + "Error when releasing collection locks for operation {}", + adminCmdContext.getAction(), + se); + } } } } catch (Exception e) { if (e instanceof InterruptedException) { Thread.currentThread().interrupt(); } - logFailedOperation(action, e, collName); - addExceptionToNamedList(action, e, results); + logFailedOperation(adminCmdContext.getAction(), e, collName); + addExceptionToNamedList(adminCmdContext.getAction(), e, results); } OverseerSolrResponse res = new OverseerSolrResponse(results); // Following call marks success or failure depending on the contents of res - asyncTaskTracker.setTaskCompleted(asyncId, res); + asyncTaskTracker.setTaskCompleted(adminCmdContext.getAsyncId(), res); return res; } } diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/InstallShardDataCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/InstallShardDataCmd.java index 18ce5aa4071..39b81cd434b 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/InstallShardDataCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/InstallShardDataCmd.java @@ -58,12 +58,13 @@ public InstallShardDataCmd(CollectionCommandContext ccc) { } @Override - public void call(ClusterState state, ZkNodeProps message, NamedList results) + @SuppressWarnings("unchecked") + public void call(AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList results) throws Exception { final RemoteMessage typedMessage = new ObjectMapper().convertValue(message.getProperties(), RemoteMessage.class); final CollectionHandlingUtils.ShardRequestTracker shardRequestTracker = - CollectionHandlingUtils.asyncRequestTracker(typedMessage.asyncId, ccc); + CollectionHandlingUtils.asyncRequestTracker(adminCmdContext, ccc); final ClusterState clusterState = ccc.getZkStateReader().getClusterState(); typedMessage.validate(); @@ -108,6 +109,10 @@ public static class RemoteMessage implements JacksonReflectMapWriter { @JsonProperty public String location; + @JsonProperty public String name = ""; + + @JsonProperty public String shardBackupId; + @JsonProperty(ASYNC) public String asyncId; diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/MaintainRoutedAliasCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/MaintainRoutedAliasCmd.java index 3320ea0b5b7..e6c316c2aa6 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/MaintainRoutedAliasCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/MaintainRoutedAliasCmd.java @@ -25,10 +25,8 @@ import java.util.List; import java.util.Map; import org.apache.solr.client.solrj.SolrResponse; -import org.apache.solr.cloud.Overseer; import org.apache.solr.common.SolrException; import org.apache.solr.common.cloud.Aliases; -import org.apache.solr.common.cloud.ClusterState; import org.apache.solr.common.cloud.CollectionProperties; import org.apache.solr.common.cloud.ZkNodeProps; import org.apache.solr.common.cloud.ZkStateReader; @@ -57,14 +55,13 @@ public class MaintainRoutedAliasCmd extends AliasCmd { */ static void remoteInvoke(CollectionsHandler collHandler, String aliasName, String targetCol) throws Exception { - final CollectionParams.CollectionAction maintainroutedalias = - CollectionParams.CollectionAction.MAINTAINROUTEDALIAS; - Map msg = new HashMap<>(); - msg.put(Overseer.QUEUE_OPERATION, maintainroutedalias.toLower()); - msg.put(CollectionParams.NAME, aliasName); - msg.put(MaintainRoutedAliasCmd.ROUTED_ALIAS_TARGET_COL, targetCol); final SolrResponse rsp = - collHandler.submitCollectionApiCommand(new ZkNodeProps(msg), maintainroutedalias); + collHandler.submitCollectionApiCommand( + new AdminCmdContext(CollectionParams.CollectionAction.MAINTAINROUTEDALIAS), + new ZkNodeProps( + Map.of( + CollectionParams.NAME, aliasName, + MaintainRoutedAliasCmd.ROUTED_ALIAS_TARGET_COL, targetCol))); if (rsp.getException() != null) { throw rsp.getException(); } @@ -109,7 +106,7 @@ private void removeCollectionFromAlias( } @Override - public void call(ClusterState clusterState, ZkNodeProps message, NamedList results) + public void call(AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList results) throws Exception { // ---- PARSE PRIMARY MESSAGE PARAMS // important that we use NAME for the alias as that is what the Overseer will get a lock on @@ -146,7 +143,7 @@ public void call(ClusterState clusterState, ZkNodeProps message, NamedList { try { deleteTargetCollection( - clusterState, results, aliasName, aliasesManager, action); + adminCmdContext, results, aliasName, aliasesManager, action); } catch (Exception e) { log.warn( "Deletion of {} by {} {} failed (this might be ok if two clients were", @@ -161,7 +158,7 @@ public void call(ClusterState clusterState, ZkNodeProps message, NamedList results, String aliasName, ZkStateReader.AliasesManager aliasesManager, @@ -198,7 +195,7 @@ public void addTargetCollection( throws Exception { NamedList createResults = createCollectionAndWait( - clusterState, aliasName, aliasMetadata, action.targetCollection, ccc); + adminCmdContext, aliasName, aliasMetadata, action.targetCollection, ccc); if (createResults != null) { results.add("create", createResults); } @@ -206,7 +203,7 @@ public void addTargetCollection( } public void deleteTargetCollection( - ClusterState clusterState, + AdminCmdContext adminCmdContext, NamedList results, String aliasName, ZkStateReader.AliasesManager aliasesManager, @@ -219,6 +216,10 @@ public void deleteTargetCollection( () -> removeCollectionFromAlias(aliasName, aliasesManager, action.targetCollection)); delProps.put(NAME, action.targetCollection); ZkNodeProps messageDelete = new ZkNodeProps(delProps); - new DeleteCollectionCmd(ccc).call(clusterState, messageDelete, results); + new DeleteCollectionCmd(ccc) + .call( + adminCmdContext.subRequestContext(CollectionParams.CollectionAction.DELETE), + messageDelete, + results); } } diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/MigrateCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/MigrateCmd.java index b09f806535a..ba56c8ab9dd 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/MigrateCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/MigrateCmd.java @@ -23,7 +23,6 @@ import static org.apache.solr.common.params.CollectionParams.CollectionAction.ADDREPLICA; import static org.apache.solr.common.params.CollectionParams.CollectionAction.CREATE; import static org.apache.solr.common.params.CollectionParams.CollectionAction.DELETE; -import static org.apache.solr.common.params.CommonAdminParams.ASYNC; import static org.apache.solr.common.params.CommonParams.NAME; import java.lang.invoke.MethodHandles; @@ -38,7 +37,6 @@ import org.apache.solr.cloud.api.collections.CollectionHandlingUtils.ShardRequestTracker; import org.apache.solr.cloud.overseer.OverseerAction; import org.apache.solr.common.SolrException; -import org.apache.solr.common.cloud.ClusterState; import org.apache.solr.common.cloud.CompositeIdRouter; import org.apache.solr.common.cloud.DocCollection; import org.apache.solr.common.cloud.DocRouter; @@ -65,7 +63,7 @@ public MigrateCmd(CollectionCommandContext ccc) { } @Override - public void call(ClusterState clusterState, ZkNodeProps message, NamedList results) + public void call(AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList results) throws Exception { String extSourceCollectionName = message.getStr("collection"); String splitKey = message.getStr("split.key"); @@ -89,13 +87,15 @@ public void call(ClusterState clusterState, ZkNodeProps message, NamedList results, - String asyncId, ZkNodeProps message) throws Exception { String tempSourceCollectionName = "split_" + sourceSlice.getName() + "_temp_" + targetSlice.getName(); ZkStateReader zkStateReader = ccc.getZkStateReader(); - if (clusterState.hasCollection(tempSourceCollectionName)) { + if (adminCmdContext.getClusterState().hasCollection(tempSourceCollectionName)) { log.info("Deleting temporary collection: {}", tempSourceCollectionName); - Map props = - Map.of(Overseer.QUEUE_OPERATION, DELETE.toLower(), NAME, tempSourceCollectionName); - try { new DeleteCollectionCmd(ccc) - .call(zkStateReader.getClusterState(), new ZkNodeProps(props), results); - clusterState = zkStateReader.getClusterState(); + .call( + adminCmdContext + .subRequestContext(DELETE) + .withClusterState(zkStateReader.getClusterState()), + new ZkNodeProps(Map.of(NAME, tempSourceCollectionName)), + results); + adminCmdContext.withClusterState(zkStateReader.getClusterState()); } catch (Exception e) { log.warn( "Unable to clean up existing temporary collection: {}", tempSourceCollectionName, e); @@ -234,7 +231,7 @@ private void migrateKey( { final ShardRequestTracker shardRequestTracker = - CollectionHandlingUtils.asyncRequestTracker(asyncId, ccc); + CollectionHandlingUtils.asyncRequestTracker(adminCmdContext, ccc); shardRequestTracker.sendShardRequest(targetLeader.getNodeName(), params, shardHandler); shardRequestTracker.processResponses( @@ -312,17 +309,26 @@ private void migrateKey( configName, CollectionHandlingUtils.CREATE_NODE_SET, sourceLeader.getNodeName()); - if (asyncId != null) { - String internalAsyncId = asyncId + Math.abs(System.nanoTime()); - props.put(ASYNC, internalAsyncId); + String internalAsyncId = null; + if (adminCmdContext.getAsyncId() != null) { + internalAsyncId = adminCmdContext.getAsyncId() + Math.abs(System.nanoTime()); } log.info("Creating temporary collection: {}", props); - new CreateCollectionCmd(ccc).call(clusterState, new ZkNodeProps(props), results); + new CreateCollectionCmd(ccc) + .call( + adminCmdContext.subRequestContext(CREATE, internalAsyncId), + new ZkNodeProps(props), + results); // refresh cluster state - clusterState = zkStateReader.getClusterState(); + adminCmdContext.withClusterState(zkStateReader.getClusterState()); Slice tempSourceSlice = - clusterState.getCollection(tempSourceCollectionName).getSlices().iterator().next(); + adminCmdContext + .getClusterState() + .getCollection(tempSourceCollectionName) + .getSlices() + .iterator() + .next(); Replica tempSourceLeader = zkStateReader.getLeaderRetry(tempSourceCollectionName, tempSourceSlice.getName(), 120000); @@ -349,7 +355,7 @@ private void migrateKey( cmd.setOnlyIfLeader(true); { final ShardRequestTracker syncRequestTracker = - CollectionHandlingUtils.syncRequestTracker(ccc); + CollectionHandlingUtils.syncRequestTracker(adminCmdContext, ccc); // we don't want this to happen asynchronously syncRequestTracker.sendShardRequest( tempSourceLeader.getNodeName(), new ModifiableSolrParams(cmd.getParams()), shardHandler); @@ -374,7 +380,7 @@ private void migrateKey( { final ShardRequestTracker shardRequestTracker = - CollectionHandlingUtils.asyncRequestTracker(asyncId, ccc); + CollectionHandlingUtils.asyncRequestTracker(adminCmdContext, ccc); shardRequestTracker.sendShardRequest(tempNodeName, params, shardHandler); shardRequestTracker.processResponses( results, shardHandler, true, "MIGRATE failed to invoke SPLIT core admin command"); @@ -392,7 +398,6 @@ private void migrateKey( tempSourceSlice.getName(), Replica.Type.defaultType()); props = new HashMap<>(); - props.put(Overseer.QUEUE_OPERATION, ADDREPLICA.toLower()); props.put(COLLECTION_PROP, tempSourceCollectionName); props.put(SHARD_ID_PROP, tempSourceSlice.getName()); props.put("node", targetLeader.getNodeName()); @@ -404,14 +409,13 @@ private void migrateKey( } } // add async param - if (asyncId != null) { - props.put(ASYNC, asyncId); - } - new AddReplicaCmd(ccc).addReplica(clusterState, new ZkNodeProps(props), results, null); + new AddReplicaCmd(ccc) + .addReplica( + adminCmdContext.subRequestContext(ADDREPLICA), new ZkNodeProps(props), results, null); { final ShardRequestTracker syncRequestTracker = - CollectionHandlingUtils.syncRequestTracker(ccc); + CollectionHandlingUtils.syncRequestTracker(adminCmdContext, ccc); syncRequestTracker.processResponses( results, shardHandler, @@ -442,7 +446,7 @@ private void migrateKey( { final ShardRequestTracker shardRequestTracker = - CollectionHandlingUtils.asyncRequestTracker(asyncId, ccc); + CollectionHandlingUtils.asyncRequestTracker(adminCmdContext, ccc); shardRequestTracker.sendShardRequest(tempSourceLeader.getNodeName(), params, shardHandler); shardRequestTracker.processResponses( @@ -462,7 +466,7 @@ private void migrateKey( { final ShardRequestTracker shardRequestTracker = - CollectionHandlingUtils.asyncRequestTracker(asyncId, ccc); + CollectionHandlingUtils.asyncRequestTracker(adminCmdContext, ccc); shardRequestTracker.sendShardRequest(targetLeader.getNodeName(), params, shardHandler); String msg = @@ -482,16 +486,20 @@ private void migrateKey( { final ShardRequestTracker shardRequestTracker = - CollectionHandlingUtils.asyncRequestTracker(asyncId, ccc); + CollectionHandlingUtils.asyncRequestTracker(adminCmdContext, ccc); shardRequestTracker.sendShardRequest(targetLeader.getNodeName(), params, shardHandler); shardRequestTracker.processResponses( results, shardHandler, true, "MIGRATE failed to request node to apply buffered updates"); } try { log.info("Deleting temporary collection: {}", tempSourceCollectionName); - props = Map.of(Overseer.QUEUE_OPERATION, DELETE.toLower(), NAME, tempSourceCollectionName); new DeleteCollectionCmd(ccc) - .call(zkStateReader.getClusterState(), new ZkNodeProps(props), results); + .call( + adminCmdContext + .subRequestContext(DELETE) + .withClusterState(zkStateReader.getClusterState()), + new ZkNodeProps(Map.of(NAME, tempSourceCollectionName)), + results); } catch (Exception e) { log.error( "Unable to delete temporary collection: {}. Please remove it manually", diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/MigrateReplicasCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/MigrateReplicasCmd.java index 3912e096cb6..e4381ad888e 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/MigrateReplicasCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/MigrateReplicasCmd.java @@ -17,8 +17,6 @@ package org.apache.solr.cloud.api.collections; -import static org.apache.solr.common.params.CommonAdminParams.ASYNC; - import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -48,7 +46,7 @@ public MigrateReplicasCmd(CollectionCommandContext ccc) { } @Override - public void call(ClusterState state, ZkNodeProps message, NamedList results) + public void call(AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList results) throws Exception { ZkStateReader zkStateReader = ccc.getZkStateReader(); Set sourceNodes = getNodesFromParam(message, CollectionParams.SOURCE_NODES); @@ -58,7 +56,6 @@ public void call(ClusterState state, ZkNodeProps message, NamedList resu throw new SolrException( SolrException.ErrorCode.BAD_REQUEST, "sourceNodes is a required param"); } - String async = message.getStr(ASYNC); int timeout = message.getInt("timeout", 10 * 60); // 10 minutes boolean parallel = message.getBool("parallel", false); ClusterState clusterState = zkStateReader.getClusterState(); @@ -121,7 +118,7 @@ public void call(ClusterState state, ZkNodeProps message, NamedList resu boolean migrationSuccessful = ReplicaMigrationUtils.migrateReplicas( - ccc, replicaMovements, parallel, waitForFinalState, timeout, async, results); + ccc, adminCmdContext, replicaMovements, parallel, waitForFinalState, timeout, results); if (migrationSuccessful) { results.add( "success", diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/MoveReplicaCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/MoveReplicaCmd.java index 9bcae03e812..89a5f32c930 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/MoveReplicaCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/MoveReplicaCmd.java @@ -22,7 +22,6 @@ import static org.apache.solr.common.cloud.ZkStateReader.REPLICA_PROP; import static org.apache.solr.common.cloud.ZkStateReader.SHARD_ID_PROP; import static org.apache.solr.common.params.CollectionAdminParams.FOLLOW_ALIASES; -import static org.apache.solr.common.params.CommonAdminParams.ASYNC; import static org.apache.solr.common.params.CommonAdminParams.IN_PLACE_MOVE; import static org.apache.solr.common.params.CommonAdminParams.TIMEOUT; import static org.apache.solr.common.params.CommonAdminParams.WAIT_FOR_FINAL_STATE; @@ -37,7 +36,6 @@ import org.apache.solr.cloud.ActiveReplicaWatcher; import org.apache.solr.common.SolrCloseableLatch; import org.apache.solr.common.SolrException; -import org.apache.solr.common.cloud.ClusterState; import org.apache.solr.common.cloud.DocCollection; import org.apache.solr.common.cloud.Replica; import org.apache.solr.common.cloud.Slice; @@ -60,13 +58,14 @@ public MoveReplicaCmd(CollectionCommandContext ccc) { } @Override - public void call(ClusterState state, ZkNodeProps message, NamedList results) + public void call(AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList results) throws Exception { - moveReplica(ccc.getZkStateReader().getClusterState(), message, results); + moveReplica(adminCmdContext, message, results); } private void moveReplica( - ClusterState clusterState, ZkNodeProps message, NamedList results) throws Exception { + AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList results) + throws Exception { if (log.isDebugEnabled()) { log.debug("moveReplica() : {}", Utils.toJSONString(message)); } @@ -77,8 +76,6 @@ private void moveReplica( boolean inPlaceMove = message.getBool(IN_PLACE_MOVE, true); int timeout = message.getInt(TIMEOUT, 10 * 60); // 10 minutes - String async = message.getStr(ASYNC); - boolean followAliases = message.getBool(FOLLOW_ALIASES, false); String collection; if (followAliases) { @@ -88,15 +85,18 @@ private void moveReplica( collection = extCollection; } - DocCollection coll = clusterState.getCollection(collection); + DocCollection coll = adminCmdContext.getClusterState().getCollection(collection); if (coll == null) { throw new SolrException( SolrException.ErrorCode.BAD_REQUEST, "Collection: " + collection + " does not exist"); } - if (!clusterState.getLiveNodes().contains(targetNode)) { + if (!adminCmdContext.getClusterState().getLiveNodes().contains(targetNode)) { throw new SolrException( SolrException.ErrorCode.BAD_REQUEST, - "Target node: " + targetNode + " not in live nodes: " + clusterState.getLiveNodes()); + "Target node: " + + targetNode + + " not in live nodes: " + + adminCmdContext.getClusterState().getLiveNodes()); } Replica replica = null; if (message.containsKey(REPLICA_PROP)) { @@ -156,41 +156,31 @@ private void moveReplica( if (isSharedFS && inPlaceMove) { log.debug("-- moveSharedFsReplica"); moveSharedFsReplica( - clusterState, + adminCmdContext, results, dataDir.toString(), targetNode, - async, replica, timeout, waitForFinalState); } else { log.debug("-- moveNormalReplica (inPlaceMove={}, isSharedFS={}", inPlaceMove, isSharedFS); moveNormalReplica( - clusterState, - results, - targetNode, - async, - coll, - replica, - slice, - timeout, - waitForFinalState); + adminCmdContext, results, targetNode, coll, replica, slice, timeout, waitForFinalState); } } private void moveSharedFsReplica( - ClusterState clusterState, + AdminCmdContext adminCmdContext, NamedList results, String dataDir, String targetNode, - String async, Replica replica, int timeout, boolean waitForFinalState) throws Exception { String skipCreateReplicaInClusterState = "true"; - if (clusterState.getLiveNodes().contains(replica.getNodeName())) { + if (adminCmdContext.getClusterState().getLiveNodes().contains(replica.getNodeName())) { skipCreateReplicaInClusterState = "false"; ZkNodeProps removeReplicasProps = new ZkNodeProps( @@ -199,11 +189,14 @@ private void moveSharedFsReplica( REPLICA_PROP, replica.getName()); removeReplicasProps.getProperties().put(CoreAdminParams.DELETE_DATA_DIR, false); removeReplicasProps.getProperties().put(CoreAdminParams.DELETE_INDEX, false); - if (async != null) removeReplicasProps.getProperties().put(ASYNC, async); NamedList deleteResult = new NamedList<>(); try { new DeleteReplicaCmd(ccc) - .deleteReplica(clusterState, removeReplicasProps, deleteResult, null); + .deleteReplica( + adminCmdContext.subRequestContext(CollectionParams.CollectionAction.DELETEREPLICA), + removeReplicasProps, + deleteResult, + null); } catch (SolrException e) { // assume this failed completely so there's nothing to roll back deleteResult.add("failure", e.toString()); @@ -259,11 +252,16 @@ private void moveSharedFsReplica( ZkStateReader.REPLICA_TYPE, replica.getType().name()); - if (async != null) addReplicasProps.getProperties().put(ASYNC, async); NamedList addResult = new NamedList<>(); try { new AddReplicaCmd(ccc) - .addReplica(ccc.getZkStateReader().getClusterState(), addReplicasProps, addResult, null); + .addReplica( + adminCmdContext + .subRequestContext(CollectionParams.CollectionAction.ADDREPLICA) + .withClusterState(ccc.getZkStateReader().getClusterState()), + addReplicasProps, + addResult, + null); } catch (Exception e) { // fatal error - try rolling back String errorString = @@ -279,7 +277,13 @@ private void moveSharedFsReplica( addReplicasProps = addReplicasProps.plus(CoreAdminParams.NODE, replica.getNodeName()); NamedList rollback = new NamedList<>(); new AddReplicaCmd(ccc) - .addReplica(ccc.getZkStateReader().getClusterState(), addReplicasProps, rollback, null); + .addReplica( + adminCmdContext + .subRequestContext(CollectionParams.CollectionAction.ADDREPLICA) + .withClusterState(ccc.getZkStateReader().getClusterState()), + addReplicasProps, + rollback, + null); if (rollback.get("failure") != null) { throw new SolrException( SolrException.ErrorCode.SERVER_ERROR, @@ -307,7 +311,13 @@ private void moveSharedFsReplica( NamedList rollback = new NamedList<>(); try { new AddReplicaCmd(ccc) - .addReplica(ccc.getZkStateReader().getClusterState(), addReplicasProps, rollback, null); + .addReplica( + adminCmdContext + .subRequestContext(CollectionParams.CollectionAction.ADDREPLICA) + .withClusterState(ccc.getZkStateReader().getClusterState()), + addReplicasProps, + rollback, + null); } catch (Exception e) { throw new SolrException( SolrException.ErrorCode.SERVER_ERROR, @@ -337,10 +347,9 @@ private void moveSharedFsReplica( } private void moveNormalReplica( - ClusterState clusterState, + AdminCmdContext adminCmdContext, NamedList results, String targetNode, - String async, DocCollection coll, Replica replica, Slice slice, @@ -366,12 +375,17 @@ private void moveNormalReplica( ZkStateReader.REPLICA_TYPE, replica.getType().name()); - if (async != null) addReplicasProps.getProperties().put(ASYNC, async); NamedList addResult = new NamedList<>(); SolrCloseableLatch countDownLatch = new SolrCloseableLatch(1, ccc.getCloseableToLatchOn()); ActiveReplicaWatcher watcher = null; ZkNodeProps props = - new AddReplicaCmd(ccc).addReplica(clusterState, addReplicasProps, addResult, null).get(0); + new AddReplicaCmd(ccc) + .addReplica( + adminCmdContext.subRequestContext(CollectionParams.CollectionAction.ADDREPLICA), + addReplicasProps, + addResult, + null) + .get(0); log.debug("props {}", props); if (replica.equals(slice.getLeader()) || waitForFinalState) { watcher = @@ -427,11 +441,14 @@ private void moveNormalReplica( COLLECTION_PROP, coll.getName(), SHARD_ID_PROP, slice.getName(), REPLICA_PROP, replica.getName()); - if (async != null) removeReplicasProps.getProperties().put(ASYNC, async); NamedList deleteResult = new NamedList<>(); try { new DeleteReplicaCmd(ccc) - .deleteReplica(clusterState, removeReplicasProps, deleteResult, null); + .deleteReplica( + adminCmdContext.subRequestContext(CollectionParams.CollectionAction.DELETEREPLICA), + removeReplicasProps, + deleteResult, + null); } catch (SolrException e) { deleteResult.add("failure", e.toString()); } diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerCollectionMessageHandler.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerCollectionMessageHandler.java index df933b597fe..8573bf63764 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerCollectionMessageHandler.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerCollectionMessageHandler.java @@ -21,11 +21,14 @@ import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP; import static org.apache.solr.common.cloud.ZkStateReader.REPLICA_PROP; import static org.apache.solr.common.cloud.ZkStateReader.SHARD_ID_PROP; +import static org.apache.solr.common.params.CollectionAdminParams.CALLING_LOCK_IDS_HEADER; +import static org.apache.solr.common.params.CommonAdminParams.ASYNC; import static org.apache.solr.common.params.CommonParams.NAME; import java.io.IOException; import java.lang.invoke.MethodHandles; import java.util.Arrays; +import java.util.List; import java.util.concurrent.ExecutorService; import java.util.concurrent.SynchronousQueue; import java.util.concurrent.TimeUnit; @@ -45,6 +48,7 @@ import org.apache.solr.common.util.ExecutorUtil; import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.SolrNamedThreadFactory; +import org.apache.solr.common.util.StrUtils; import org.apache.solr.common.util.TimeSource; import org.apache.solr.handler.component.HttpShardHandlerFactory; import org.apache.solr.logging.MDCLoggingContext; @@ -112,7 +116,7 @@ public OverseerCollectionMessageHandler( } @Override - public OverseerSolrResponse processMessage(ZkNodeProps message, String operation) { + public OverseerSolrResponse processMessage(ZkNodeProps message, String operation, Lock lock) { // sometimes overseer messages have the collection name in 'name' field, not 'collection' MDCLoggingContext.setCollection( message.getStr(COLLECTION_PROP) != null @@ -127,7 +131,11 @@ public OverseerSolrResponse processMessage(ZkNodeProps message, String operation CollectionAction action = getCollectionAction(operation); CollApiCmds.CollectionApiCommand command = commandMapper.getActionCommand(action); if (command != null) { - command.call(cloudManager.getClusterState(), message, results); + AdminCmdContext adminCmdContext = new AdminCmdContext(action, message.getStr(ASYNC)); + adminCmdContext.setLockId(lock.id()); + adminCmdContext.setCallingLockIds(message.getStr(CALLING_LOCK_IDS_HEADER)); + adminCmdContext.withClusterState(cloudManager.getClusterState()); + command.call(adminCmdContext, message, results); } else { throw new SolrException(ErrorCode.BAD_REQUEST, "Unknown operation:" + operation); } @@ -188,12 +196,18 @@ public Lock lockTask(ZkNodeProps message, long batchSessionId) { sessionId = batchSessionId; } + List callingLockIds = null; + String callingLockIdsString = message.getStr(CALLING_LOCK_IDS_HEADER); + if (StrUtils.isNotBlank(callingLockIdsString)) { + callingLockIds = List.of(callingLockIdsString.split(",")); + } return lockSession.lock( getCollectionAction(message.getStr(Overseer.QUEUE_OPERATION)), Arrays.asList( getTaskKey(message), message.getStr(ZkStateReader.SHARD_ID_PROP), - message.getStr(ZkStateReader.REPLICA_PROP))); + message.getStr(ZkStateReader.REPLICA_PROP)), + callingLockIds); } @Override diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerRoleCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerRoleCmd.java index fc400ffb26a..32f10bc1c6f 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerRoleCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerRoleCmd.java @@ -25,7 +25,6 @@ import java.util.List; import java.util.Map; import org.apache.solr.cloud.OverseerNodePrioritizer; -import org.apache.solr.common.cloud.ClusterState; import org.apache.solr.common.cloud.SolrZkClient; import org.apache.solr.common.cloud.ZkNodeProps; import org.apache.solr.common.cloud.ZkStateReader; @@ -55,7 +54,7 @@ public OverseerRoleCmd( } @Override - public void call(ClusterState state, ZkNodeProps message, NamedList results) + public void call(AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList results) throws Exception { if (ccc.isDistributedCollectionAPI()) { // No Overseer (not accessible from Collection API command execution in any case) so this diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerStatusCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerStatusCmd.java index 74b2fd57bf9..031808b5b05 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerStatusCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerStatusCmd.java @@ -24,7 +24,6 @@ import java.util.Map; import org.apache.solr.cloud.OverseerTaskProcessor; import org.apache.solr.cloud.Stats; -import org.apache.solr.common.cloud.ClusterState; import org.apache.solr.common.cloud.ZkNodeProps; import org.apache.solr.common.cloud.ZkStateReader; import org.apache.solr.common.util.NamedList; @@ -157,7 +156,7 @@ public OverseerStatusCmd(CollectionCommandContext ccc) { } @Override - public void call(ClusterState state, ZkNodeProps message, NamedList results) + public void call(AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList results) throws Exception { // If Collection API execution is distributed, we're not running on the Overseer node so can't // return any Overseer stats. diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/ReindexCollectionCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/ReindexCollectionCmd.java index 8909841c936..e72cfbe04a4 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/ReindexCollectionCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/ReindexCollectionCmd.java @@ -169,7 +169,7 @@ public ReindexCollectionCmd(CollectionCommandContext ccc) { } @Override - public void call(ClusterState clusterState, ZkNodeProps message, NamedList results) + public void call(AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList results) throws Exception { log.debug("*** called: {}", message); @@ -188,6 +188,7 @@ public void call(ClusterState clusterState, ZkNodeProps message, NamedList propMap = new HashMap<>(); - propMap.put(Overseer.QUEUE_OPERATION, CollectionParams.CollectionAction.CREATE.toLower()); propMap.put(CommonParams.NAME, targetCollection); propMap.put(ZkStateReader.NUM_SHARDS_PROP, numShards); propMap.put(CollectionAdminParams.COLL_CONF, configName); @@ -338,7 +338,13 @@ public void call(ClusterState clusterState, ZkNodeProps message, NamedList(); - new CreateCollectionCmd(ccc).call(clusterState, cmd, cmdResults); + new CreateCollectionCmd(ccc) + .call( + adminCmdContext + .subRequestContext(CollectionParams.CollectionAction.CREATE, null) + .withClusterState(ccc.getSolrCloudManager().getClusterState()), + cmd, + cmdResults); createdTarget = true; CollectionHandlingUtils.checkResults( "creating target collection " + targetCollection, cmdResults, true); @@ -346,14 +352,19 @@ public void call(ClusterState clusterState, ZkNodeProps message, NamedList(); - new CreateCollectionCmd(ccc).call(clusterState, cmd, cmdResults); + new CreateCollectionCmd(ccc) + .call( + adminCmdContext + .subRequestContext(CollectionParams.CollectionAction.CREATE, null) + .withClusterState(ccc.getSolrCloudManager().getClusterState()), + cmd, + cmdResults); CollectionHandlingUtils.checkResults( "creating checkpoint collection " + chkCollection, cmdResults, true); // wait for a while until we see both collections @@ -366,8 +377,6 @@ public void call(ClusterState clusterState, ZkNodeProps message, NamedList(); - new CreateAliasCmd(ccc).call(clusterState, cmd, cmdResults); + new CreateAliasCmd(ccc) + .call( + adminCmdContext + .subRequestContext(CollectionParams.CollectionAction.CREATEALIAS, null) + .withClusterState(ccc.getSolrCloudManager().getClusterState()), + cmd, + cmdResults); CollectionHandlingUtils.checkResults( "setting up alias " + extCollection + " -> " + targetCollection, cmdResults, true); reindexingState.put("alias", extCollection + " -> " + targetCollection); @@ -498,30 +513,29 @@ public void call(ClusterState clusterState, ZkNodeProps message, NamedList(); - new DeleteCollectionCmd(ccc).call(clusterState, cmd, cmdResults); + new DeleteCollectionCmd(ccc) + .call( + adminCmdContext + .subRequestContext(CollectionParams.CollectionAction.DELETE, null) + .withClusterState(ccc.getSolrCloudManager().getClusterState()), + new ZkNodeProps(CommonParams.NAME, chkCollection), + cmdResults); CollectionHandlingUtils.checkResults( "deleting checkpoint collection " + chkCollection, cmdResults, true); // 7. optionally delete the source collection if (removeSource) { log.debug("- deleting source collection"); - cmd = - new ZkNodeProps( - Overseer.QUEUE_OPERATION, - CollectionParams.CollectionAction.DELETE.toLower(), - CommonParams.NAME, - collection, - FOLLOW_ALIASES, - "false"); + cmd = new ZkNodeProps(CommonParams.NAME, collection, FOLLOW_ALIASES, "false"); cmdResults = new NamedList<>(); - new DeleteCollectionCmd(ccc).call(clusterState, cmd, cmdResults); + new DeleteCollectionCmd(ccc) + .call( + adminCmdContext + .subRequestContext(CollectionParams.CollectionAction.DELETE, null) + .withClusterState(ccc.getSolrCloudManager().getClusterState()), + cmd, + cmdResults); CollectionHandlingUtils.checkResults( "deleting source collection " + collection, cmdResults, true); } else { @@ -575,6 +589,7 @@ public void call(ClusterState clusterState, ZkNodeProps message, NamedList executeDaemonAction( } private void cleanup( + AdminCmdContext adminCmdContext, String collection, String targetCollection, String chkCollection, @@ -891,30 +907,29 @@ private void cleanup( && clusterState.hasCollection(targetCollection)) { log.debug(" -- removing {}", targetCollection); ZkNodeProps cmd = - new ZkNodeProps( - Overseer.QUEUE_OPERATION, - CollectionParams.CollectionAction.DELETE.toLower(), - CommonParams.NAME, - targetCollection, - FOLLOW_ALIASES, - "false"); - new DeleteCollectionCmd(ccc).call(clusterState, cmd, cmdResults); + new ZkNodeProps(CommonParams.NAME, targetCollection, FOLLOW_ALIASES, "false"); + new DeleteCollectionCmd(ccc) + .call( + adminCmdContext + .subRequestContext(CollectionParams.CollectionAction.DELETE, null) + .withClusterState(clusterState), + cmd, + cmdResults); CollectionHandlingUtils.checkResults( "CLEANUP: deleting target collection " + targetCollection, cmdResults, false); } // remove chk collection if (clusterState.hasCollection(chkCollection)) { log.debug(" -- removing {}", chkCollection); - ZkNodeProps cmd = - new ZkNodeProps( - Overseer.QUEUE_OPERATION, - CollectionParams.CollectionAction.DELETE.toLower(), - CommonParams.NAME, - chkCollection, - FOLLOW_ALIASES, - "false"); + ZkNodeProps cmd = new ZkNodeProps(CommonParams.NAME, chkCollection, FOLLOW_ALIASES, "false"); cmdResults = new NamedList<>(); - new DeleteCollectionCmd(ccc).call(clusterState, cmd, cmdResults); + new DeleteCollectionCmd(ccc) + .call( + adminCmdContext + .subRequestContext(CollectionParams.CollectionAction.DELETE, null) + .withClusterState(clusterState), + cmd, + cmdResults); CollectionHandlingUtils.checkResults( "CLEANUP: deleting checkpoint collection " + chkCollection, cmdResults, false); } diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/RenameCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/RenameCmd.java index 8677ecf72ef..be0c3cfa32b 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/RenameCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/RenameCmd.java @@ -22,7 +22,6 @@ import java.lang.invoke.MethodHandles; import org.apache.solr.common.SolrException; import org.apache.solr.common.cloud.Aliases; -import org.apache.solr.common.cloud.ClusterState; import org.apache.solr.common.cloud.ZkNodeProps; import org.apache.solr.common.params.CollectionAdminParams; import org.apache.solr.common.params.CoreAdminParams; @@ -41,7 +40,7 @@ public RenameCmd(CollectionCommandContext ccc) { } @Override - public void call(ClusterState state, ZkNodeProps message, NamedList results) + public void call(AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList results) throws Exception { String extCollectionName = message.getStr(CoreAdminParams.NAME); String target = message.getStr(CollectionAdminParams.TARGET); @@ -64,7 +63,7 @@ public void call(ClusterState state, ZkNodeProps message, NamedList resu } else { collectionName = extCollectionName; } - if (!state.hasCollection(collectionName)) { + if (!adminCmdContext.getClusterState().hasCollection(collectionName)) { throw new SolrException( SolrException.ErrorCode.BAD_REQUEST, "source collection '" + collectionName + "' not found."); diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/ReplaceNodeCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/ReplaceNodeCmd.java index 5134e9953bd..382f7dd9e2e 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/ReplaceNodeCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/ReplaceNodeCmd.java @@ -17,8 +17,6 @@ package org.apache.solr.cloud.api.collections; -import static org.apache.solr.common.params.CommonAdminParams.ASYNC; - import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -45,7 +43,7 @@ public ReplaceNodeCmd(CollectionCommandContext ccc) { } @Override - public void call(ClusterState state, ZkNodeProps message, NamedList results) + public void call(AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList results) throws Exception { ZkStateReader zkStateReader = ccc.getZkStateReader(); String source = message.getStr(CollectionParams.SOURCE_NODE); @@ -55,7 +53,6 @@ public void call(ClusterState state, ZkNodeProps message, NamedList resu throw new SolrException( SolrException.ErrorCode.BAD_REQUEST, "sourceNode is a required param"); } - String async = message.getStr(ASYNC); int timeout = message.getInt("timeout", 10 * 60); // 10 minutes boolean parallel = message.getBool("parallel", false); ClusterState clusterState = zkStateReader.getClusterState(); @@ -107,7 +104,7 @@ public void call(ClusterState state, ZkNodeProps message, NamedList resu boolean migrationSuccessful = ReplicaMigrationUtils.migrateReplicas( - ccc, replicaMovements, parallel, waitForFinalState, timeout, async, results); + ccc, adminCmdContext, replicaMovements, parallel, waitForFinalState, timeout, results); if (migrationSuccessful) { results.add( "success", diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/ReplicaMigrationUtils.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/ReplicaMigrationUtils.java index d21aa3a5f95..498695f4f9f 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/ReplicaMigrationUtils.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/ReplicaMigrationUtils.java @@ -17,8 +17,6 @@ package org.apache.solr.cloud.api.collections; -import static org.apache.solr.common.params.CommonAdminParams.ASYNC; - import java.io.IOException; import java.lang.invoke.MethodHandles; import java.util.ArrayList; @@ -37,6 +35,7 @@ import org.apache.solr.common.cloud.Replica; import org.apache.solr.common.cloud.ZkNodeProps; import org.apache.solr.common.cloud.ZkStateReader; +import org.apache.solr.common.params.CollectionParams; import org.apache.solr.common.params.CoreAdminParams; import org.apache.solr.common.util.NamedList; import org.apache.zookeeper.KeeperException; @@ -53,22 +52,22 @@ public class ReplicaMigrationUtils { *

If an error occurs during the creation of new replicas, all new replicas will be deleted. * * @param ccc The collection command context to use from the API that calls this method + * @param adminCmdContext The context of the originating admin request * @param movements a map from replica to the new node that the replica should live on * @param parallel whether the replica creations should be done in parallel * @param waitForFinalState wait for the final state of all newly created replicas before * continuing * @param timeout the amount of time to wait for new replicas to be created - * @param asyncId If provided, the command will be run under the given asyncId * @param results push results (successful and failure) onto this list * @return whether the command was successful */ static boolean migrateReplicas( CollectionCommandContext ccc, + AdminCmdContext adminCmdContext, Map movements, boolean parallel, boolean waitForFinalState, int timeout, - String asyncId, NamedList results) throws IOException, InterruptedException, KeeperException { // how many leaders are we moving? for these replicas we have to make sure that either: @@ -92,8 +91,6 @@ static boolean migrateReplicas( SolrCloseableLatch replicasToRecover = new SolrCloseableLatch(numLeaders, ccc.getCloseableToLatchOn()); - ClusterState clusterState = ccc.getZkStateReader().getClusterState(); - for (Map.Entry movement : movements.entrySet()) { Replica sourceReplica = movement.getKey(); String targetNode = movement.getValue(); @@ -111,12 +108,13 @@ static boolean migrateReplicas( .toFullProps() .plus("parallel", String.valueOf(parallel)) .plus(CoreAdminParams.NODE, targetNode); - if (asyncId != null) msg.getProperties().put(ASYNC, asyncId); NamedList nl = new NamedList<>(); final ZkNodeProps addedReplica = new AddReplicaCmd(ccc) .addReplica( - clusterState, + adminCmdContext + .subRequestContext(CollectionParams.CollectionAction.ADDREPLICA) + .withClusterState(ccc.getZkStateReader().getClusterState()), msg, nl, () -> { @@ -213,7 +211,9 @@ static boolean migrateReplicas( try { new DeleteReplicaCmd(ccc) .deleteReplica( - ccc.getZkStateReader().getClusterState(), + adminCmdContext + .subRequestContext(CollectionParams.CollectionAction.DELETEREPLICA) + .withClusterState(ccc.getZkStateReader().getClusterState()), createdReplica.plus("parallel", "true"), deleteResult, () -> { @@ -241,16 +241,14 @@ static boolean migrateReplicas( // we have reached this far, meaning all replicas should have been recreated. // now cleanup the original replicas - return cleanupReplicas( - results, ccc.getZkStateReader().getClusterState(), movements.keySet(), ccc, asyncId); + return cleanupReplicas(results, adminCmdContext, movements.keySet(), ccc); } static boolean cleanupReplicas( NamedList results, - ClusterState clusterState, + AdminCmdContext adminCmdContext, Collection sourceReplicas, - CollectionCommandContext ccc, - String async) + CollectionCommandContext ccc) throws IOException, InterruptedException { SolrCloseableLatch cleanupLatch = new SolrCloseableLatch(sourceReplicas.size(), ccc.getCloseableToLatchOn()); @@ -268,10 +266,11 @@ static boolean cleanupReplicas( NamedList deleteResult = new NamedList<>(); try { ZkNodeProps cmdMessage = sourceReplica.toFullProps(); - if (async != null) cmdMessage = cmdMessage.plus(ASYNC, async); new DeleteReplicaCmd(ccc) .deleteReplica( - clusterState, + adminCmdContext + .subRequestContext(CollectionParams.CollectionAction.DELETEREPLICA) + .withClusterState(ccc.getZkStateReader().getClusterState()), cmdMessage.plus("parallel", "true"), deleteResult, () -> { diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/RestoreCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/RestoreCmd.java index 55d1725e6ec..93df952eb27 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/RestoreCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/RestoreCmd.java @@ -21,9 +21,10 @@ import static org.apache.solr.common.cloud.ZkStateReader.REPLICATION_FACTOR; import static org.apache.solr.common.cloud.ZkStateReader.REPLICA_TYPE; import static org.apache.solr.common.cloud.ZkStateReader.SHARD_ID_PROP; +import static org.apache.solr.common.params.CollectionParams.CollectionAction.ADDREPLICA; import static org.apache.solr.common.params.CollectionParams.CollectionAction.CREATE; import static org.apache.solr.common.params.CollectionParams.CollectionAction.CREATESHARD; -import static org.apache.solr.common.params.CommonAdminParams.ASYNC; +import static org.apache.solr.common.params.CollectionParams.CollectionAction.MODIFYCOLLECTION; import static org.apache.solr.common.params.CommonParams.NAME; import java.io.Closeable; @@ -92,10 +93,10 @@ public RestoreCmd(CollectionCommandContext ccc) { } @Override - public void call(ClusterState state, ZkNodeProps message, NamedList results) + public void call(AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList results) throws Exception { - try (RestoreContext restoreContext = new RestoreContext(message, ccc)) { - if (state.hasCollection(restoreContext.restoreCollectionName)) { + try (RestoreContext restoreContext = new RestoreContext(adminCmdContext, message, ccc)) { + if (adminCmdContext.getClusterState().hasCollection(restoreContext.restoreCollectionName)) { RestoreOnExistingCollection restoreOnExistingCollection = new RestoreOnExistingCollection(restoreContext); restoreOnExistingCollection.process(restoreContext, results); @@ -111,14 +112,13 @@ public void call(ClusterState state, ZkNodeProps message, NamedList resu private void requestReplicasToRestore( NamedList results, DocCollection restoreCollection, - ClusterState clusterState, + AdminCmdContext adminCmdContext, BackupProperties backupProperties, URI backupPath, String repo, - ShardHandler shardHandler, - String asyncId) { + ShardHandler shardHandler) { ShardRequestTracker shardRequestTracker = - CollectionHandlingUtils.asyncRequestTracker(asyncId, ccc); + CollectionHandlingUtils.asyncRequestTracker(adminCmdContext, ccc); // Copy data from backed up index to each replica for (Slice slice : restoreCollection.getSlices()) { ModifiableSolrParams params = new ModifiableSolrParams(); @@ -131,7 +131,8 @@ private void requestReplicasToRestore( } params.set(CoreAdminParams.BACKUP_LOCATION, backupPath.toASCIIString()); params.set(CoreAdminParams.BACKUP_REPOSITORY, repo); - shardRequestTracker.sliceCmd(clusterState, params, null, slice, shardHandler); + shardRequestTracker.sliceCmd( + adminCmdContext.getClusterState(), params, null, slice, shardHandler); } shardRequestTracker.processResponses( new NamedList<>(), shardHandler, true, "Could not restore core"); @@ -140,10 +141,10 @@ private void requestReplicasToRestore( /** Encapsulates the parsing and access for common parameters restore parameters and values */ private static class RestoreContext implements Closeable { + final AdminCmdContext adminCmdContext; final String restoreCollectionName; final String backupName; final String backupCollection; - final String asyncId; final String repo; final String restoreConfigName; final int backupId; @@ -159,10 +160,12 @@ private static class RestoreContext implements Closeable { final DocCollection backupCollectionState; final ShardHandler shardHandler; - private RestoreContext(ZkNodeProps message, CollectionCommandContext ccc) throws IOException { + private RestoreContext( + AdminCmdContext adminCmdContext, ZkNodeProps message, CollectionCommandContext ccc) + throws IOException { + this.adminCmdContext = adminCmdContext; this.restoreCollectionName = message.getStr(COLLECTION_PROP); this.backupName = message.getStr(NAME); // of backup - this.asyncId = message.getStr(ASYNC); this.repo = message.getStr(CoreAdminParams.BACKUP_REPOSITORY); this.backupId = message.getInt(CoreAdminParams.BACKUP_ID, -1); @@ -236,10 +239,10 @@ public void process(NamedList results, RestoreContext rc) throws Excepti rc.backupName, rc.location); createCoreLessCollection( + rc.adminCmdContext.withClusterState(rc.zkStateReader.getClusterState()), rc.restoreCollectionName, rc.restoreConfigName, - rc.backupCollectionState, - rc.zkStateReader.getClusterState()); + rc.backupCollectionState); // note: when createCollection() returns, the collection exists (no race) // Restore collection properties @@ -248,8 +251,6 @@ public void process(NamedList results, RestoreContext rc) throws Excepti DocCollection restoreCollection = rc.zkStateReader.getClusterState().getCollection(rc.restoreCollectionName); markAllShardsAsConstruction(restoreCollection); - // TODO how do we leverage the RULE / SNITCH logic in createCollection? - ClusterState clusterState = rc.zkStateReader.getClusterState(); List sliceNames = new ArrayList<>(); restoreCollection.getSlices().forEach(x -> sliceNames.add(x.getName())); @@ -258,11 +259,15 @@ public void process(NamedList results, RestoreContext rc) throws Excepti getReplicaPositions(rc.restoreCollectionName, rc.nodeList, sliceNames); createSingleReplicaPerShard( - results, restoreCollection, rc.asyncId, clusterState, replicaPositions); + results, + restoreCollection, + rc.adminCmdContext.withClusterState(rc.zkStateReader.getClusterState()), + replicaPositions); Object failures = results.get("failure"); if (failures != null && ((SimpleOrderedMap) failures).size() > 0) { log.error("Restore failed to create initial replicas."); - CollectionHandlingUtils.cleanupCollection(rc.restoreCollectionName, new NamedList<>(), ccc); + CollectionHandlingUtils.cleanupCollection( + rc.adminCmdContext, rc.restoreCollectionName, new NamedList<>(), ccc); return; } @@ -272,14 +277,17 @@ public void process(NamedList results, RestoreContext rc) throws Excepti requestReplicasToRestore( results, restoreCollection, - clusterState, + rc.adminCmdContext.withClusterState(rc.zkStateReader.getClusterState()), rc.backupProperties, rc.backupPath, rc.repo, - rc.shardHandler, - rc.asyncId); + rc.shardHandler); markAllShardsAsActive(restoreCollection); - addReplicasToShards(results, clusterState, restoreCollection, replicaPositions, rc.asyncId); + addReplicasToShards( + results, + restoreCollection, + replicaPositions, + rc.adminCmdContext.withClusterState(rc.zkStateReader.getClusterState())); restoringAlias(rc.backupProperties); log.info("Completed restoring collection={} backupName={}", restoreCollection, rc.backupName); @@ -310,10 +318,10 @@ private void uploadConfig( } private void createCoreLessCollection( + AdminCmdContext adminCmdContext, String restoreCollectionName, String restoreConfigName, - DocCollection backupCollectionState, - ClusterState clusterState) + DocCollection backupCollectionState) throws Exception { Map propMap = new HashMap<>(); propMap.put(Overseer.QUEUE_OPERATION, CREATE.toString()); @@ -368,7 +376,11 @@ private void createCoreLessCollection( propMap.put(CollectionHandlingUtils.SHARDS_PROP, newSlices); } - new CreateCollectionCmd(ccc).call(clusterState, new ZkNodeProps(propMap), new NamedList<>()); + new CreateCollectionCmd(ccc) + .call( + adminCmdContext.subRequestContext(CREATE, null), + new ZkNodeProps(propMap), + new NamedList<>()); // note: when createCollection() returns, the collection exists (no race) } @@ -411,8 +423,7 @@ private List getReplicaPositions( private void createSingleReplicaPerShard( NamedList results, DocCollection restoreCollection, - String asyncId, - ClusterState clusterState, + AdminCmdContext adminCmdContext, List replicaPositions) throws Exception { CountDownLatch countDownLatch = new CountDownLatch(restoreCollection.getSlices().size()); @@ -438,15 +449,11 @@ private void createSingleReplicaPerShard( } } - // add async param - if (asyncId != null) { - propMap.put(ASYNC, asyncId); - } CollectionHandlingUtils.addPropertyParams(message, propMap); final NamedList addReplicaResult = new NamedList<>(); new AddReplicaCmd(ccc) .addReplica( - clusterState, + adminCmdContext.subRequestContext(ADDREPLICA), new ZkNodeProps(propMap), addReplicaResult, () -> { @@ -508,10 +515,9 @@ private void markAllShardsAsActive(DocCollection restoreCollection) private void addReplicasToShards( NamedList results, - ClusterState clusterState, DocCollection restoreCollection, List replicaPositions, - String asyncId) + AdminCmdContext adminCmdContext) throws Exception { int totalReplicasPerShard = numReplicas.total(); if (totalReplicasPerShard > 1) { @@ -558,14 +564,14 @@ private void addReplicasToShards( } } - // add async param - if (asyncId != null) { - propMap.put(ASYNC, asyncId); - } CollectionHandlingUtils.addPropertyParams(message, propMap); new AddReplicaCmd(ccc) - .addReplica(clusterState, new ZkNodeProps(propMap), results, null); + .addReplica( + adminCmdContext.subRequestContext(ADDREPLICA), + new ZkNodeProps(propMap), + results, + null); } } } @@ -616,23 +622,26 @@ public void process(RestoreContext rc, NamedList results) throws Excepti ClusterState clusterState = rc.zkStateReader.getClusterState(); DocCollection restoreCollection = clusterState.getCollection(rc.restoreCollectionName); - enableReadOnly(clusterState, restoreCollection); + enableReadOnly( + rc.adminCmdContext.withClusterState(rc.zkStateReader.getClusterState()), + restoreCollection); try { requestReplicasToRestore( results, restoreCollection, - clusterState, + rc.adminCmdContext.withClusterState(rc.zkStateReader.getClusterState()), rc.backupProperties, rc.backupPath, rc.repo, - rc.shardHandler, - rc.asyncId); + rc.shardHandler); } finally { - disableReadOnly(clusterState, restoreCollection); + disableReadOnly( + rc.adminCmdContext.withClusterState(rc.zkStateReader.getClusterState()), + restoreCollection); } } - private void disableReadOnly(ClusterState clusterState, DocCollection restoreCollection) + private void disableReadOnly(AdminCmdContext adminCmdContext, DocCollection restoreCollection) throws Exception { ZkNodeProps params = new ZkNodeProps( @@ -640,10 +649,12 @@ private void disableReadOnly(ClusterState clusterState, DocCollection restoreCol CollectionParams.CollectionAction.MODIFYCOLLECTION.toString(), ZkStateReader.COLLECTION_PROP, restoreCollection.getName(), ZkStateReader.READ_ONLY, null); - new CollApiCmds.ModifyCollectionCmd(ccc).call(clusterState, params, new NamedList<>()); + new CollApiCmds.ModifyCollectionCmd(ccc) + .call( + adminCmdContext.subRequestContext(MODIFYCOLLECTION, null), params, new NamedList<>()); } - private void enableReadOnly(ClusterState clusterState, DocCollection restoreCollection) + private void enableReadOnly(AdminCmdContext adminCmdContext, DocCollection restoreCollection) throws Exception { ZkNodeProps params = new ZkNodeProps( @@ -651,7 +662,9 @@ private void enableReadOnly(ClusterState clusterState, DocCollection restoreColl CollectionParams.CollectionAction.MODIFYCOLLECTION.toString(), ZkStateReader.COLLECTION_PROP, restoreCollection.getName(), ZkStateReader.READ_ONLY, "true"); - new CollApiCmds.ModifyCollectionCmd(ccc).call(clusterState, params, new NamedList<>()); + new CollApiCmds.ModifyCollectionCmd(ccc) + .call( + adminCmdContext.subRequestContext(MODIFYCOLLECTION, null), params, new NamedList<>()); } } } diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/SetAliasPropCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/SetAliasPropCmd.java index 671b71dddb6..f43e9ceeef2 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/SetAliasPropCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/SetAliasPropCmd.java @@ -26,7 +26,6 @@ import java.util.Locale; import java.util.Map; import org.apache.solr.common.SolrException; -import org.apache.solr.common.cloud.ClusterState; import org.apache.solr.common.cloud.ZkNodeProps; import org.apache.solr.common.cloud.ZkStateReader; import org.apache.solr.common.util.NamedList; @@ -46,7 +45,7 @@ public SetAliasPropCmd(CollectionCommandContext ccc) { } @Override - public void call(ClusterState state, ZkNodeProps message, NamedList results) + public void call(AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList results) throws Exception { String aliasName = message.getStr(NAME); diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/SplitShardCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/SplitShardCmd.java index bf7f698d346..c2f0fd3d500 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/SplitShardCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/SplitShardCmd.java @@ -24,7 +24,6 @@ import static org.apache.solr.common.params.CollectionParams.CollectionAction.ADDREPLICA; import static org.apache.solr.common.params.CollectionParams.CollectionAction.CREATESHARD; import static org.apache.solr.common.params.CollectionParams.CollectionAction.DELETESHARD; -import static org.apache.solr.common.params.CommonAdminParams.ASYNC; import static org.apache.solr.common.params.CommonAdminParams.NUM_SUB_SHARDS; import java.lang.invoke.MethodHandles; @@ -101,9 +100,9 @@ public SplitShardCmd(CollectionCommandContext ccc) { } @Override - public void call(ClusterState state, ZkNodeProps message, NamedList results) + public void call(AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList results) throws Exception { - split(state, message, results); + split(adminCmdContext, message, results); } /** @@ -131,10 +130,9 @@ public void call(ClusterState state, ZkNodeProps message, NamedList resu *

There is a shard split doc (dev-docs/shard-split/shard-split.adoc) on how shard split works; * illustrated with diagrams. */ - public boolean split(ClusterState clusterState, ZkNodeProps message, NamedList results) + public boolean split( + AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList results) throws Exception { - final String asyncId = message.getStr(ASYNC); - boolean waitForFinalState = message.getBool(CommonAdminParams.WAIT_FOR_FINAL_STATE, false); String methodStr = message.getStr( @@ -165,6 +163,7 @@ public boolean split(ClusterState clusterState, ZkNodeProps message, NamedList offlineSlices = new HashSet<>(); RTimerTree timings = new RTimerTree(); + ClusterState clusterState = zkStateReader.getClusterState(); String splitKey = message.getStr("split.key"); DocCollection collection = clusterState.getCollection(collectionName); @@ -275,7 +274,7 @@ public boolean split(ClusterState clusterState, ZkNodeProps message, NamedList getRangesResults = new SimpleOrderedMap<>(); @@ -338,7 +337,13 @@ public boolean split(ClusterState clusterState, ZkNodeProps message, NamedList()); + new DeleteShardCmd(ccc) + .call( + adminCmdContext + .subRequestContext(DELETESHARD, null) + .withClusterState(clusterState), + m, + new NamedList<>()); } catch (Exception e) { throw new SolrException( SolrException.ErrorCode.SERVER_ERROR, @@ -416,17 +421,17 @@ public boolean split(ClusterState clusterState, ZkNodeProps message, NamedList replica : replicas) { - new AddReplicaCmd(ccc).addReplica(clusterState, new ZkNodeProps(replica), results, null); + new AddReplicaCmd(ccc) + .addReplica( + adminCmdContext.subRequestContext(ADDREPLICA).withClusterState(clusterState), + new ZkNodeProps(replica), + results, + null); } assert TestInjection.injectSplitFailureAfterReplicaCreation(); { final ShardRequestTracker syncRequestTracker = - CollectionHandlingUtils.syncRequestTracker(ccc); + CollectionHandlingUtils.syncRequestTracker(adminCmdContext, ccc); String msgOnError = "SPLITSHARD failed to create subshard replicas"; syncRequestTracker.processResponses(results, shardHandler, true, msgOnError); handleFailureOnAsyncRequest(results, msgOnError); @@ -822,7 +828,12 @@ public boolean split(ClusterState clusterState, ZkNodeProps message, NamedList()); + new DeleteShardCmd(ccc) + .call( + adminCmdContext.subRequestContext(DELETESHARD, null).withClusterState(clusterState), + m, + new NamedList<>()); } catch (Exception e) { log.warn( "Cleanup failed after failed split of {}/{} : (deleting existing sub shard{})", diff --git a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java index c2cd0148938..20213193389 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java @@ -34,6 +34,7 @@ import static org.apache.solr.common.cloud.ZkStateReader.REPLICATION_FACTOR; import static org.apache.solr.common.cloud.ZkStateReader.REPLICA_PROP; import static org.apache.solr.common.cloud.ZkStateReader.SHARD_ID_PROP; +import static org.apache.solr.common.params.CollectionAdminParams.CALLING_LOCK_IDS_HEADER; import static org.apache.solr.common.params.CollectionAdminParams.COLLECTION; import static org.apache.solr.common.params.CollectionAdminParams.CREATE_NODE_SET_PARAM; import static org.apache.solr.common.params.CollectionAdminParams.FOLLOW_ALIASES; @@ -100,6 +101,7 @@ import static org.apache.solr.common.params.CommonParams.VALUE_LONG; import static org.apache.solr.common.params.CoreAdminParams.BACKUP_LOCATION; import static org.apache.solr.common.params.CoreAdminParams.BACKUP_REPOSITORY; +import static org.apache.solr.common.params.CoreAdminParams.SHARD_BACKUP_ID; import static org.apache.solr.common.util.StrUtils.formatString; import java.lang.invoke.MethodHandles; @@ -107,6 +109,7 @@ import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.HashMap; import java.util.Iterator; import java.util.LinkedHashMap; import java.util.List; @@ -115,6 +118,7 @@ import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; +import org.apache.commons.lang3.StringUtils; import org.apache.solr.api.AnnotatedApi; import org.apache.solr.api.Api; import org.apache.solr.api.JerseyResource; @@ -138,6 +142,7 @@ import org.apache.solr.cloud.OverseerTaskQueue.QueueEvent; import org.apache.solr.cloud.ZkController; import org.apache.solr.cloud.ZkController.NotInClusterStateException; +import org.apache.solr.cloud.api.collections.AdminCmdContext; import org.apache.solr.cloud.api.collections.CollectionHandlingUtils; import org.apache.solr.cloud.api.collections.DistributedCollectionConfigSetCommandRunner; import org.apache.solr.cloud.api.collections.ReindexCollectionCmd; @@ -315,17 +320,14 @@ void invokeAction( return; } - String asyncId = req.getParams().get(ASYNC); - if (asyncId != null) { - props.put(ASYNC, asyncId); - } - - props.put(QUEUE_OPERATION, operation.action.toLower()); + AdminCmdContext adminCmdContext = + new AdminCmdContext(operation.action, req.getParams().get(ASYNC)); + adminCmdContext.setCallingLockIds((String) req.getContext().get(CALLING_LOCK_IDS_HEADER)); ZkNodeProps zkProps = new ZkNodeProps(props); final SolrResponse overseerResponse; - overseerResponse = submitCollectionApiCommand(zkProps, operation.action, operation.timeOut); + overseerResponse = submitCollectionApiCommand(adminCmdContext, zkProps, operation.timeOut); rsp.getValues().addAll(overseerResponse.getResponse()); Exception exp = overseerResponse.getException(); @@ -338,13 +340,13 @@ void invokeAction( public static long DEFAULT_COLLECTION_OP_TIMEOUT = 180 * 1000; - public SolrResponse submitCollectionApiCommand(ZkNodeProps m, CollectionAction action) + public SolrResponse submitCollectionApiCommand(AdminCmdContext adminCmdContext, ZkNodeProps m) throws KeeperException, InterruptedException { - return submitCollectionApiCommand(m, action, DEFAULT_COLLECTION_OP_TIMEOUT); + return submitCollectionApiCommand(adminCmdContext, m, DEFAULT_COLLECTION_OP_TIMEOUT); } public static SolrResponse submitCollectionApiCommand( - ZkController zkController, ZkNodeProps m, CollectionAction action, long timeout) + ZkController zkController, AdminCmdContext adminCmdContext, ZkNodeProps m, long timeout) throws KeeperException, InterruptedException { // Collection API messages are either sent to Overseer and processed there, or processed // locally. Distributing Collection API implies we're also distributing Cluster State Updates. @@ -359,14 +361,20 @@ public static SolrResponse submitCollectionApiCommand( Optional distribCommandRunner = zkController.getDistributedCommandRunner(); if (distribCommandRunner.isPresent()) { - return distribCommandRunner.get().runCollectionCommand(m, action, timeout); + return distribCommandRunner.get().runCollectionCommand(adminCmdContext, m, timeout); } else { // Sending the Collection API message to Overseer via a Zookeeper queue - String operation = m.getStr(QUEUE_OPERATION); - if (operation == null) { - throw new SolrException(ErrorCode.BAD_REQUEST, "missing key " + QUEUE_OPERATION); + String operation = adminCmdContext.getAction().lowerName; + HashMap additionalProps = new HashMap<>(); + additionalProps.put(QUEUE_OPERATION, operation); + if (adminCmdContext.getAsyncId() != null && !adminCmdContext.getAsyncId().isBlank()) { + additionalProps.put(ASYNC, adminCmdContext.getAsyncId()); + } + if (StringUtils.isNotBlank(adminCmdContext.getCallingLockIds())) { + additionalProps.put(CALLING_LOCK_IDS_HEADER, adminCmdContext.getCallingLockIds()); } - if (m.get(ASYNC) != null) { - String asyncId = m.getStr(ASYNC); + m = m.plus(additionalProps); + if (adminCmdContext.getAsyncId() != null) { + String asyncId = adminCmdContext.getAsyncId(); NamedList r = new NamedList<>(); if (zkController.claimAsyncId(asyncId)) { @@ -389,7 +397,7 @@ public static SolrResponse submitCollectionApiCommand( throw new SolrException( BAD_REQUEST, "Task with the same requestid already exists. (" + asyncId + ")"); } - r.add(CoreAdminParams.REQUESTID, m.get(ASYNC)); + r.add(CoreAdminParams.REQUESTID, asyncId); return new OverseerSolrResponse(r); } @@ -424,9 +432,9 @@ public static SolrResponse submitCollectionApiCommand( } public SolrResponse submitCollectionApiCommand( - ZkNodeProps m, CollectionAction action, long timeout) + AdminCmdContext adminCmdContext, ZkNodeProps m, long timeout) throws KeeperException, InterruptedException { - return submitCollectionApiCommand(coreContainer.getZkController(), m, action, timeout); + return submitCollectionApiCommand(coreContainer.getZkController(), adminCmdContext, m, timeout); } private boolean overseerCollectionQueueContains(String asyncId) @@ -1061,6 +1069,8 @@ public Map execute( reqBody.async = req.getParams().get(ASYNC); reqBody.repository = req.getParams().get(BACKUP_REPOSITORY); reqBody.location = req.getParams().get(BACKUP_LOCATION); + reqBody.name = req.getParams().get(NAME); + reqBody.shardBackupId = req.getParams().get(SHARD_BACKUP_ID); final InstallShardData installApi = new InstallShardData(h.coreContainer, req, rsp); final SolrJerseyResponse installResponse = diff --git a/solr/core/src/java/org/apache/solr/handler/admin/RebalanceLeaders.java b/solr/core/src/java/org/apache/solr/handler/admin/RebalanceLeaders.java index 2bebe2ba3ac..26afe771d5f 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/RebalanceLeaders.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/RebalanceLeaders.java @@ -16,7 +16,6 @@ */ package org.apache.solr.handler.admin; -import static org.apache.solr.cloud.Overseer.QUEUE_OPERATION; import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP; import static org.apache.solr.common.cloud.ZkStateReader.CORE_NAME_PROP; import static org.apache.solr.common.cloud.ZkStateReader.CORE_NODE_NAME_PROP; @@ -25,7 +24,6 @@ import static org.apache.solr.common.cloud.ZkStateReader.MAX_WAIT_SECONDS_PROP; import static org.apache.solr.common.cloud.ZkStateReader.REJOIN_AT_HEAD_PROP; import static org.apache.solr.common.params.CollectionParams.CollectionAction.REBALANCELEADERS; -import static org.apache.solr.common.params.CommonAdminParams.ASYNC; import java.lang.invoke.MethodHandles; import java.util.HashMap; @@ -38,6 +36,7 @@ import java.util.concurrent.TimeUnit; import org.apache.solr.cloud.LeaderElector; import org.apache.solr.cloud.OverseerTaskProcessor; +import org.apache.solr.cloud.api.collections.AdminCmdContext; import org.apache.solr.cloud.overseer.SliceMutator; import org.apache.solr.common.SolrException; import org.apache.solr.common.cloud.ClusterState; @@ -46,7 +45,6 @@ import org.apache.solr.common.cloud.Slice; import org.apache.solr.common.cloud.ZkNodeProps; import org.apache.solr.common.cloud.ZkStateReader; -import org.apache.solr.common.params.CollectionParams; import org.apache.solr.common.util.SimpleOrderedMap; import org.apache.solr.common.util.StrUtils; import org.apache.solr.core.CoreContainer; @@ -444,10 +442,8 @@ private void rejoinElectionQueue( Slice slice, String electionNode, String core, boolean rejoinAtHead) throws KeeperException, InterruptedException { Replica replica = slice.getReplica(LeaderElector.getNodeName(electionNode)); - final CollectionParams.CollectionAction rebalanceleaders = REBALANCELEADERS; Map propMap = new HashMap<>(); propMap.put(COLLECTION_PROP, collectionName); - propMap.put(QUEUE_OPERATION, rebalanceleaders.toLower()); propMap.put(CORE_NAME_PROP, core); propMap.put(CORE_NODE_NAME_PROP, replica.getName()); propMap.put(ZkStateReader.NODE_NAME_PROP, replica.getNodeName()); @@ -459,12 +455,12 @@ private void rejoinElectionQueue( .getBaseUrlForNodeName(replica.getNodeName())); propMap.put( REJOIN_AT_HEAD_PROP, Boolean.toString(rejoinAtHead)); // Get ourselves to be first in line. - String asyncId = rebalanceleaders.toLower() + "_" + core + "_" + Math.abs(System.nanoTime()); - propMap.put(ASYNC, asyncId); + String asyncId = REBALANCELEADERS.toLower() + "_" + core + "_" + Math.abs(System.nanoTime()); asyncRequests.add(asyncId); collectionsHandler.submitCollectionApiCommand( - new ZkNodeProps(propMap), rebalanceleaders); // ignore response; we construct our own + new AdminCmdContext(REBALANCELEADERS, asyncId), + new ZkNodeProps(propMap)); // ignore response; we construct our own } // maxWaitSecs - How long are we going to wait? Defaults to 30 seconds. diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/AddReplicaProperty.java b/solr/core/src/java/org/apache/solr/handler/admin/api/AddReplicaProperty.java index 81b2dbce137..baf66c3e4d3 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/AddReplicaProperty.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/AddReplicaProperty.java @@ -16,7 +16,6 @@ */ package org.apache.solr.handler.admin.api; -import static org.apache.solr.cloud.Overseer.QUEUE_OPERATION; import static org.apache.solr.cloud.api.collections.CollectionHandlingUtils.SHARD_UNIQUE; import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP; import static org.apache.solr.common.cloud.ZkStateReader.PROPERTY_PROP; @@ -24,7 +23,6 @@ import static org.apache.solr.common.cloud.ZkStateReader.REPLICA_PROP; import static org.apache.solr.common.cloud.ZkStateReader.SHARD_ID_PROP; import static org.apache.solr.common.params.CollectionAdminParams.PROPERTY_PREFIX; -import static org.apache.solr.handler.admin.CollectionsHandler.DEFAULT_COLLECTION_OP_TIMEOUT; import static org.apache.solr.security.PermissionNameProvider.Name.COLL_EDIT_PERM; import jakarta.inject.Inject; @@ -33,14 +31,12 @@ import java.util.Map; import org.apache.solr.client.api.endpoint.AddReplicaPropertyApi; import org.apache.solr.client.api.model.AddReplicaPropertyRequestBody; -import org.apache.solr.client.api.model.SolrJerseyResponse; -import org.apache.solr.client.solrj.SolrResponse; +import org.apache.solr.client.api.model.SubResponseAccumulatingJerseyResponse; import org.apache.solr.cloud.overseer.SliceMutator; import org.apache.solr.common.SolrException; import org.apache.solr.common.cloud.ZkNodeProps; import org.apache.solr.common.params.CollectionParams; import org.apache.solr.core.CoreContainer; -import org.apache.solr.handler.admin.CollectionsHandler; import org.apache.solr.jersey.PermissionName; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.response.SolrQueryResponse; @@ -62,7 +58,7 @@ public AddReplicaProperty( @Override @PermissionName(COLL_EDIT_PERM) - public SolrJerseyResponse addReplicaProperty( + public SubResponseAccumulatingJerseyResponse addReplicaProperty( String collName, String shardName, String replicaName, @@ -72,21 +68,14 @@ public SolrJerseyResponse addReplicaProperty( if (requestBody == null) { throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Missing required request body"); } - final SolrJerseyResponse response = instantiateJerseyResponse(SolrJerseyResponse.class); - final CoreContainer coreContainer = fetchAndValidateZooKeeperAwareCoreContainer(); + final var response = instantiateJerseyResponse(SubResponseAccumulatingJerseyResponse.class); + fetchAndValidateZooKeeperAwareCoreContainer(); recordCollectionForLogAndTracing(collName, solrQueryRequest); final ZkNodeProps remoteMessage = createRemoteMessage(collName, shardName, replicaName, propertyName, requestBody); - final SolrResponse remoteResponse = - CollectionsHandler.submitCollectionApiCommand( - coreContainer.getZkController(), - remoteMessage, - CollectionParams.CollectionAction.ADDREPLICAPROP, - DEFAULT_COLLECTION_OP_TIMEOUT); - if (remoteResponse.getException() != null) { - throw remoteResponse.getException(); - } + submitRemoteMessageAndHandleResponse( + response, CollectionParams.CollectionAction.ADDREPLICAPROP, remoteMessage, null); disableResponseCaching(); return response; @@ -104,7 +93,6 @@ public ZkNodeProps createRemoteMessage( remoteMessage.put(SHARD_ID_PROP, shardName); remoteMessage.put(REPLICA_PROP, replicaName); remoteMessage.put(PROPERTY_VALUE_PROP, requestBody.value); - remoteMessage.put(QUEUE_OPERATION, CollectionParams.CollectionAction.ADDREPLICAPROP.toLower()); if (requestBody.shardUnique != null) { remoteMessage.put(SHARD_UNIQUE, requestBody.shardUnique); } diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/AdminAPIBase.java b/solr/core/src/java/org/apache/solr/handler/admin/api/AdminAPIBase.java index c4e001a3ad3..94f0e9c36ba 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/AdminAPIBase.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/AdminAPIBase.java @@ -21,8 +21,10 @@ import java.util.Map; import org.apache.solr.api.JerseyResource; +import org.apache.solr.client.api.model.SolrJerseyResponse; import org.apache.solr.client.api.model.SubResponseAccumulatingJerseyResponse; import org.apache.solr.client.solrj.SolrResponse; +import org.apache.solr.cloud.api.collections.AdminCmdContext; import org.apache.solr.common.SolrException; import org.apache.solr.common.cloud.ClusterState; import org.apache.solr.common.cloud.ZkNodeProps; @@ -70,11 +72,7 @@ protected String resolveAndValidateAliasIfEnabled( } private String resolveAlias(String aliasName) { - return coreContainer - .getZkController() - .getZkStateReader() - .getAliases() - .resolveSimpleAlias(aliasName); + return coreContainer.getAliases().resolveSimpleAlias(aliasName); } public static void validateZooKeeperAwareCoreContainer(CoreContainer coreContainer) { @@ -92,13 +90,7 @@ public static void validateZooKeeperAwareCoreContainer(CoreContainer coreContain protected String resolveCollectionName(String collName, boolean followAliases) { final String collectionName = - followAliases - ? coreContainer - .getZkController() - .getZkStateReader() - .getAliases() - .resolveSimpleAlias(collName) - : collName; + followAliases ? coreContainer.getAliases().resolveSimpleAlias(collName) : collName; final ClusterState clusterState = coreContainer.getZkController().getClusterState(); if (!clusterState.hasCollection(collectionName)) { @@ -130,12 +122,9 @@ protected SolrResponse submitRemoteMessageAndHandleResponse( ZkNodeProps remoteMessage, String asyncId) throws Exception { - final SolrResponse remoteResponse = - CollectionsHandler.submitCollectionApiCommand( - coreContainer.getZkController(), remoteMessage, action, DEFAULT_COLLECTION_OP_TIMEOUT); - if (remoteResponse.getException() != null) { - throw remoteResponse.getException(); - } + var remoteResponse = + submitRemoteMessageAndHandleResponse( + response, new AdminCmdContext(action, asyncId), remoteMessage); if (asyncId != null) { response.requestId = asyncId; @@ -144,7 +133,32 @@ protected SolrResponse submitRemoteMessageAndHandleResponse( // Values fetched from remoteResponse may be null response.successfulSubResponsesByNodeName = remoteResponse.getResponse().get("success"); response.failedSubResponsesByNodeName = remoteResponse.getResponse().get("failure"); + response.warning = (String) remoteResponse.getResponse().get("warning"); + + return remoteResponse; + } + + protected SolrResponse submitRemoteMessageAndHandleResponse( + SolrJerseyResponse response, + CollectionParams.CollectionAction action, + ZkNodeProps remoteMessage) + throws Exception { + return submitRemoteMessageAndHandleResponse( + response, new AdminCmdContext(action, null), remoteMessage); + } + protected SolrResponse submitRemoteMessageAndHandleResponse( + SolrJerseyResponse response, AdminCmdContext adminCmdContext, ZkNodeProps remoteMessage) + throws Exception { + final SolrResponse remoteResponse = + CollectionsHandler.submitCollectionApiCommand( + coreContainer.getZkController(), + adminCmdContext, + remoteMessage, + DEFAULT_COLLECTION_OP_TIMEOUT); + if (remoteResponse.getException() != null) { + throw remoteResponse.getException(); + } return remoteResponse; } diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/AliasProperty.java b/solr/core/src/java/org/apache/solr/handler/admin/api/AliasProperty.java index 56af9177a96..a400f0edb31 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/AliasProperty.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/AliasProperty.java @@ -16,30 +16,26 @@ */ package org.apache.solr.handler.admin.api; -import static org.apache.solr.cloud.Overseer.QUEUE_OPERATION; -import static org.apache.solr.common.params.CommonAdminParams.ASYNC; import static org.apache.solr.common.params.CommonParams.NAME; -import static org.apache.solr.handler.admin.CollectionsHandler.DEFAULT_COLLECTION_OP_TIMEOUT; import static org.apache.solr.security.PermissionNameProvider.Name.COLL_EDIT_PERM; import static org.apache.solr.security.PermissionNameProvider.Name.COLL_READ_PERM; import jakarta.inject.Inject; +import java.util.Collections; import java.util.HashMap; import java.util.Map; import org.apache.solr.client.api.endpoint.AliasPropertyApis; import org.apache.solr.client.api.model.GetAliasPropertyResponse; import org.apache.solr.client.api.model.GetAllAliasPropertiesResponse; -import org.apache.solr.client.api.model.SolrJerseyResponse; +import org.apache.solr.client.api.model.SubResponseAccumulatingJerseyResponse; import org.apache.solr.client.api.model.UpdateAliasPropertiesRequestBody; import org.apache.solr.client.api.model.UpdateAliasPropertyRequestBody; -import org.apache.solr.client.solrj.SolrResponse; import org.apache.solr.common.SolrException; import org.apache.solr.common.cloud.Aliases; import org.apache.solr.common.cloud.ZkNodeProps; import org.apache.solr.common.cloud.ZkStateReader; import org.apache.solr.common.params.CollectionParams; import org.apache.solr.core.CoreContainer; -import org.apache.solr.handler.admin.CollectionsHandler; import org.apache.solr.jersey.PermissionName; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.response.SolrQueryResponse; @@ -103,7 +99,7 @@ private Aliases readAliasesFromZk() throws Exception { @Override @PermissionName(COLL_EDIT_PERM) - public SolrJerseyResponse updateAliasProperties( + public SubResponseAccumulatingJerseyResponse updateAliasProperties( String aliasName, UpdateAliasPropertiesRequestBody requestBody) throws Exception { if (requestBody == null) { @@ -112,14 +108,14 @@ public SolrJerseyResponse updateAliasProperties( recordCollectionForLogAndTracing(null, solrQueryRequest); - SolrJerseyResponse response = instantiateJerseyResponse(SolrJerseyResponse.class); - modifyAliasProperties(aliasName, requestBody.properties, requestBody.async); + var response = instantiateJerseyResponse(SubResponseAccumulatingJerseyResponse.class); + modifyAliasProperties(response, aliasName, requestBody.properties, requestBody.async); return response; } @Override @PermissionName(COLL_EDIT_PERM) - public SolrJerseyResponse createOrUpdateAliasProperty( + public SubResponseAccumulatingJerseyResponse createOrUpdateAliasProperty( String aliasName, String propName, UpdateAliasPropertyRequestBody requestBody) throws Exception { if (requestBody == null) { @@ -128,63 +124,60 @@ public SolrJerseyResponse createOrUpdateAliasProperty( recordCollectionForLogAndTracing(null, solrQueryRequest); - SolrJerseyResponse response = instantiateJerseyResponse(SolrJerseyResponse.class); - modifyAliasProperty(aliasName, propName, requestBody.value); + var response = instantiateJerseyResponse(SubResponseAccumulatingJerseyResponse.class); + modifyAliasProperty(response, aliasName, propName, requestBody.value); return response; } @Override @PermissionName(COLL_EDIT_PERM) - public SolrJerseyResponse deleteAliasProperty(String aliasName, String propName) - throws Exception { + public SubResponseAccumulatingJerseyResponse deleteAliasProperty( + String aliasName, String propName) throws Exception { recordCollectionForLogAndTracing(null, solrQueryRequest); - SolrJerseyResponse response = instantiateJerseyResponse(SolrJerseyResponse.class); - modifyAliasProperty(aliasName, propName, null); + var response = instantiateJerseyResponse(SubResponseAccumulatingJerseyResponse.class); + modifyAliasProperty(response, aliasName, propName, null); return response; } - private void modifyAliasProperty(String alias, String proertyName, Object value) + private void modifyAliasProperty( + SubResponseAccumulatingJerseyResponse response, + String alias, + String proertyName, + Object value) throws Exception { Map props = new HashMap<>(); // value can be null props.put(proertyName, value); - modifyAliasProperties(alias, props, null); + modifyAliasProperties(response, alias, props, null); } + private static final String PROPERTIES = "property"; + /** * @param alias alias */ - private void modifyAliasProperties(String alias, Map properties, String async) + private void modifyAliasProperties( + SubResponseAccumulatingJerseyResponse response, + String alias, + Map properties, + String async) throws Exception { // Note: success/no-op in the event of no properties supplied is intentional. Keeps code // simple and one less case for api-callers to check for. - final CoreContainer coreContainer = fetchAndValidateZooKeeperAwareCoreContainer(); - final ZkNodeProps remoteMessage = createRemoteMessage(alias, properties, async); - final SolrResponse remoteResponse = - CollectionsHandler.submitCollectionApiCommand( - coreContainer.getZkController(), - remoteMessage, - CollectionParams.CollectionAction.ALIASPROP, - DEFAULT_COLLECTION_OP_TIMEOUT); - if (remoteResponse.getException() != null) { - throw remoteResponse.getException(); + if (properties == null) { + properties = Collections.emptyMap(); } + fetchAndValidateZooKeeperAwareCoreContainer(); + submitRemoteMessageAndHandleResponse( + response, + CollectionParams.CollectionAction.ALIASPROP, + new ZkNodeProps( + Map.of( + NAME, alias, + PROPERTIES, properties)), + async); disableResponseCaching(); } - - private static final String PROPERTIES = "property"; - - public ZkNodeProps createRemoteMessage( - String alias, Map properties, String async) { - final Map remoteMessage = new HashMap<>(); - remoteMessage.put(QUEUE_OPERATION, CollectionParams.CollectionAction.ALIASPROP.toLower()); - remoteMessage.put(NAME, alias); - remoteMessage.put(PROPERTIES, properties); - if (async != null) { - remoteMessage.put(ASYNC, async); - } - return new ZkNodeProps(remoteMessage); - } } diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/BalanceReplicas.java b/solr/core/src/java/org/apache/solr/handler/admin/api/BalanceReplicas.java index b582e39464b..d5ff9aa9ad4 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/BalanceReplicas.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/BalanceReplicas.java @@ -16,11 +16,8 @@ */ package org.apache.solr.handler.admin.api; -import static org.apache.solr.cloud.Overseer.QUEUE_OPERATION; import static org.apache.solr.common.params.CollectionParams.NODES; -import static org.apache.solr.common.params.CommonAdminParams.ASYNC; import static org.apache.solr.common.params.CommonAdminParams.WAIT_FOR_FINAL_STATE; -import static org.apache.solr.handler.admin.CollectionsHandler.DEFAULT_COLLECTION_OP_TIMEOUT; import static org.apache.solr.security.PermissionNameProvider.Name.COLL_EDIT_PERM; import jakarta.inject.Inject; @@ -28,12 +25,10 @@ import java.util.Map; import org.apache.solr.client.api.endpoint.BalanceReplicasApi; import org.apache.solr.client.api.model.BalanceReplicasRequestBody; -import org.apache.solr.client.api.model.SolrJerseyResponse; -import org.apache.solr.client.solrj.SolrResponse; +import org.apache.solr.client.api.model.SubResponseAccumulatingJerseyResponse; import org.apache.solr.common.cloud.ZkNodeProps; import org.apache.solr.common.params.CollectionParams.CollectionAction; import org.apache.solr.core.CoreContainer; -import org.apache.solr.handler.admin.CollectionsHandler; import org.apache.solr.jersey.PermissionName; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.response.SolrQueryResponse; @@ -51,21 +46,14 @@ public BalanceReplicas( @Override @PermissionName(COLL_EDIT_PERM) - public SolrJerseyResponse balanceReplicas(BalanceReplicasRequestBody requestBody) - throws Exception { - final SolrJerseyResponse response = instantiateJerseyResponse(SolrJerseyResponse.class); - final CoreContainer coreContainer = fetchAndValidateZooKeeperAwareCoreContainer(); - // TODO Record node for log and tracing + public SubResponseAccumulatingJerseyResponse balanceReplicas( + BalanceReplicasRequestBody requestBody) throws Exception { + final var response = instantiateJerseyResponse(SubResponseAccumulatingJerseyResponse.class); + recordCollectionForLogAndTracing(null, solrQueryRequest); + fetchAndValidateZooKeeperAwareCoreContainer(); final ZkNodeProps remoteMessage = createRemoteMessage(requestBody); - final SolrResponse remoteResponse = - CollectionsHandler.submitCollectionApiCommand( - coreContainer.getZkController(), - remoteMessage, - CollectionAction.BALANCE_REPLICAS, - DEFAULT_COLLECTION_OP_TIMEOUT); - if (remoteResponse.getException() != null) { - throw remoteResponse.getException(); - } + submitRemoteMessageAndHandleResponse( + response, CollectionAction.BALANCE_REPLICAS, remoteMessage, requestBody.async); disableResponseCaching(); return response; @@ -76,9 +64,7 @@ public ZkNodeProps createRemoteMessage(BalanceReplicasRequestBody requestBody) { if (requestBody != null) { insertIfNotNull(remoteMessage, NODES, requestBody.nodes); insertIfNotNull(remoteMessage, WAIT_FOR_FINAL_STATE, requestBody.waitForFinalState); - insertIfNotNull(remoteMessage, ASYNC, requestBody.async); } - remoteMessage.put(QUEUE_OPERATION, CollectionAction.BALANCE_REPLICAS.toLower()); return new ZkNodeProps(remoteMessage); } diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/BalanceShardUnique.java b/solr/core/src/java/org/apache/solr/handler/admin/api/BalanceShardUnique.java index ae54b0ab98d..3b655bbb46f 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/BalanceShardUnique.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/BalanceShardUnique.java @@ -16,7 +16,6 @@ */ package org.apache.solr.handler.admin.api; -import static org.apache.solr.cloud.Overseer.QUEUE_OPERATION; import static org.apache.solr.cloud.api.collections.CollectionHandlingUtils.ONLY_ACTIVE_NODES; import static org.apache.solr.cloud.api.collections.CollectionHandlingUtils.SHARD_UNIQUE; import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP; @@ -83,13 +82,10 @@ public SubResponseAccumulatingJerseyResponse balanceShardUnique( public static ZkNodeProps createRemoteMessage( String collectionName, BalanceShardUniqueRequestBody requestBody) { final Map remoteMessage = new HashMap<>(); - remoteMessage.put( - QUEUE_OPERATION, CollectionParams.CollectionAction.BALANCESHARDUNIQUE.toLower()); remoteMessage.put(COLLECTION_PROP, collectionName); remoteMessage.put(PROPERTY_PROP, requestBody.property); insertIfNotNull(remoteMessage, ONLY_ACTIVE_NODES, requestBody.onlyActiveNodes); insertIfNotNull(remoteMessage, SHARD_UNIQUE, requestBody.shardUnique); - insertIfNotNull(remoteMessage, ASYNC, requestBody.async); return new ZkNodeProps(remoteMessage); } diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/CreateAlias.java b/solr/core/src/java/org/apache/solr/handler/admin/api/CreateAlias.java index bbeafb210e5..71879b7abb3 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/CreateAlias.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/CreateAlias.java @@ -18,7 +18,6 @@ package org.apache.solr.handler.admin.api; import static org.apache.solr.client.solrj.request.beans.V2ApiConstants.COLLECTIONS; -import static org.apache.solr.cloud.Overseer.QUEUE_OPERATION; import static org.apache.solr.cloud.api.collections.RoutedAlias.CATEGORY; import static org.apache.solr.cloud.api.collections.RoutedAlias.CREATE_COLLECTION_PREFIX; import static org.apache.solr.cloud.api.collections.RoutedAlias.ROUTER_TYPE_NAME; @@ -30,7 +29,6 @@ import static org.apache.solr.common.params.CommonAdminParams.ASYNC; import static org.apache.solr.common.params.CommonParams.NAME; import static org.apache.solr.common.params.CommonParams.START; -import static org.apache.solr.handler.admin.CollectionsHandler.DEFAULT_COLLECTION_OP_TIMEOUT; import static org.apache.solr.security.PermissionNameProvider.Name.COLL_EDIT_PERM; import jakarta.inject.Inject; @@ -43,10 +41,8 @@ import org.apache.solr.client.api.model.CategoryRoutedAliasProperties; import org.apache.solr.client.api.model.CreateAliasRequestBody; import org.apache.solr.client.api.model.RoutedAliasProperties; -import org.apache.solr.client.api.model.SolrJerseyResponse; import org.apache.solr.client.api.model.SubResponseAccumulatingJerseyResponse; import org.apache.solr.client.api.model.TimeRoutedAliasProperties; -import org.apache.solr.client.solrj.SolrResponse; import org.apache.solr.client.solrj.request.RoutedAliasTypes; import org.apache.solr.client.solrj.util.SolrIdentifierValidator; import org.apache.solr.cloud.api.collections.RoutedAlias; @@ -61,7 +57,6 @@ import org.apache.solr.common.util.CollectionUtil; import org.apache.solr.common.util.StrUtils; import org.apache.solr.core.CoreContainer; -import org.apache.solr.handler.admin.CollectionsHandler; import org.apache.solr.jersey.PermissionName; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.response.SolrQueryResponse; @@ -83,9 +78,9 @@ public CreateAlias( @Override @PermissionName(COLL_EDIT_PERM) - public SolrJerseyResponse createAlias(CreateAliasRequestBody requestBody) throws Exception { - final SubResponseAccumulatingJerseyResponse response = - instantiateJerseyResponse(SubResponseAccumulatingJerseyResponse.class); + public SubResponseAccumulatingJerseyResponse createAlias(CreateAliasRequestBody requestBody) + throws Exception { + final var response = instantiateJerseyResponse(SubResponseAccumulatingJerseyResponse.class); recordCollectionForLogAndTracing(null, solrQueryRequest); if (requestBody == null) { @@ -109,19 +104,8 @@ public SolrJerseyResponse createAlias(CreateAliasRequestBody requestBody) throws remoteMessage = createRemoteMessageForRoutedAlias(requestBody); } - final SolrResponse remoteResponse = - CollectionsHandler.submitCollectionApiCommand( - coreContainer.getZkController(), - remoteMessage, - CollectionParams.CollectionAction.CREATEALIAS, - DEFAULT_COLLECTION_OP_TIMEOUT); - if (remoteResponse.getException() != null) { - throw remoteResponse.getException(); - } - - if (requestBody.async != null) { - response.requestId = requestBody.async; - } + submitRemoteMessageAndHandleResponse( + response, CollectionParams.CollectionAction.CREATEALIAS, remoteMessage, requestBody.async); return response; } @@ -129,19 +113,15 @@ public static ZkNodeProps createRemoteMessageForTraditionalAlias( CreateAliasRequestBody requestBody) { final Map remoteMessage = new HashMap<>(); - remoteMessage.put(QUEUE_OPERATION, CollectionParams.CollectionAction.CREATEALIAS.toLower()); remoteMessage.put(NAME, requestBody.name); remoteMessage.put("collections", String.join(",", requestBody.collections)); - remoteMessage.put(ASYNC, requestBody.async); return new ZkNodeProps(remoteMessage); } public static ZkNodeProps createRemoteMessageForRoutedAlias(CreateAliasRequestBody requestBody) { final Map remoteMessage = new HashMap<>(); - remoteMessage.put(QUEUE_OPERATION, CollectionParams.CollectionAction.CREATEALIAS.toLower()); remoteMessage.put(NAME, requestBody.name); - if (StrUtils.isNotBlank(requestBody.async)) remoteMessage.put(ASYNC, requestBody.async); if (requestBody.routers.size() > 1) { // Multi-dimensional alias for (int i = 0; i < requestBody.routers.size(); i++) { diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/CreateCollection.java b/solr/core/src/java/org/apache/solr/handler/admin/api/CreateCollection.java index fef25cfdb52..18c95583275 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/CreateCollection.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/CreateCollection.java @@ -19,7 +19,6 @@ import static org.apache.solr.client.solrj.request.beans.V2ApiConstants.ROUTER_KEY; import static org.apache.solr.client.solrj.request.beans.V2ApiConstants.SHARD_NAMES; -import static org.apache.solr.cloud.Overseer.QUEUE_OPERATION; import static org.apache.solr.cloud.api.collections.CollectionHandlingUtils.CREATE_NODE_SET; import static org.apache.solr.cloud.api.collections.CollectionHandlingUtils.CREATE_NODE_SET_SHUFFLE; import static org.apache.solr.cloud.api.collections.CollectionHandlingUtils.NUM_SLICES; @@ -36,7 +35,6 @@ import static org.apache.solr.common.params.CommonAdminParams.ASYNC; import static org.apache.solr.common.params.CommonAdminParams.WAIT_FOR_FINAL_STATE; import static org.apache.solr.common.params.CoreAdminParams.NAME; -import static org.apache.solr.handler.admin.CollectionsHandler.DEFAULT_COLLECTION_OP_TIMEOUT; import static org.apache.solr.handler.admin.CollectionsHandler.waitForActiveCollection; import static org.apache.solr.handler.api.V2ApiUtils.flattenMapWithPrefix; import static org.apache.solr.security.PermissionNameProvider.Name.COLL_EDIT_PERM; @@ -70,7 +68,6 @@ import org.apache.solr.common.util.CollectionUtil; import org.apache.solr.common.util.Utils; import org.apache.solr.core.CoreContainer; -import org.apache.solr.handler.admin.CollectionsHandler; import org.apache.solr.jersey.PermissionName; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.response.SolrQueryResponse; @@ -99,8 +96,7 @@ public SubResponseAccumulatingJerseyResponse createCollection( throw new SolrException(BAD_REQUEST, "Request body is missing but required"); } - final SubResponseAccumulatingJerseyResponse response = - instantiateJerseyResponse(SubResponseAccumulatingJerseyResponse.class); + final var response = instantiateJerseyResponse(SubResponseAccumulatingJerseyResponse.class); final CoreContainer coreContainer = fetchAndValidateZooKeeperAwareCoreContainer(); recordCollectionForLogAndTracing(requestBody.name, solrQueryRequest); @@ -109,26 +105,12 @@ public SubResponseAccumulatingJerseyResponse createCollection( // Populate any 'null' creation parameters that support COLLECTIONPROP defaults. populateDefaultsIfNecessary(coreContainer, requestBody); - final ZkNodeProps remoteMessage = createRemoteMessage(requestBody); final SolrResponse remoteResponse = - CollectionsHandler.submitCollectionApiCommand( - coreContainer.getZkController(), - remoteMessage, + submitRemoteMessageAndHandleResponse( + response, CollectionParams.CollectionAction.CREATE, - DEFAULT_COLLECTION_OP_TIMEOUT); - if (remoteResponse.getException() != null) { - throw remoteResponse.getException(); - } - - if (requestBody.async != null) { - response.requestId = requestBody.async; - return response; - } - - // Values fetched from remoteResponse may be null - response.successfulSubResponsesByNodeName = remoteResponse.getResponse().get("success"); - response.failedSubResponsesByNodeName = remoteResponse.getResponse().get("failure"); - response.warning = (String) remoteResponse.getResponse().get("warning"); + createRemoteMessage(requestBody), + requestBody.async); // Even if Overseer does wait for the collection to be created, it sees a different cluster // state than this node, so this wait is required to make sure the local node Zookeeper watches @@ -163,7 +145,6 @@ public static ZkNodeProps createRemoteMessage(CreateCollectionRequestBody reqBod final Map rawProperties = new HashMap<>(); rawProperties.put("fromApi", "true"); - rawProperties.put(QUEUE_OPERATION, CollectionParams.CollectionAction.CREATE.toLower()); rawProperties.put(NAME, reqBody.name); rawProperties.put(COLL_CONF, reqBody.config); rawProperties.put(NUM_SLICES, reqBody.numShards); @@ -176,7 +157,6 @@ public static ZkNodeProps createRemoteMessage(CreateCollectionRequestBody reqBod rawProperties.put(WAIT_FOR_FINAL_STATE, reqBody.waitForFinalState); rawProperties.put(PER_REPLICA_STATE, reqBody.perReplicaState); rawProperties.put(ALIAS, reqBody.alias); - rawProperties.put(ASYNC, reqBody.async); if (reqBody.createReplicas == null || reqBody.createReplicas) { // The remote message expects a single comma-delimited string, so nodeSet requires flattening if (reqBody.nodeSet != null) { @@ -233,7 +213,7 @@ public static Map copyPrefixedPropertiesWithoutPrefix( private static Integer readIntegerDefaultFromClusterProp( CoreContainer coreContainer, String propName) throws IOException { final Object defaultValue = - new ClusterProperties(coreContainer.getZkController().getZkStateReader().getZkClient()) + new ClusterProperties(coreContainer.getZkController().getZkClient()) .getClusterProperty( List.of(CollectionAdminParams.DEFAULTS, CollectionAdminParams.COLLECTION, propName), null); diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/CreateCollectionBackup.java b/solr/core/src/java/org/apache/solr/handler/admin/api/CreateCollectionBackup.java index 602bda58e7e..5e856012819 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/CreateCollectionBackup.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/CreateCollectionBackup.java @@ -17,7 +17,6 @@ package org.apache.solr.handler.admin.api; -import static org.apache.solr.cloud.Overseer.QUEUE_OPERATION; import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP; import static org.apache.solr.common.params.CollectionAdminParams.FOLLOW_ALIASES; import static org.apache.solr.common.params.CollectionAdminParams.INDEX_BACKUP_STRATEGY; @@ -30,7 +29,6 @@ import static org.apache.solr.common.params.CoreAdminParams.BACKUP_REPOSITORY; import static org.apache.solr.common.params.CoreAdminParams.COMMIT_NAME; import static org.apache.solr.common.params.CoreAdminParams.MAX_NUM_BACKUP_POINTS; -import static org.apache.solr.handler.admin.CollectionsHandler.DEFAULT_COLLECTION_OP_TIMEOUT; import static org.apache.solr.handler.admin.api.CreateCollection.copyPrefixedPropertiesWithoutPrefix; import static org.apache.solr.security.PermissionNameProvider.Name.COLL_EDIT_PERM; @@ -50,7 +48,6 @@ import org.apache.solr.common.params.SolrParams; import org.apache.solr.common.util.Utils; import org.apache.solr.core.CoreContainer; -import org.apache.solr.handler.admin.CollectionsHandler; import org.apache.solr.jersey.PermissionName; import org.apache.solr.jersey.SolrJacksonMapper; import org.apache.solr.request.SolrQueryRequest; @@ -109,20 +106,12 @@ public SolrJerseyResponse createCollectionBackup( "Unknown index backup strategy " + requestBody.backupStrategy); } + final var response = instantiateJerseyResponse(CreateCollectionBackupResponseBody.class); final ZkNodeProps remoteMessage = createRemoteMessage(collectionName, backupName, requestBody); final SolrResponse remoteResponse = - CollectionsHandler.submitCollectionApiCommand( - coreContainer.getZkController(), - remoteMessage, - CollectionParams.CollectionAction.BACKUP, - DEFAULT_COLLECTION_OP_TIMEOUT); - if (remoteResponse.getException() != null) { - throw remoteResponse.getException(); - } - - final SolrJerseyResponse response = - objectMapper.convertValue( - remoteResponse.getResponse(), CreateCollectionBackupResponseBody.class); + submitRemoteMessageAndHandleResponse( + response, CollectionParams.CollectionAction.BACKUP, remoteMessage, requestBody.async); + objectMapper.updateValue(response, remoteResponse.getResponse()); return response; } @@ -130,7 +119,7 @@ public SolrJerseyResponse createCollectionBackup( public static ZkNodeProps createRemoteMessage( String collectionName, String backupName, CreateCollectionBackupRequestBody requestBody) { final Map remoteMessage = Utils.reflectToMap(requestBody); - remoteMessage.put(QUEUE_OPERATION, CollectionParams.CollectionAction.BACKUP.toLower()); + remoteMessage.remove(ASYNC); remoteMessage.put(COLLECTION_PROP, collectionName); remoteMessage.put(NAME, backupName); if (!StringUtils.isBlank(requestBody.backupStrategy)) { diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/CreateCollectionSnapshot.java b/solr/core/src/java/org/apache/solr/handler/admin/api/CreateCollectionSnapshot.java index 96cf56ff3f6..2cfa244c74a 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/CreateCollectionSnapshot.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/CreateCollectionSnapshot.java @@ -16,11 +16,8 @@ */ package org.apache.solr.handler.admin.api; -import static org.apache.solr.cloud.Overseer.QUEUE_OPERATION; import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP; import static org.apache.solr.common.params.CollectionAdminParams.FOLLOW_ALIASES; -import static org.apache.solr.common.params.CommonAdminParams.ASYNC; -import static org.apache.solr.handler.admin.CollectionsHandler.DEFAULT_COLLECTION_OP_TIMEOUT; import static org.apache.solr.security.PermissionNameProvider.Name.COLL_EDIT_PERM; import jakarta.inject.Inject; @@ -29,7 +26,6 @@ import org.apache.solr.client.api.endpoint.CollectionSnapshotApis; import org.apache.solr.client.api.model.CreateCollectionSnapshotRequestBody; import org.apache.solr.client.api.model.CreateCollectionSnapshotResponse; -import org.apache.solr.client.solrj.SolrResponse; import org.apache.solr.common.SolrException; import org.apache.solr.common.cloud.SolrZkClient; import org.apache.solr.common.cloud.ZkNodeProps; @@ -37,7 +33,6 @@ import org.apache.solr.common.params.CoreAdminParams; import org.apache.solr.core.CoreContainer; import org.apache.solr.core.snapshots.SolrSnapshotManager; -import org.apache.solr.handler.admin.CollectionsHandler; import org.apache.solr.jersey.PermissionName; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.response.SolrQueryResponse; @@ -60,8 +55,7 @@ public CreateCollectionSnapshot( public CreateCollectionSnapshotResponse createCollectionSnapshot( String collName, String snapshotName, CreateCollectionSnapshotRequestBody requestBody) throws Exception { - final CreateCollectionSnapshotResponse response = - instantiateJerseyResponse(CreateCollectionSnapshotResponse.class); + final var response = instantiateJerseyResponse(CreateCollectionSnapshotResponse.class); final CoreContainer coreContainer = fetchAndValidateZooKeeperAwareCoreContainer(); recordCollectionForLogAndTracing(collName, solrQueryRequest); @@ -78,36 +72,26 @@ public CreateCollectionSnapshotResponse createCollectionSnapshot( + "', no action taken."); } - final ZkNodeProps remoteMessage = - createRemoteMessage(collName, requestBody.followAliases, snapshotName, requestBody.async); - final SolrResponse remoteResponse = - CollectionsHandler.submitCollectionApiCommand( - coreContainer.getZkController(), - remoteMessage, - CollectionParams.CollectionAction.CREATESNAPSHOT, - DEFAULT_COLLECTION_OP_TIMEOUT); - - if (remoteResponse.getException() != null) { - throw remoteResponse.getException(); - } + submitRemoteMessageAndHandleResponse( + response, + CollectionParams.CollectionAction.CREATESNAPSHOT, + createRemoteMessage(collName, requestBody.followAliases, snapshotName), + requestBody.async); response.collection = collName; response.followAliases = requestBody.followAliases; response.snapshotName = snapshotName; - response.requestId = requestBody.async; return response; } public static ZkNodeProps createRemoteMessage( - String collectionName, boolean followAliases, String snapshotName, String asyncId) { + String collectionName, boolean followAliases, String snapshotName) { final Map remoteMessage = new HashMap<>(); - remoteMessage.put(QUEUE_OPERATION, CollectionParams.CollectionAction.CREATESNAPSHOT.toLower()); remoteMessage.put(COLLECTION_PROP, collectionName); remoteMessage.put(CoreAdminParams.COMMIT_NAME, snapshotName); remoteMessage.put(FOLLOW_ALIASES, followAliases); - if (asyncId != null) remoteMessage.put(ASYNC, asyncId); return new ZkNodeProps(remoteMessage); } diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/CreateReplica.java b/solr/core/src/java/org/apache/solr/handler/admin/api/CreateReplica.java index 72c8eb058b1..af0a6a13923 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/CreateReplica.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/CreateReplica.java @@ -17,7 +17,6 @@ package org.apache.solr.handler.admin.api; -import static org.apache.solr.cloud.Overseer.QUEUE_OPERATION; import static org.apache.solr.cloud.api.collections.CollectionHandlingUtils.CREATE_NODE_SET; import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP; import static org.apache.solr.common.cloud.ZkStateReader.NRT_REPLICAS; @@ -100,7 +99,6 @@ public SubResponseAccumulatingJerseyResponse createReplica( public static ZkNodeProps createRemoteMessage( String collectionName, String shardName, CreateReplicaRequestBody requestBody) { final Map remoteMessage = new HashMap<>(); - remoteMessage.put(QUEUE_OPERATION, CollectionParams.CollectionAction.ADDREPLICA.toLower()); remoteMessage.put(COLLECTION_PROP, collectionName); remoteMessage.put(SHARD_ID_PROP, shardName); insertIfNotNull(remoteMessage, CoreAdminParams.NAME, requestBody.name); @@ -119,7 +117,6 @@ public static ZkNodeProps createRemoteMessage( insertIfNotNull(remoteMessage, TLOG_REPLICAS, requestBody.tlogReplicas); insertIfNotNull(remoteMessage, PULL_REPLICAS, requestBody.pullReplicas); insertIfNotNull(remoteMessage, FOLLOW_ALIASES, requestBody.followAliases); - insertIfNotNull(remoteMessage, ASYNC, requestBody.async); if (requestBody.properties != null) { requestBody diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/CreateShard.java b/solr/core/src/java/org/apache/solr/handler/admin/api/CreateShard.java index 2912fbd8e07..410dcb30b05 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/CreateShard.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/CreateShard.java @@ -17,7 +17,6 @@ package org.apache.solr.handler.admin.api; -import static org.apache.solr.cloud.Overseer.QUEUE_OPERATION; import static org.apache.solr.cloud.api.collections.CollectionHandlingUtils.CREATE_NODE_SET; import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP; import static org.apache.solr.common.cloud.ZkStateReader.NRT_REPLICAS; @@ -137,7 +136,6 @@ public static void invokeFromV1Params( public static ZkNodeProps createRemoteMessage( String collectionName, CreateShardRequestBody requestBody) { final Map remoteMessage = new HashMap<>(); - remoteMessage.put(QUEUE_OPERATION, CollectionParams.CollectionAction.CREATESHARD.toLower()); remoteMessage.put(COLLECTION_PROP, collectionName); remoteMessage.put(SHARD_ID_PROP, requestBody.shardName); if (requestBody.createReplicas == null || requestBody.createReplicas) { @@ -154,7 +152,6 @@ public static ZkNodeProps createRemoteMessage( insertIfNotNull(remoteMessage, PULL_REPLICAS, requestBody.pullReplicas); insertIfNotNull(remoteMessage, WAIT_FOR_FINAL_STATE, requestBody.waitForFinalState); insertIfNotNull(remoteMessage, FOLLOW_ALIASES, requestBody.followAliases); - insertIfNotNull(remoteMessage, ASYNC, requestBody.async); if (requestBody.properties != null) { requestBody diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/DeleteAlias.java b/solr/core/src/java/org/apache/solr/handler/admin/api/DeleteAlias.java index aac01f61c84..8ca11a7298e 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/DeleteAlias.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/DeleteAlias.java @@ -17,23 +17,16 @@ package org.apache.solr.handler.admin.api; -import static org.apache.solr.cloud.Overseer.QUEUE_OPERATION; -import static org.apache.solr.common.params.CommonAdminParams.ASYNC; import static org.apache.solr.common.params.CommonParams.NAME; -import static org.apache.solr.handler.admin.CollectionsHandler.DEFAULT_COLLECTION_OP_TIMEOUT; import static org.apache.solr.security.PermissionNameProvider.Name.COLL_EDIT_PERM; import jakarta.inject.Inject; -import java.util.HashMap; import java.util.Map; import org.apache.solr.client.api.endpoint.DeleteAliasApi; -import org.apache.solr.client.api.model.AsyncJerseyResponse; -import org.apache.solr.client.api.model.SolrJerseyResponse; -import org.apache.solr.client.solrj.SolrResponse; +import org.apache.solr.client.api.model.SubResponseAccumulatingJerseyResponse; import org.apache.solr.common.cloud.ZkNodeProps; import org.apache.solr.common.params.CollectionParams; import org.apache.solr.core.CoreContainer; -import org.apache.solr.handler.admin.CollectionsHandler; import org.apache.solr.jersey.PermissionName; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.response.SolrQueryResponse; @@ -49,34 +42,16 @@ public DeleteAlias( @Override @PermissionName(COLL_EDIT_PERM) - public SolrJerseyResponse deleteAlias(String aliasName, String asyncId) throws Exception { - final AsyncJerseyResponse response = instantiateJerseyResponse(AsyncJerseyResponse.class); - final CoreContainer coreContainer = fetchAndValidateZooKeeperAwareCoreContainer(); - - final ZkNodeProps remoteMessage = createRemoteMessage(aliasName, asyncId); - final SolrResponse remoteResponse = - CollectionsHandler.submitCollectionApiCommand( - coreContainer.getZkController(), - remoteMessage, - CollectionParams.CollectionAction.DELETEALIAS, - DEFAULT_COLLECTION_OP_TIMEOUT); - if (remoteResponse.getException() != null) { - throw remoteResponse.getException(); - } - - if (asyncId != null) { - response.requestId = asyncId; - } - + public SubResponseAccumulatingJerseyResponse deleteAlias(String aliasName, String asyncId) + throws Exception { + final var response = instantiateJerseyResponse(SubResponseAccumulatingJerseyResponse.class); + fetchAndValidateZooKeeperAwareCoreContainer(); + + submitRemoteMessageAndHandleResponse( + response, + CollectionParams.CollectionAction.DELETEALIAS, + new ZkNodeProps(Map.of(NAME, aliasName)), + asyncId); return response; } - - public static ZkNodeProps createRemoteMessage(String aliasName, String asyncId) { - final Map remoteMessage = new HashMap<>(); - remoteMessage.put(QUEUE_OPERATION, CollectionParams.CollectionAction.DELETEALIAS.toLower()); - remoteMessage.put(NAME, aliasName); - if (asyncId != null) remoteMessage.put(ASYNC, asyncId); - - return new ZkNodeProps(remoteMessage); - } } diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/DeleteCollection.java b/solr/core/src/java/org/apache/solr/handler/admin/api/DeleteCollection.java index 3b6d87ac33c..414ad539eda 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/DeleteCollection.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/DeleteCollection.java @@ -16,11 +16,8 @@ */ package org.apache.solr.handler.admin.api; -import static org.apache.solr.cloud.Overseer.QUEUE_OPERATION; import static org.apache.solr.common.params.CollectionAdminParams.FOLLOW_ALIASES; -import static org.apache.solr.common.params.CommonAdminParams.ASYNC; import static org.apache.solr.common.params.CommonParams.NAME; -import static org.apache.solr.handler.admin.CollectionsHandler.DEFAULT_COLLECTION_OP_TIMEOUT; import static org.apache.solr.security.PermissionNameProvider.Name.COLL_EDIT_PERM; import jakarta.inject.Inject; @@ -28,11 +25,9 @@ import java.util.Map; import org.apache.solr.client.api.endpoint.DeleteCollectionApi; import org.apache.solr.client.api.model.SubResponseAccumulatingJerseyResponse; -import org.apache.solr.client.solrj.SolrResponse; import org.apache.solr.common.cloud.ZkNodeProps; import org.apache.solr.common.params.CollectionParams; import org.apache.solr.core.CoreContainer; -import org.apache.solr.handler.admin.CollectionsHandler; import org.apache.solr.jersey.PermissionName; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.response.SolrQueryResponse; @@ -57,42 +52,22 @@ public DeleteCollection( @PermissionName(COLL_EDIT_PERM) public SubResponseAccumulatingJerseyResponse deleteCollection( String collectionName, Boolean followAliases, String asyncId) throws Exception { - final SubResponseAccumulatingJerseyResponse response = - instantiateJerseyResponse(SubResponseAccumulatingJerseyResponse.class); - final CoreContainer coreContainer = fetchAndValidateZooKeeperAwareCoreContainer(); + final var response = instantiateJerseyResponse(SubResponseAccumulatingJerseyResponse.class); + fetchAndValidateZooKeeperAwareCoreContainer(); recordCollectionForLogAndTracing(collectionName, solrQueryRequest); - final ZkNodeProps remoteMessage = createRemoteMessage(collectionName, followAliases, asyncId); - final SolrResponse remoteResponse = - CollectionsHandler.submitCollectionApiCommand( - coreContainer.getZkController(), - remoteMessage, - CollectionParams.CollectionAction.DELETE, - DEFAULT_COLLECTION_OP_TIMEOUT); - if (remoteResponse.getException() != null) { - throw remoteResponse.getException(); - } - - if (asyncId != null) { - response.requestId = asyncId; - return response; - } - - // Values fetched from remoteResponse may be null - response.successfulSubResponsesByNodeName = remoteResponse.getResponse().get("success"); - response.failedSubResponsesByNodeName = remoteResponse.getResponse().get("failure"); + final ZkNodeProps remoteMessage = createRemoteMessage(collectionName, followAliases); + submitRemoteMessageAndHandleResponse( + response, CollectionParams.CollectionAction.DELETE, remoteMessage, asyncId); return response; } - public static ZkNodeProps createRemoteMessage( - String collectionName, Boolean followAliases, String asyncId) { + public static ZkNodeProps createRemoteMessage(String collectionName, Boolean followAliases) { final Map remoteMessage = new HashMap<>(); - remoteMessage.put(QUEUE_OPERATION, CollectionParams.CollectionAction.DELETE.toLower()); remoteMessage.put(NAME, collectionName); if (followAliases != null) remoteMessage.put(FOLLOW_ALIASES, followAliases); - if (asyncId != null) remoteMessage.put(ASYNC, asyncId); return new ZkNodeProps(remoteMessage); } diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/DeleteCollectionBackup.java b/solr/core/src/java/org/apache/solr/handler/admin/api/DeleteCollectionBackup.java index bb88c09853c..a2612984123 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/DeleteCollectionBackup.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/DeleteCollectionBackup.java @@ -17,7 +17,6 @@ package org.apache.solr.handler.admin.api; -import static org.apache.solr.cloud.Overseer.QUEUE_OPERATION; import static org.apache.solr.common.SolrException.ErrorCode.BAD_REQUEST; import static org.apache.solr.common.params.CommonAdminParams.ASYNC; import static org.apache.solr.common.params.CoreAdminParams.BACKUP_ID; @@ -87,7 +86,7 @@ public BackupDeletionResponseBody deleteSingleBackupById( location = getAndValidateBackupLocation(repositoryName, location); final ZkNodeProps remoteMessage = - createRemoteMessage(backupName, backupId, null, null, location, repositoryName, asyncId); + createRemoteMessage(backupName, backupId, null, null, location, repositoryName); final var remoteResponse = submitRemoteMessageAndHandleResponse( response, CollectionParams.CollectionAction.DELETEBACKUP, remoteMessage, asyncId); @@ -113,8 +112,7 @@ public BackupDeletionResponseBody deleteMultipleBackupsByRecency( location = getAndValidateBackupLocation(repositoryName, location); final ZkNodeProps remoteMessage = - createRemoteMessage( - backupName, null, versionsToRetain, null, location, repositoryName, asyncId); + createRemoteMessage(backupName, null, versionsToRetain, null, location, repositoryName); final var remoteResponse = submitRemoteMessageAndHandleResponse( response, CollectionParams.CollectionAction.DELETEBACKUP, remoteMessage, asyncId); @@ -139,13 +137,7 @@ public PurgeUnusedResponse garbageCollectUnusedBackupFiles( final ZkNodeProps remoteMessage = createRemoteMessage( - backupName, - null, - null, - Boolean.TRUE, - requestBody.location, - requestBody.repositoryName, - requestBody.async); + backupName, null, null, Boolean.TRUE, requestBody.location, requestBody.repositoryName); final var remoteResponse = submitRemoteMessageAndHandleResponse( response, @@ -167,12 +159,10 @@ public static ZkNodeProps createRemoteMessage( Integer versionsToRetain, Boolean purgeUnused, String location, - String repositoryName, - String asyncId) { + String repositoryName) { final Map remoteMessage = new HashMap<>(); // Always provided - remoteMessage.put(QUEUE_OPERATION, CollectionParams.CollectionAction.DELETEBACKUP.toLower()); remoteMessage.put(NAME, backupName); // Mutually exclusive assert backupId != null || versionsToRetain != null || purgeUnused != null; @@ -182,7 +172,6 @@ public static ZkNodeProps createRemoteMessage( // Remaining params are truly optional insertIfNotNull(remoteMessage, BACKUP_LOCATION, location); insertIfNotNull(remoteMessage, BACKUP_REPOSITORY, repositoryName); - insertIfNotNull(remoteMessage, ASYNC, asyncId); return new ZkNodeProps(remoteMessage); } diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/DeleteCollectionSnapshot.java b/solr/core/src/java/org/apache/solr/handler/admin/api/DeleteCollectionSnapshot.java index dec728026a3..086fde0cff0 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/DeleteCollectionSnapshot.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/DeleteCollectionSnapshot.java @@ -16,11 +16,8 @@ */ package org.apache.solr.handler.admin.api; -import static org.apache.solr.cloud.Overseer.QUEUE_OPERATION; import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP; import static org.apache.solr.common.params.CollectionAdminParams.FOLLOW_ALIASES; -import static org.apache.solr.common.params.CommonAdminParams.ASYNC; -import static org.apache.solr.handler.admin.CollectionsHandler.DEFAULT_COLLECTION_OP_TIMEOUT; import static org.apache.solr.security.PermissionNameProvider.Name.COLL_EDIT_PERM; import jakarta.inject.Inject; @@ -28,12 +25,10 @@ import java.util.Map; import org.apache.solr.client.api.endpoint.CollectionSnapshotApis; import org.apache.solr.client.api.model.DeleteCollectionSnapshotResponse; -import org.apache.solr.client.solrj.SolrResponse; import org.apache.solr.common.cloud.ZkNodeProps; import org.apache.solr.common.params.CollectionParams; import org.apache.solr.common.params.CoreAdminParams; import org.apache.solr.core.CoreContainer; -import org.apache.solr.handler.admin.CollectionsHandler; import org.apache.solr.jersey.PermissionName; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.response.SolrQueryResponse; @@ -56,23 +51,16 @@ public DeleteCollectionSnapshotResponse deleteCollectionSnapshot( String collName, String snapshotName, boolean followAliases, String asyncId) throws Exception { final var response = instantiateJerseyResponse(DeleteCollectionSnapshotResponse.class); - final CoreContainer coreContainer = fetchAndValidateZooKeeperAwareCoreContainer(); + fetchAndValidateZooKeeperAwareCoreContainer(); recordCollectionForLogAndTracing(collName, solrQueryRequest); final String collectionName = resolveCollectionName(collName, followAliases); - final ZkNodeProps remoteMessage = - createRemoteMessage(collectionName, followAliases, snapshotName, asyncId); - final SolrResponse remoteResponse = - CollectionsHandler.submitCollectionApiCommand( - coreContainer.getZkController(), - remoteMessage, - CollectionParams.CollectionAction.DELETESNAPSHOT, - DEFAULT_COLLECTION_OP_TIMEOUT); - - if (remoteResponse.getException() != null) { - throw remoteResponse.getException(); - } + submitRemoteMessageAndHandleResponse( + response, + CollectionParams.CollectionAction.DELETESNAPSHOT, + createRemoteMessage(collectionName, followAliases, snapshotName), + asyncId); response.collection = collName; response.snapshotName = snapshotName; @@ -83,16 +71,13 @@ public DeleteCollectionSnapshotResponse deleteCollectionSnapshot( } public static ZkNodeProps createRemoteMessage( - String collectionName, boolean followAliases, String snapshotName, String asyncId) { + String collectionName, boolean followAliases, String snapshotName) { final Map remoteMessage = new HashMap<>(); - remoteMessage.put(QUEUE_OPERATION, CollectionParams.CollectionAction.DELETESNAPSHOT.toLower()); remoteMessage.put(COLLECTION_PROP, collectionName); remoteMessage.put(CoreAdminParams.COMMIT_NAME, snapshotName); remoteMessage.put(FOLLOW_ALIASES, followAliases); - if (asyncId != null) remoteMessage.put(ASYNC, asyncId); - return new ZkNodeProps(remoteMessage); } } diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/DeleteNode.java b/solr/core/src/java/org/apache/solr/handler/admin/api/DeleteNode.java index 78af96b4ec2..43c95ad3cb5 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/DeleteNode.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/DeleteNode.java @@ -16,25 +16,21 @@ */ package org.apache.solr.handler.admin.api; -import static org.apache.solr.cloud.Overseer.QUEUE_OPERATION; import static org.apache.solr.common.params.CommonAdminParams.ASYNC; import static org.apache.solr.common.params.CoreAdminParams.NODE; -import static org.apache.solr.handler.admin.CollectionsHandler.DEFAULT_COLLECTION_OP_TIMEOUT; import static org.apache.solr.security.PermissionNameProvider.Name.COLL_EDIT_PERM; import jakarta.inject.Inject; -import java.util.HashMap; import java.util.Map; import org.apache.solr.client.api.endpoint.DeleteNodeApi; import org.apache.solr.client.api.model.DeleteNodeRequestBody; import org.apache.solr.client.api.model.SolrJerseyResponse; -import org.apache.solr.client.solrj.SolrResponse; +import org.apache.solr.client.api.model.SubResponseAccumulatingJerseyResponse; import org.apache.solr.common.cloud.ZkNodeProps; import org.apache.solr.common.params.CollectionParams; import org.apache.solr.common.params.RequiredSolrParams; import org.apache.solr.common.params.SolrParams; import org.apache.solr.core.CoreContainer; -import org.apache.solr.handler.admin.CollectionsHandler; import org.apache.solr.jersey.PermissionName; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.response.SolrQueryResponse; @@ -57,20 +53,15 @@ public DeleteNode( @Override @PermissionName(COLL_EDIT_PERM) - public SolrJerseyResponse deleteNode(String nodeName, DeleteNodeRequestBody requestBody) - throws Exception { - final SolrJerseyResponse response = instantiateJerseyResponse(SolrJerseyResponse.class); - final CoreContainer coreContainer = fetchAndValidateZooKeeperAwareCoreContainer(); - final ZkNodeProps remoteMessage = createRemoteMessage(nodeName, requestBody); - final SolrResponse remoteResponse = - CollectionsHandler.submitCollectionApiCommand( - coreContainer.getZkController(), - remoteMessage, - CollectionParams.CollectionAction.DELETENODE, - DEFAULT_COLLECTION_OP_TIMEOUT); - if (remoteResponse.getException() != null) { - throw remoteResponse.getException(); - } + public SubResponseAccumulatingJerseyResponse deleteNode( + String nodeName, DeleteNodeRequestBody requestBody) throws Exception { + final var response = instantiateJerseyResponse(SubResponseAccumulatingJerseyResponse.class); + fetchAndValidateZooKeeperAwareCoreContainer(); + submitRemoteMessageAndHandleResponse( + response, + CollectionParams.CollectionAction.DELETENODE, + new ZkNodeProps(Map.of(NODE, nodeName)), + requestBody != null ? requestBody.async : null); disableResponseCaching(); return response; } @@ -82,18 +73,4 @@ public static SolrJerseyResponse invokeUsingV1Inputs(DeleteNode apiInstance, Sol requestBody.async = params.get(ASYNC); return apiInstance.deleteNode(requiredParams.get(NODE), requestBody); } - - public static ZkNodeProps createRemoteMessage( - String nodeName, DeleteNodeRequestBody requestBody) { - Map remoteMessage = new HashMap<>(); - remoteMessage.put(NODE, nodeName); - if (requestBody != null) { - if (requestBody.async != null) { - remoteMessage.put(ASYNC, requestBody.async); - } - } - remoteMessage.put(QUEUE_OPERATION, CollectionParams.CollectionAction.DELETENODE.toLower()); - - return new ZkNodeProps(remoteMessage); - } } diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/DeleteReplica.java b/solr/core/src/java/org/apache/solr/handler/admin/api/DeleteReplica.java index 857624b4852..2e6a2d648a9 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/DeleteReplica.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/DeleteReplica.java @@ -17,7 +17,6 @@ package org.apache.solr.handler.admin.api; -import static org.apache.solr.cloud.Overseer.QUEUE_OPERATION; import static org.apache.solr.cloud.api.collections.CollectionHandlingUtils.ONLY_IF_DOWN; import static org.apache.solr.common.SolrException.ErrorCode.BAD_REQUEST; import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP; @@ -92,8 +91,7 @@ public SubResponseAccumulatingJerseyResponse deleteReplicaByName( deleteInstanceDir, deleteDataDir, deleteIndex, - onlyIfDown, - async); + onlyIfDown); submitRemoteMessageAndHandleResponse( response, CollectionParams.CollectionAction.DELETEREPLICA, remoteMessage, async); return response; @@ -129,8 +127,7 @@ public SubResponseAccumulatingJerseyResponse deleteReplicasByCount( deleteInstanceDir, deleteDataDir, deleteIndex, - onlyIfDown, - asyncId); + onlyIfDown); submitRemoteMessageAndHandleResponse( response, CollectionParams.CollectionAction.DELETEREPLICA, remoteMessage, asyncId); return response; @@ -172,8 +169,7 @@ public SubResponseAccumulatingJerseyResponse deleteReplicasByCountAllShards( requestBody.deleteInstanceDir, requestBody.deleteDataDir, requestBody.deleteIndex, - requestBody.onlyIfDown, - requestBody.async); + requestBody.onlyIfDown); submitRemoteMessageAndHandleResponse( response, CollectionParams.CollectionAction.DELETEREPLICA, @@ -191,10 +187,8 @@ public static ZkNodeProps createRemoteMessage( Boolean deleteInstanceDir, Boolean deleteDataDir, Boolean deleteIndex, - Boolean onlyIfDown, - String asyncId) { + Boolean onlyIfDown) { final Map remoteMessage = new HashMap<>(); - remoteMessage.put(QUEUE_OPERATION, CollectionParams.CollectionAction.DELETEREPLICA.toLower()); remoteMessage.put(COLLECTION_PROP, collectionName); insertIfNotNull(remoteMessage, SHARD_ID_PROP, shardName); insertIfNotNull(remoteMessage, REPLICA, replicaName); @@ -204,7 +198,6 @@ public static ZkNodeProps createRemoteMessage( insertIfNotNull(remoteMessage, DELETE_DATA_DIR, deleteDataDir); insertIfNotNull(remoteMessage, DELETE_INDEX, deleteIndex); insertIfNotNull(remoteMessage, ONLY_IF_DOWN, onlyIfDown); - insertIfNotNull(remoteMessage, ASYNC, asyncId); return new ZkNodeProps(remoteMessage); } diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/DeleteReplicaProperty.java b/solr/core/src/java/org/apache/solr/handler/admin/api/DeleteReplicaProperty.java index 9c2486fdc39..d4da2469090 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/DeleteReplicaProperty.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/DeleteReplicaProperty.java @@ -17,25 +17,21 @@ package org.apache.solr.handler.admin.api; -import static org.apache.solr.cloud.Overseer.QUEUE_OPERATION; import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP; import static org.apache.solr.common.cloud.ZkStateReader.PROPERTY_PROP; import static org.apache.solr.common.cloud.ZkStateReader.REPLICA_PROP; import static org.apache.solr.common.cloud.ZkStateReader.SHARD_ID_PROP; import static org.apache.solr.common.params.CollectionAdminParams.PROPERTY_PREFIX; -import static org.apache.solr.handler.admin.CollectionsHandler.DEFAULT_COLLECTION_OP_TIMEOUT; import jakarta.inject.Inject; import java.util.Map; import org.apache.solr.client.api.endpoint.DeleteReplicaPropertyApi; import org.apache.solr.client.api.model.SolrJerseyResponse; -import org.apache.solr.client.solrj.SolrResponse; import org.apache.solr.common.cloud.ZkNodeProps; import org.apache.solr.common.params.CollectionParams; import org.apache.solr.common.params.RequiredSolrParams; import org.apache.solr.common.params.SolrParams; import org.apache.solr.core.CoreContainer; -import org.apache.solr.handler.admin.CollectionsHandler; import org.apache.solr.jersey.PermissionName; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.response.SolrQueryResponse; @@ -61,20 +57,13 @@ public DeleteReplicaProperty( public SolrJerseyResponse deleteReplicaProperty( String collName, String shardName, String replicaName, String propertyName) throws Exception { final SolrJerseyResponse response = instantiateJerseyResponse(SolrJerseyResponse.class); - final CoreContainer coreContainer = fetchAndValidateZooKeeperAwareCoreContainer(); + fetchAndValidateZooKeeperAwareCoreContainer(); recordCollectionForLogAndTracing(collName, solrQueryRequest); - final ZkNodeProps remoteMessage = - createRemoteMessage(collName, shardName, replicaName, propertyName); - final SolrResponse remoteResponse = - CollectionsHandler.submitCollectionApiCommand( - coreContainer.getZkController(), - remoteMessage, - CollectionParams.CollectionAction.DELETEREPLICAPROP, - DEFAULT_COLLECTION_OP_TIMEOUT); - if (remoteResponse.getException() != null) { - throw remoteResponse.getException(); - } + submitRemoteMessageAndHandleResponse( + response, + CollectionParams.CollectionAction.DELETEREPLICAPROP, + createRemoteMessage(collName, shardName, replicaName, propertyName)); return response; } @@ -99,8 +88,6 @@ public static ZkNodeProps createRemoteMessage( String collName, String shardName, String replicaName, String propName) { final Map messageProperties = Map.of( - QUEUE_OPERATION, - CollectionParams.CollectionAction.DELETEREPLICAPROP.toLower(), COLLECTION_PROP, collName, SHARD_ID_PROP, diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/DeleteShard.java b/solr/core/src/java/org/apache/solr/handler/admin/api/DeleteShard.java index 5b9b5407274..374738894f6 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/DeleteShard.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/DeleteShard.java @@ -17,7 +17,6 @@ package org.apache.solr.handler.admin.api; -import static org.apache.solr.cloud.Overseer.QUEUE_OPERATION; import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP; import static org.apache.solr.common.cloud.ZkStateReader.SHARD_ID_PROP; import static org.apache.solr.common.params.CollectionAdminParams.FOLLOW_ALIASES; @@ -96,14 +95,12 @@ public static ZkNodeProps createRemoteMessage( Boolean followAliases, String asyncId) { final Map remoteMessage = new HashMap<>(); - remoteMessage.put(QUEUE_OPERATION, CollectionParams.CollectionAction.DELETESHARD.toLower()); remoteMessage.put(COLLECTION_PROP, collectionName); remoteMessage.put(SHARD_ID_PROP, shardName); insertIfNotNull(remoteMessage, FOLLOW_ALIASES, followAliases); insertIfNotNull(remoteMessage, DELETE_INSTANCE_DIR, deleteInstanceDir); insertIfNotNull(remoteMessage, DELETE_DATA_DIR, deleteDataDir); insertIfNotNull(remoteMessage, DELETE_INDEX, deleteIndex); - insertIfNotNull(remoteMessage, ASYNC, asyncId); return new ZkNodeProps(remoteMessage); } diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/InstallShardData.java b/solr/core/src/java/org/apache/solr/handler/admin/api/InstallShardData.java index 98623f7d209..4033fa6f097 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/InstallShardData.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/InstallShardData.java @@ -17,15 +17,13 @@ package org.apache.solr.handler.admin.api; -import static org.apache.solr.handler.admin.CollectionsHandler.DEFAULT_COLLECTION_OP_TIMEOUT; import static org.apache.solr.security.PermissionNameProvider.Name.COLL_EDIT_PERM; import jakarta.inject.Inject; import java.util.HashMap; import org.apache.solr.client.api.endpoint.InstallShardDataApi; -import org.apache.solr.client.api.model.AsyncJerseyResponse; import org.apache.solr.client.api.model.InstallShardDataRequestBody; -import org.apache.solr.client.solrj.SolrResponse; +import org.apache.solr.client.api.model.SubResponseAccumulatingJerseyResponse; import org.apache.solr.cloud.api.collections.InstallShardDataCmd; import org.apache.solr.common.SolrException; import org.apache.solr.common.cloud.ClusterState; @@ -35,7 +33,6 @@ import org.apache.solr.common.params.CollectionParams; import org.apache.solr.common.util.StrUtils; import org.apache.solr.core.CoreContainer; -import org.apache.solr.handler.admin.CollectionsHandler; import org.apache.solr.jersey.PermissionName; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.response.SolrQueryResponse; @@ -59,9 +56,9 @@ public InstallShardData( @Override @PermissionName(COLL_EDIT_PERM) - public AsyncJerseyResponse installShardData( + public SubResponseAccumulatingJerseyResponse installShardData( String collName, String shardName, InstallShardDataRequestBody requestBody) throws Exception { - final var response = instantiateJerseyResponse(AsyncJerseyResponse.class); + final var response = instantiateJerseyResponse(SubResponseAccumulatingJerseyResponse.class); final CoreContainer coreContainer = fetchAndValidateZooKeeperAwareCoreContainer(); recordCollectionForLogAndTracing(collName, solrQueryRequest); if (requestBody == null) { @@ -81,27 +78,17 @@ public AsyncJerseyResponse installShardData( // Only install data to shards which belong to a collection in read-only mode final DocCollection dc = coreContainer.getZkController().getZkStateReader().getCollection(collName); - if (!dc.isReadOnly()) { + if (dc.getSlice(shardName).getReplicas().size() > 1 && !dc.isReadOnly()) { throw new SolrException( SolrException.ErrorCode.BAD_REQUEST, - "Collection must be in readOnly mode before installing data to shard"); + "Collection must be in readOnly mode before installing data to shard with more than 1 replica"); } - final ZkNodeProps remoteMessage = createRemoteMessage(collName, shardName, requestBody); - final SolrResponse remoteResponse = - CollectionsHandler.submitCollectionApiCommand( - coreContainer.getZkController(), - remoteMessage, - CollectionParams.CollectionAction.INSTALLSHARDDATA, - DEFAULT_COLLECTION_OP_TIMEOUT); - if (remoteResponse.getException() != null) { - throw remoteResponse.getException(); - } - - if (requestBody.async != null) { - response.requestId = requestBody.async; - return response; - } + submitRemoteMessageAndHandleResponse( + response, + CollectionParams.CollectionAction.INSTALLSHARDDATA, + createRemoteMessage(collName, shardName, requestBody), + requestBody.async); return response; } @@ -125,7 +112,8 @@ public static ZkNodeProps createRemoteMessage( if (requestBody != null) { messageTyped.location = requestBody.location; messageTyped.repository = requestBody.repository; - messageTyped.asyncId = requestBody.async; + messageTyped.name = requestBody.name; + messageTyped.shardBackupId = requestBody.shardBackupId; } messageTyped.validate(); diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/MigrateReplicas.java b/solr/core/src/java/org/apache/solr/handler/admin/api/MigrateReplicas.java index 670540b4f74..ad9b9604080 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/MigrateReplicas.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/MigrateReplicas.java @@ -16,12 +16,9 @@ */ package org.apache.solr.handler.admin.api; -import static org.apache.solr.cloud.Overseer.QUEUE_OPERATION; import static org.apache.solr.common.params.CollectionParams.SOURCE_NODES; import static org.apache.solr.common.params.CollectionParams.TARGET_NODES; -import static org.apache.solr.common.params.CommonAdminParams.ASYNC; import static org.apache.solr.common.params.CommonAdminParams.WAIT_FOR_FINAL_STATE; -import static org.apache.solr.handler.admin.CollectionsHandler.DEFAULT_COLLECTION_OP_TIMEOUT; import static org.apache.solr.security.PermissionNameProvider.Name.COLL_EDIT_PERM; import jakarta.inject.Inject; @@ -29,13 +26,11 @@ import java.util.Map; import org.apache.solr.client.api.endpoint.MigrateReplicasApi; import org.apache.solr.client.api.model.MigrateReplicasRequestBody; -import org.apache.solr.client.api.model.SolrJerseyResponse; -import org.apache.solr.client.solrj.SolrResponse; +import org.apache.solr.client.api.model.SubResponseAccumulatingJerseyResponse; import org.apache.solr.common.SolrException; import org.apache.solr.common.cloud.ZkNodeProps; import org.apache.solr.common.params.CollectionParams.CollectionAction; import org.apache.solr.core.CoreContainer; -import org.apache.solr.handler.admin.CollectionsHandler; import org.apache.solr.jersey.PermissionName; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.response.SolrQueryResponse; @@ -53,21 +48,16 @@ public MigrateReplicas( @Override @PermissionName(COLL_EDIT_PERM) - public SolrJerseyResponse migrateReplicas(MigrateReplicasRequestBody requestBody) - throws Exception { - final SolrJerseyResponse response = instantiateJerseyResponse(SolrJerseyResponse.class); - final CoreContainer coreContainer = fetchAndValidateZooKeeperAwareCoreContainer(); - // TODO Record node for log and tracing - final ZkNodeProps remoteMessage = createRemoteMessage(requestBody); - final SolrResponse remoteResponse = - CollectionsHandler.submitCollectionApiCommand( - coreContainer.getZkController(), - remoteMessage, - CollectionAction.MIGRATE_REPLICAS, - DEFAULT_COLLECTION_OP_TIMEOUT); - if (remoteResponse.getException() != null) { - throw remoteResponse.getException(); - } + public SubResponseAccumulatingJerseyResponse migrateReplicas( + MigrateReplicasRequestBody requestBody) throws Exception { + final var response = instantiateJerseyResponse(SubResponseAccumulatingJerseyResponse.class); + fetchAndValidateZooKeeperAwareCoreContainer(); + recordCollectionForLogAndTracing(null, solrQueryRequest); + submitRemoteMessageAndHandleResponse( + response, + CollectionAction.MIGRATE_REPLICAS, + createRemoteMessage(requestBody), + requestBody.async); disableResponseCaching(); return response; @@ -84,13 +74,11 @@ public ZkNodeProps createRemoteMessage(MigrateReplicasRequestBody requestBody) { insertIfNotNull(remoteMessage, SOURCE_NODES, requestBody.sourceNodes); insertIfNotNull(remoteMessage, TARGET_NODES, requestBody.targetNodes); insertIfNotNull(remoteMessage, WAIT_FOR_FINAL_STATE, requestBody.waitForFinalState); - insertIfNotNull(remoteMessage, ASYNC, requestBody.async); } else { throw new SolrException( SolrException.ErrorCode.BAD_REQUEST, "No request body sent with request. The MigrateReplicas API requires a body."); } - remoteMessage.put(QUEUE_OPERATION, CollectionAction.MIGRATE_REPLICAS.toLower()); return new ZkNodeProps(remoteMessage); } diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/ReloadCollectionAPI.java b/solr/core/src/java/org/apache/solr/handler/admin/api/ReloadCollectionAPI.java index b620dd430a7..227d944e8e3 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/ReloadCollectionAPI.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/ReloadCollectionAPI.java @@ -16,7 +16,6 @@ */ package org.apache.solr.handler.admin.api; -import static org.apache.solr.cloud.Overseer.QUEUE_OPERATION; import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP; import static org.apache.solr.common.params.CommonAdminParams.ASYNC; import static org.apache.solr.common.params.CommonParams.NAME; @@ -24,7 +23,6 @@ import jakarta.inject.Inject; import java.lang.invoke.MethodHandles; -import java.util.HashMap; import java.util.Map; import org.apache.solr.client.api.endpoint.ReloadCollectionApi; import org.apache.solr.client.api.model.ReloadCollectionRequestBody; @@ -66,27 +64,14 @@ public SubResponseAccumulatingJerseyResponse reloadCollection( fetchAndValidateZooKeeperAwareCoreContainer(); recordCollectionForLogAndTracing(collectionName, solrQueryRequest); - final ZkNodeProps remoteMessage = createRemoteMessage(collectionName, requestBody); submitRemoteMessageAndHandleResponse( response, CollectionParams.CollectionAction.RELOAD, - remoteMessage, + new ZkNodeProps(Map.of(NAME, collectionName)), requestBody != null ? requestBody.async : null); return response; } - public static ZkNodeProps createRemoteMessage( - String collectionName, ReloadCollectionRequestBody requestBody) { - final Map remoteMessage = new HashMap<>(); - remoteMessage.put(QUEUE_OPERATION, CollectionParams.CollectionAction.RELOAD.toLower()); - remoteMessage.put(NAME, collectionName); - if (requestBody != null) { - insertIfNotNull(remoteMessage, ASYNC, requestBody.async); - } - - return new ZkNodeProps(remoteMessage); - } - public static void invokeFromV1Params( CoreContainer coreContainer, SolrQueryRequest request, SolrQueryResponse response) throws Exception { diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/RenameCollection.java b/solr/core/src/java/org/apache/solr/handler/admin/api/RenameCollection.java index 1a6ed3a3946..6562ae1d872 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/RenameCollection.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/RenameCollection.java @@ -16,7 +16,6 @@ */ package org.apache.solr.handler.admin.api; -import static org.apache.solr.cloud.Overseer.QUEUE_OPERATION; import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP; import static org.apache.solr.common.params.CollectionAdminParams.FOLLOW_ALIASES; import static org.apache.solr.common.params.CollectionAdminParams.TARGET; @@ -69,21 +68,16 @@ public SubResponseAccumulatingJerseyResponse renameCollection( final ZkNodeProps remoteMessage = createRemoteMessage(collectionName, requestBody); submitRemoteMessageAndHandleResponse( - response, - CollectionParams.CollectionAction.RENAME, - remoteMessage, - requestBody != null ? requestBody.async : null); + response, CollectionParams.CollectionAction.RENAME, remoteMessage, requestBody.async); return response; } public static ZkNodeProps createRemoteMessage( String collectionName, RenameCollectionRequestBody requestBody) { final Map remoteMessage = new HashMap<>(); - remoteMessage.put(QUEUE_OPERATION, CollectionParams.CollectionAction.RENAME.toLower()); remoteMessage.put(NAME, collectionName); remoteMessage.put(TARGET, requestBody.to); insertIfNotNull(remoteMessage, FOLLOW_ALIASES, requestBody.followAliases); - insertIfNotNull(remoteMessage, ASYNC, requestBody.async); return new ZkNodeProps(remoteMessage); } diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/ReplaceNode.java b/solr/core/src/java/org/apache/solr/handler/admin/api/ReplaceNode.java index bb36f2036df..cd162659087 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/ReplaceNode.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/ReplaceNode.java @@ -16,12 +16,9 @@ */ package org.apache.solr.handler.admin.api; -import static org.apache.solr.cloud.Overseer.QUEUE_OPERATION; import static org.apache.solr.common.params.CollectionParams.SOURCE_NODE; import static org.apache.solr.common.params.CollectionParams.TARGET_NODE; -import static org.apache.solr.common.params.CommonAdminParams.ASYNC; import static org.apache.solr.common.params.CommonAdminParams.WAIT_FOR_FINAL_STATE; -import static org.apache.solr.handler.admin.CollectionsHandler.DEFAULT_COLLECTION_OP_TIMEOUT; import static org.apache.solr.security.PermissionNameProvider.Name.COLL_EDIT_PERM; import jakarta.inject.Inject; @@ -29,13 +26,10 @@ import java.util.Map; import org.apache.solr.client.api.endpoint.ReplaceNodeApi; import org.apache.solr.client.api.model.ReplaceNodeRequestBody; -import org.apache.solr.client.api.model.SolrJerseyResponse; -import org.apache.solr.client.solrj.SolrResponse; +import org.apache.solr.client.api.model.SubResponseAccumulatingJerseyResponse; import org.apache.solr.common.cloud.ZkNodeProps; import org.apache.solr.common.params.CollectionParams; -import org.apache.solr.common.params.CollectionParams.CollectionAction; import org.apache.solr.core.CoreContainer; -import org.apache.solr.handler.admin.CollectionsHandler; import org.apache.solr.jersey.PermissionName; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.response.SolrQueryResponse; @@ -57,21 +51,16 @@ public ReplaceNode( @Override @PermissionName(COLL_EDIT_PERM) - public SolrJerseyResponse replaceNode(String sourceNodeName, ReplaceNodeRequestBody requestBody) - throws Exception { - final SolrJerseyResponse response = instantiateJerseyResponse(SolrJerseyResponse.class); - final CoreContainer coreContainer = fetchAndValidateZooKeeperAwareCoreContainer(); - // TODO Record node for log and tracing - final ZkNodeProps remoteMessage = createRemoteMessage(sourceNodeName, requestBody); - final SolrResponse remoteResponse = - CollectionsHandler.submitCollectionApiCommand( - coreContainer.getZkController(), - remoteMessage, - CollectionParams.CollectionAction.REPLACENODE, - DEFAULT_COLLECTION_OP_TIMEOUT); - if (remoteResponse.getException() != null) { - throw remoteResponse.getException(); - } + public SubResponseAccumulatingJerseyResponse replaceNode( + String sourceNodeName, ReplaceNodeRequestBody requestBody) throws Exception { + final var response = instantiateJerseyResponse(SubResponseAccumulatingJerseyResponse.class); + fetchAndValidateZooKeeperAwareCoreContainer(); + recordCollectionForLogAndTracing(null, solrQueryRequest); + submitRemoteMessageAndHandleResponse( + response, + CollectionParams.CollectionAction.REPLACENODE, + createRemoteMessage(sourceNodeName, requestBody), + requestBody != null ? requestBody.async : null); disableResponseCaching(); return response; @@ -83,9 +72,7 @@ public ZkNodeProps createRemoteMessage(String nodeName, ReplaceNodeRequestBody r if (requestBody != null) { insertIfValueNotNull(remoteMessage, TARGET_NODE, requestBody.targetNodeName); insertIfValueNotNull(remoteMessage, WAIT_FOR_FINAL_STATE, requestBody.waitForFinalState); - insertIfValueNotNull(remoteMessage, ASYNC, requestBody.async); } - remoteMessage.put(QUEUE_OPERATION, CollectionAction.REPLACENODE.toLower()); return new ZkNodeProps(remoteMessage); } diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/RestoreCollection.java b/solr/core/src/java/org/apache/solr/handler/admin/api/RestoreCollection.java index fb39fafa02e..84cd3a3377c 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/RestoreCollection.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/RestoreCollection.java @@ -17,7 +17,6 @@ package org.apache.solr.handler.admin.api; -import static org.apache.solr.cloud.Overseer.QUEUE_OPERATION; import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP; import static org.apache.solr.common.params.CollectionAdminParams.COLL_CONF; import static org.apache.solr.common.params.CollectionAdminParams.CREATE_NODE_SET_PARAM; @@ -29,7 +28,6 @@ import static org.apache.solr.common.params.CoreAdminParams.BACKUP_ID; import static org.apache.solr.common.params.CoreAdminParams.BACKUP_LOCATION; import static org.apache.solr.common.params.CoreAdminParams.BACKUP_REPOSITORY; -import static org.apache.solr.handler.admin.CollectionsHandler.DEFAULT_COLLECTION_OP_TIMEOUT; import static org.apache.solr.security.PermissionNameProvider.Name.COLL_EDIT_PERM; import jakarta.inject.Inject; @@ -41,7 +39,6 @@ import org.apache.solr.client.api.model.RestoreCollectionRequestBody; import org.apache.solr.client.api.model.SolrJerseyResponse; import org.apache.solr.client.api.model.SubResponseAccumulatingJerseyResponse; -import org.apache.solr.client.solrj.SolrResponse; import org.apache.solr.client.solrj.util.SolrIdentifierValidator; import org.apache.solr.cloud.api.collections.CollectionHandlingUtils; import org.apache.solr.common.SolrException; @@ -51,7 +48,6 @@ import org.apache.solr.common.params.SolrParams; import org.apache.solr.common.util.Utils; import org.apache.solr.core.CoreContainer; -import org.apache.solr.handler.admin.CollectionsHandler; import org.apache.solr.jersey.PermissionName; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.response.SolrQueryResponse; @@ -113,6 +109,7 @@ public SubResponseAccumulatingJerseyResponse restoreCollection( final var createRequestBody = requestBody.createCollectionParams; if (createRequestBody != null) { + createRequestBody.name = collectionName; CreateCollection.populateDefaultsIfNecessary(coreContainer, createRequestBody); CreateCollection.validateRequestBody(createRequestBody); if (Boolean.FALSE.equals(createRequestBody.createReplicas)) { @@ -123,31 +120,15 @@ public SubResponseAccumulatingJerseyResponse restoreCollection( } final ZkNodeProps remoteMessage = createRemoteMessage(backupName, requestBody); - final SolrResponse remoteResponse = - CollectionsHandler.submitCollectionApiCommand( - coreContainer.getZkController(), - remoteMessage, - CollectionParams.CollectionAction.RESTORE, - DEFAULT_COLLECTION_OP_TIMEOUT); - - if (remoteResponse.getException() != null) { - throw remoteResponse.getException(); - } - - if (requestBody.async != null) { - response.requestId = requestBody.async; - return response; - } - - // Values fetched from remoteResponse may be null - response.successfulSubResponsesByNodeName = remoteResponse.getResponse().get("success"); - response.failedSubResponsesByNodeName = remoteResponse.getResponse().get("failure"); + submitRemoteMessageAndHandleResponse( + response, CollectionParams.CollectionAction.RESTORE, remoteMessage, requestBody.async); return response; } public ZkNodeProps createRemoteMessage( String backupName, RestoreCollectionRequestBody requestBody) { final Map remoteMessage = Utils.reflectToMap(requestBody); + remoteMessage.remove(ASYNC); // If the RESTORE is setup to create a new collection, copy those parameters first final var createReqBody = requestBody.createCollectionParams; @@ -164,7 +145,6 @@ public ZkNodeProps createRemoteMessage( } // Copy restore-specific parameters - remoteMessage.put(QUEUE_OPERATION, CollectionParams.CollectionAction.RESTORE.toLower()); remoteMessage.put(NAME, backupName); return new ZkNodeProps(remoteMessage); } diff --git a/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandler.java b/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandler.java index df0f39ccea5..43165272027 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandler.java @@ -188,6 +188,9 @@ private LBSolrClient.Req prepareLBRequest( params.remove(CommonParams.WT); // use default (currently javabin) QueryRequest req = createQueryRequest(sreq, params, shard); req.setMethod(SolrRequest.METHOD.POST); + if (sreq.headers != null) { + req.addHeaders(sreq.headers); + } SolrRequestInfo requestInfo = SolrRequestInfo.getRequestInfo(); if (requestInfo != null) { req.setUserPrincipal(requestInfo.getUserPrincipal()); diff --git a/solr/core/src/java/org/apache/solr/handler/component/ShardRequest.java b/solr/core/src/java/org/apache/solr/handler/component/ShardRequest.java index d7d7da04f22..385e7b72036 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/ShardRequest.java +++ b/solr/core/src/java/org/apache/solr/handler/component/ShardRequest.java @@ -18,6 +18,7 @@ import java.util.ArrayList; import java.util.List; +import java.util.Map; import org.apache.solr.common.params.ModifiableSolrParams; // todo... when finalized make accessors @@ -56,6 +57,9 @@ public class ShardRequest { /** may be null */ public String nodeName; + /** may be null */ + public Map headers; + // TODO: one could store a list of numbers to correlate where returned docs // go in the top-level response rather than looking up by id... // this would work well if we ever transitioned to using internal ids and diff --git a/solr/core/src/java/org/apache/solr/util/TimeOut.java b/solr/core/src/java/org/apache/solr/util/TimeOut.java index a1d682a6ed9..6fdc9bd2609 100644 --- a/solr/core/src/java/org/apache/solr/util/TimeOut.java +++ b/solr/core/src/java/org/apache/solr/util/TimeOut.java @@ -18,6 +18,7 @@ import static java.util.concurrent.TimeUnit.NANOSECONDS; +import java.time.Duration; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.function.Supplier; @@ -30,6 +31,14 @@ public class TimeOut { private final long timeoutAt, startTime; private final TimeSource timeSource; + /** + * @param timeout after this maximum time, {@link #hasTimedOut()} will return true. + * @param timeSource the source of the time. + */ + public TimeOut(Duration timeout, TimeSource timeSource) { + this(timeout.toNanos(), NANOSECONDS, timeSource); + } + /** * @param timeout after this maximum time, {@link #hasTimedOut()} will return true. * @param unit the time unit of the timeout argument. diff --git a/solr/core/src/test/org/apache/solr/cloud/TestLockTree.java b/solr/core/src/test/org/apache/solr/cloud/TestLockTree.java index 2604feadd65..08ab76b7f7d 100644 --- a/solr/core/src/test/org/apache/solr/cloud/TestLockTree.java +++ b/solr/core/src/test/org/apache/solr/cloud/TestLockTree.java @@ -41,23 +41,26 @@ public class TestLockTree extends SolrTestCaseJ4 { public void testLocks() throws Exception { LockTree lockTree = new LockTree(); - Lock coll1Lock = lockTree.getSession().lock(CollectionAction.CREATE, Arrays.asList("coll1")); + Lock coll1Lock = + lockTree.getSession().lock(CollectionAction.CREATE, Arrays.asList("coll1"), null); assertNotNull(coll1Lock); assertNull( "Should not be able to lock coll1/shard1", lockTree .getSession() - .lock(CollectionAction.BALANCESHARDUNIQUE, Arrays.asList("coll1", "shard1"))); + .lock(CollectionAction.BALANCESHARDUNIQUE, Arrays.asList("coll1", "shard1"), null)); coll1Lock.unlock(); Lock shard1Lock = lockTree .getSession() - .lock(CollectionAction.BALANCESHARDUNIQUE, Arrays.asList("coll1", "shard1")); + .lock(CollectionAction.BALANCESHARDUNIQUE, Arrays.asList("coll1", "shard1"), null); assertNotNull(shard1Lock); shard1Lock.unlock(); Lock replica1Lock = - lockTree.getSession().lock(ADDREPLICAPROP, Arrays.asList("coll1", "shard1", "core_node2")); + lockTree + .getSession() + .lock(ADDREPLICAPROP, Arrays.asList("coll1", "shard1", "core_node2"), null); assertNotNull(replica1Lock); List>> operations = new ArrayList<>(); @@ -80,7 +83,7 @@ public void testLocks() throws Exception { List locks = new CopyOnWriteArrayList<>(); List threads = new ArrayList<>(); for (Pair> operation : operations) { - final Lock lock = session.lock(operation.first(), operation.second()); + final Lock lock = session.lock(operation.first(), operation.second(), null); if (lock != null) { Thread thread = new Thread(getRunnable(completedOps, operation, locks, lock)); threads.add(thread); diff --git a/solr/core/src/test/org/apache/solr/cloud/ZkDistributedLockTest.java b/solr/core/src/test/org/apache/solr/cloud/ZkDistributedLockTest.java index 2c767b91fef..c6b132ccb94 100644 --- a/solr/core/src/test/org/apache/solr/cloud/ZkDistributedLockTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/ZkDistributedLockTest.java @@ -17,6 +17,7 @@ package org.apache.solr.cloud; +import java.util.Collections; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import org.apache.solr.SolrTestCaseJ4; @@ -75,17 +76,32 @@ private void monothreadedCollectionTests(DistributedCollectionLockFactory factor // Collection level locks DistributedLock collRL1 = factory.createLock( - false, CollectionParams.LockLevel.COLLECTION, COLLECTION_NAME, null, null); + false, + CollectionParams.LockLevel.COLLECTION, + COLLECTION_NAME, + null, + null, + Collections.emptyList()); assertTrue("collRL1 should have been acquired", collRL1.isAcquired()); DistributedLock collRL2 = factory.createLock( - false, CollectionParams.LockLevel.COLLECTION, COLLECTION_NAME, null, null); + false, + CollectionParams.LockLevel.COLLECTION, + COLLECTION_NAME, + null, + null, + Collections.emptyList()); assertTrue("collRL1 should have been acquired", collRL2.isAcquired()); DistributedLock collWL3 = factory.createLock( - true, CollectionParams.LockLevel.COLLECTION, COLLECTION_NAME, null, null); + true, + CollectionParams.LockLevel.COLLECTION, + COLLECTION_NAME, + null, + null, + Collections.emptyList()); assertFalse( "collWL3 should not have been acquired, due to collRL1 and collRL2", collWL3.isAcquired()); @@ -100,7 +116,12 @@ private void monothreadedCollectionTests(DistributedCollectionLockFactory factor DistributedLock collRL4 = factory.createLock( - false, CollectionParams.LockLevel.COLLECTION, COLLECTION_NAME, null, null); + false, + CollectionParams.LockLevel.COLLECTION, + COLLECTION_NAME, + null, + null, + Collections.emptyList()); assertFalse( "collRL4 should not have been acquired, due to collWL3 locking the collection", collRL4.isAcquired()); @@ -110,14 +131,24 @@ private void monothreadedCollectionTests(DistributedCollectionLockFactory factor // should see no impact. DistributedLock shardWL5 = factory.createLock( - true, CollectionParams.LockLevel.SHARD, COLLECTION_NAME, SHARD_NAME, null); + true, + CollectionParams.LockLevel.SHARD, + COLLECTION_NAME, + SHARD_NAME, + null, + Collections.emptyList()); assertTrue( "shardWL5 should have been acquired, there is no lock on that shard", shardWL5.isAcquired()); DistributedLock shardWL6 = factory.createLock( - true, CollectionParams.LockLevel.SHARD, COLLECTION_NAME, SHARD_NAME, null); + true, + CollectionParams.LockLevel.SHARD, + COLLECTION_NAME, + SHARD_NAME, + null, + Collections.emptyList()); assertFalse( "shardWL6 should not have been acquired, shardWL5 is locking that shard", shardWL6.isAcquired()); @@ -125,12 +156,22 @@ private void monothreadedCollectionTests(DistributedCollectionLockFactory factor // Get a lock on a Replica. Again this is independent of collection or shard level DistributedLock replicaRL7 = factory.createLock( - false, CollectionParams.LockLevel.REPLICA, COLLECTION_NAME, SHARD_NAME, REPLICA_NAME); + false, + CollectionParams.LockLevel.REPLICA, + COLLECTION_NAME, + SHARD_NAME, + REPLICA_NAME, + Collections.emptyList()); assertTrue("replicaRL7 should have been acquired", replicaRL7.isAcquired()); DistributedLock replicaWL8 = factory.createLock( - true, CollectionParams.LockLevel.REPLICA, COLLECTION_NAME, SHARD_NAME, REPLICA_NAME); + true, + CollectionParams.LockLevel.REPLICA, + COLLECTION_NAME, + SHARD_NAME, + REPLICA_NAME, + Collections.emptyList()); assertFalse( "replicaWL8 should not have been acquired, replicaRL7 is read locking that replica", replicaWL8.isAcquired()); @@ -164,13 +205,23 @@ private void multithreadedCollectionTests(DistributedCollectionLockFactory facto // Acquiring right away a read lock DistributedLock readLock = factory.createLock( - false, CollectionParams.LockLevel.COLLECTION, COLLECTION_NAME, null, null); + false, + CollectionParams.LockLevel.COLLECTION, + COLLECTION_NAME, + null, + null, + Collections.emptyList()); assertTrue("readLock should have been acquired", readLock.isAcquired()); // And now creating a write lock, that can't be acquired just yet, because of the read lock DistributedLock writeLock = factory.createLock( - true, CollectionParams.LockLevel.COLLECTION, COLLECTION_NAME, null, null); + true, + CollectionParams.LockLevel.COLLECTION, + COLLECTION_NAME, + null, + null, + Collections.emptyList()); assertFalse("writeLock should not have been acquired", writeLock.isAcquired()); // Wait for acquisition of the write lock on another thread (and be notified via a latch) diff --git a/solr/core/src/test/org/apache/solr/cloud/api/collections/CollectionApiLockingTest.java b/solr/core/src/test/org/apache/solr/cloud/api/collections/CollectionApiLockingTest.java index fb943f6a5e6..7f6f7e7daf4 100644 --- a/solr/core/src/test/org/apache/solr/cloud/api/collections/CollectionApiLockingTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/api/collections/CollectionApiLockingTest.java @@ -65,7 +65,10 @@ private void monothreadedTests(CollectionApiLockFactory apiLockingHelper) { // hierarchy) DistributedMultiLock collLock = apiLockingHelper.createCollectionApiLock( - CollectionParams.LockLevel.COLLECTION, COLLECTION_NAME, null, null); + new AdminCmdContext(CollectionParams.CollectionAction.CREATE), + COLLECTION_NAME, + null, + null); assertTrue("Collection should have been acquired", collLock.isAcquired()); assertEquals( "Lock at collection level expected to need one distributed lock", @@ -76,7 +79,10 @@ private void monothreadedTests(CollectionApiLockFactory apiLockingHelper) { // above DistributedMultiLock shard1Lock = apiLockingHelper.createCollectionApiLock( - CollectionParams.LockLevel.SHARD, COLLECTION_NAME, SHARD1_NAME, null); + new AdminCmdContext(CollectionParams.CollectionAction.SPLITSHARD), + COLLECTION_NAME, + SHARD1_NAME, + null); assertFalse("Shard1 should not have been acquired", shard1Lock.isAcquired()); assertEquals( "Lock at shard level expected to need two distributed locks", @@ -87,7 +93,10 @@ private void monothreadedTests(CollectionApiLockFactory apiLockingHelper) { // collection lock above DistributedMultiLock shard2Lock = apiLockingHelper.createCollectionApiLock( - CollectionParams.LockLevel.SHARD, COLLECTION_NAME, SHARD2_NAME, null); + new AdminCmdContext(CollectionParams.CollectionAction.SPLITSHARD), + COLLECTION_NAME, + SHARD2_NAME, + null); assertFalse("Shard2 should not have been acquired", shard2Lock.isAcquired()); assertTrue("Collection should still be acquired", collLock.isAcquired()); @@ -104,7 +113,10 @@ private void monothreadedTests(CollectionApiLockFactory apiLockingHelper) { // Request a lock on replica of shard1 DistributedMultiLock replicaShard1Lock = apiLockingHelper.createCollectionApiLock( - CollectionParams.LockLevel.SHARD, COLLECTION_NAME, SHARD1_NAME, REPLICA_NAME); + new AdminCmdContext(CollectionParams.CollectionAction.MOCK_REPLICA_TASK), + COLLECTION_NAME, + SHARD1_NAME, + REPLICA_NAME); assertFalse( "replicaShard1Lock should not have been acquired, shard1 is locked", replicaShard1Lock.isAcquired()); @@ -112,7 +124,10 @@ private void monothreadedTests(CollectionApiLockFactory apiLockingHelper) { // Now ask for a new lock on the collection collLock = apiLockingHelper.createCollectionApiLock( - CollectionParams.LockLevel.COLLECTION, COLLECTION_NAME, null, null); + new AdminCmdContext(CollectionParams.CollectionAction.CREATE), + COLLECTION_NAME, + null, + null); assertFalse( "Collection should not have been acquired, shard1 and shard2 locks preventing it", @@ -131,7 +146,10 @@ private void monothreadedTests(CollectionApiLockFactory apiLockingHelper) { // Request a lock on replica of shard2 DistributedMultiLock replicaShard2Lock = apiLockingHelper.createCollectionApiLock( - CollectionParams.LockLevel.SHARD, COLLECTION_NAME, SHARD2_NAME, REPLICA_NAME); + new AdminCmdContext(CollectionParams.CollectionAction.MOCK_REPLICA_TASK), + COLLECTION_NAME, + SHARD2_NAME, + REPLICA_NAME); assertFalse( "replicaShard2Lock should not have been acquired, shard2 is locked", replicaShard2Lock.isAcquired()); @@ -158,13 +176,19 @@ private void multithreadedTests(CollectionApiLockFactory apiLockingHelper) throw // Lock on collection... DistributedMultiLock collLock = apiLockingHelper.createCollectionApiLock( - CollectionParams.LockLevel.COLLECTION, COLLECTION_NAME, null, null); + new AdminCmdContext(CollectionParams.CollectionAction.CREATE), + COLLECTION_NAME, + null, + null); assertTrue("Collection should have been acquired", collLock.isAcquired()); // ...blocks a lock on replica from being acquired final DistributedMultiLock replicaShard1Lock = apiLockingHelper.createCollectionApiLock( - CollectionParams.LockLevel.SHARD, COLLECTION_NAME, SHARD1_NAME, REPLICA_NAME); + new AdminCmdContext(CollectionParams.CollectionAction.MOCK_REPLICA_TASK), + COLLECTION_NAME, + SHARD1_NAME, + REPLICA_NAME); assertFalse( "replicaShard1Lock should not have been acquired, because collection is locked", replicaShard1Lock.isAcquired()); diff --git a/solr/core/src/test/org/apache/solr/handler/admin/api/AddReplicaPropertyAPITest.java b/solr/core/src/test/org/apache/solr/handler/admin/api/AddReplicaPropertyAPITest.java index 07874f26486..4b4cd003315 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/api/AddReplicaPropertyAPITest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/api/AddReplicaPropertyAPITest.java @@ -17,70 +17,34 @@ package org.apache.solr.handler.admin.api; -import static org.apache.solr.cloud.api.collections.CollectionHandlingUtils.SHARD_UNIQUE; -import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyLong; -import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; -import io.opentelemetry.api.trace.Span; import java.util.Map; -import java.util.Optional; -import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.client.api.model.AddReplicaPropertyRequestBody; -import org.apache.solr.cloud.OverseerSolrResponse; -import org.apache.solr.cloud.ZkController; -import org.apache.solr.cloud.api.collections.DistributedCollectionConfigSetCommandRunner; +import org.apache.solr.cloud.api.collections.AdminCmdContext; import org.apache.solr.common.SolrException; import org.apache.solr.common.cloud.ZkNodeProps; -import org.apache.solr.common.util.NamedList; -import org.apache.solr.core.CoreContainer; -import org.apache.solr.request.SolrQueryRequest; -import org.apache.solr.response.SolrQueryResponse; +import org.apache.solr.common.params.CollectionParams; import org.junit.Before; -import org.junit.BeforeClass; import org.junit.Test; -import org.mockito.ArgumentCaptor; /** Unit tests for {@link AddReplicaProperty} */ -public class AddReplicaPropertyAPITest extends SolrTestCaseJ4 { +public class AddReplicaPropertyAPITest extends MockAPITest { private static final AddReplicaPropertyRequestBody ANY_REQ_BODY = new AddReplicaPropertyRequestBody("anyValue"); - private CoreContainer mockCoreContainer; - private ZkController mockZkController; - private DistributedCollectionConfigSetCommandRunner mockCommandRunner; - private SolrQueryRequest mockQueryRequest; - private SolrQueryResponse queryResponse; - private ArgumentCaptor messageCapturer; - - private AddReplicaProperty addReplicaPropApi; - - @BeforeClass - public static void ensureWorkingMockito() { - assumeWorkingMockito(); - } + private AddReplicaProperty api; @Override @Before public void setUp() throws Exception { super.setUp(); + when(mockCoreContainer.isZooKeeperAware()).thenReturn(true); - mockCoreContainer = mock(CoreContainer.class); - mockZkController = mock(ZkController.class); - mockCommandRunner = mock(DistributedCollectionConfigSetCommandRunner.class); - when(mockCoreContainer.getZkController()).thenReturn(mockZkController); - when(mockZkController.getDistributedCommandRunner()).thenReturn(Optional.of(mockCommandRunner)); - when(mockCommandRunner.runCollectionCommand(any(), any(), anyLong())) - .thenReturn(new OverseerSolrResponse(new NamedList<>())); - mockQueryRequest = mock(SolrQueryRequest.class); - when(mockQueryRequest.getSpan()).thenReturn(Span.getInvalid()); - queryResponse = new SolrQueryResponse(); - messageCapturer = ArgumentCaptor.forClass(ZkNodeProps.class); - - addReplicaPropApi = new AddReplicaProperty(mockCoreContainer, mockQueryRequest, queryResponse); + api = new AddReplicaProperty(mockCoreContainer, mockQueryRequest, queryResponse); } @Test @@ -91,7 +55,7 @@ public void testReportsErrorWhenCalledInStandaloneMode() { expectThrows( SolrException.class, () -> { - addReplicaPropApi.addReplicaProperty( + api.addReplicaProperty( "someColl", "someShard", "someReplica", "somePropName", ANY_REQ_BODY); }); assertEquals(400, e.code()); @@ -104,21 +68,21 @@ public void testReportsErrorWhenCalledInStandaloneMode() { public void testCreatesValidOverseerMessage() throws Exception { when(mockCoreContainer.isZooKeeperAware()).thenReturn(true); - addReplicaPropApi.addReplicaProperty( - "someColl", "someShard", "someReplica", "somePropName", ANY_REQ_BODY); - verify(mockCommandRunner).runCollectionCommand(messageCapturer.capture(), any(), anyLong()); + api.addReplicaProperty("someColl", "someShard", "someReplica", "somePropName", ANY_REQ_BODY); + verify(mockCommandRunner) + .runCollectionCommand(contextCapturer.capture(), messageCapturer.capture(), anyLong()); final ZkNodeProps createdMessage = messageCapturer.getValue(); final Map createdMessageProps = createdMessage.getProperties(); - assertEquals(6, createdMessageProps.size()); - assertEquals("addreplicaprop", createdMessageProps.get("operation")); + assertEquals(5, createdMessageProps.size()); assertEquals("someColl", createdMessageProps.get("collection")); assertEquals("someShard", createdMessageProps.get("shard")); assertEquals("someReplica", createdMessageProps.get("replica")); assertEquals("somePropName", createdMessageProps.get("property")); assertEquals("anyValue", createdMessageProps.get("property.value")); - assertFalse( - createdMessageProps.containsKey( - SHARD_UNIQUE)); // Omitted since not specified on request body + + final AdminCmdContext context = contextCapturer.getValue(); + assertEquals(CollectionParams.CollectionAction.ADDREPLICAPROP, context.getAction()); + assertNull("asyncId should be null", context.getAsyncId()); } } diff --git a/solr/core/src/test/org/apache/solr/handler/admin/api/BackupCoreAPITest.java b/solr/core/src/test/org/apache/solr/handler/admin/api/BackupCoreAPITest.java index 2e5af32ff5d..8bb81e676cb 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/api/BackupCoreAPITest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/api/BackupCoreAPITest.java @@ -38,7 +38,7 @@ public class BackupCoreAPITest extends SolrTestCaseJ4 { - private CreateCoreBackup backupCoreAPI; + private CreateCoreBackup api; private static final String backupName = "my-new-backup"; @BeforeClass @@ -57,7 +57,7 @@ public void setUp() throws Exception { CoreAdminHandler.CoreAdminAsyncTracker coreAdminAsyncTracker = new CoreAdminHandler.CoreAdminAsyncTracker(); - backupCoreAPI = + api = new CreateCoreBackup( coreContainer, solrQueryRequest, solrQueryResponse, coreAdminAsyncTracker); } @@ -68,8 +68,7 @@ public void testCreateNonIncrementalBackupReturnsValidResponse() throws Exceptio backupCoreRequestBody.incremental = false; backupCoreRequestBody.backupName = backupName; SnapShooter.CoreSnapshotResponse response = - (SnapShooter.CoreSnapshotResponse) - backupCoreAPI.createBackup(coreName, backupCoreRequestBody); + (SnapShooter.CoreSnapshotResponse) api.createBackup(coreName, backupCoreRequestBody); assertEquals(backupName, response.snapshotName); assertEquals("snapshot." + backupName, response.directoryName); @@ -84,11 +83,7 @@ public void testMissingLocationParameter() throws Exception { backupCoreRequestBody.incremental = false; backupCoreRequestBody.backupName = backupName; final SolrException solrException = - expectThrows( - SolrException.class, - () -> { - backupCoreAPI.createBackup(coreName, backupCoreRequestBody); - }); + expectThrows(SolrException.class, () -> api.createBackup(coreName, backupCoreRequestBody)); assertEquals(500, solrException.code()); assertTrue( "Exception message differed from expected: " + solrException.getMessage(), @@ -105,11 +100,7 @@ public void testMissingCoreNameParameter() throws Exception { backupCoreRequestBody.backupName = backupName; final SolrException solrException = - expectThrows( - SolrException.class, - () -> { - backupCoreAPI.createBackup(null, backupCoreRequestBody); - }); + expectThrows(SolrException.class, () -> api.createBackup(null, backupCoreRequestBody)); assertEquals(400, solrException.code()); assertTrue( "Exception message differed from expected: " + solrException.getMessage(), @@ -125,9 +116,7 @@ public void testNonIncrementalBackupForNonExistentCore() throws Exception { final SolrException solrException = expectThrows( SolrException.class, - () -> { - backupCoreAPI.createBackup("non-existent-core", backupCoreRequestBody); - }); + () -> api.createBackup("non-existent-core", backupCoreRequestBody)); assertEquals(500, solrException.code()); } @@ -138,7 +127,7 @@ public void testCreateIncrementalBackupReturnsValidResponse() throws Exception { backupCoreRequestBody.shardBackupId = "md_shard1_0"; IncrementalShardBackup.IncrementalShardSnapshotResponse response = (IncrementalShardBackup.IncrementalShardSnapshotResponse) - backupCoreAPI.createBackup(coreName, backupCoreRequestBody); + api.createBackup(coreName, backupCoreRequestBody); assertEquals(1, response.indexFileCount); assertEquals(1, response.uploadedIndexFileCount); diff --git a/solr/core/src/test/org/apache/solr/handler/admin/api/BalanceShardUniqueAPITest.java b/solr/core/src/test/org/apache/solr/handler/admin/api/BalanceShardUniqueAPITest.java index c986f332da8..fd1ea94d874 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/api/BalanceShardUniqueAPITest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/api/BalanceShardUniqueAPITest.java @@ -17,30 +17,41 @@ package org.apache.solr.handler.admin.api; -import static org.apache.solr.cloud.Overseer.QUEUE_OPERATION; import static org.apache.solr.cloud.api.collections.CollectionHandlingUtils.ONLY_ACTIVE_NODES; import static org.apache.solr.cloud.api.collections.CollectionHandlingUtils.SHARD_UNIQUE; import static org.apache.solr.common.cloud.ZkStateReader.PROPERTY_PROP; import static org.apache.solr.common.params.CollectionAdminParams.COLLECTION; -import static org.apache.solr.common.params.CommonAdminParams.ASYNC; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; -import org.apache.solr.SolrTestCaseJ4; +import java.util.Map; import org.apache.solr.client.api.model.BalanceShardUniqueRequestBody; +import org.apache.solr.cloud.api.collections.AdminCmdContext; import org.apache.solr.common.SolrException; +import org.apache.solr.common.cloud.ZkNodeProps; +import org.apache.solr.common.params.CollectionParams; +import org.junit.Before; import org.junit.Test; /** Unit tests for {@link BalanceShardUnique} */ -public class BalanceShardUniqueAPITest extends SolrTestCaseJ4 { +public class BalanceShardUniqueAPITest extends MockAPITest { + + private BalanceShardUnique api; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + when(mockCoreContainer.isZooKeeperAware()).thenReturn(true); + + api = new BalanceShardUnique(mockCoreContainer, mockQueryRequest, queryResponse); + } @Test public void testReportsErrorIfRequestBodyMissing() { final SolrException thrown = - expectThrows( - SolrException.class, - () -> { - final var api = new BalanceShardUnique(null, null, null); - api.balanceShardUnique("someCollectionName", null); - }); + expectThrows(SolrException.class, () -> api.balanceShardUnique("someCollectionName", null)); assertEquals(400, thrown.code()); assertEquals("Missing required request body", thrown.getMessage()); @@ -51,12 +62,7 @@ public void testReportsErrorIfCollectionNameMissing() { final var requestBody = new BalanceShardUniqueRequestBody(); requestBody.property = "preferredLeader"; final SolrException thrown = - expectThrows( - SolrException.class, - () -> { - final var api = new BalanceShardUnique(null, null, null); - api.balanceShardUnique(null, requestBody); - }); + expectThrows(SolrException.class, () -> api.balanceShardUnique(null, requestBody)); assertEquals(400, thrown.code()); assertEquals("Missing required parameter: collection", thrown.getMessage()); @@ -68,32 +74,35 @@ public void testReportsErrorIfPropertyToBalanceIsMissing() { final var requestBody = new BalanceShardUniqueRequestBody(); final SolrException thrown = expectThrows( - SolrException.class, - () -> { - final var api = new BalanceShardUnique(null, null, null); - api.balanceShardUnique("someCollName", requestBody); - }); + SolrException.class, () -> api.balanceShardUnique("someCollName", requestBody)); assertEquals(400, thrown.code()); assertEquals("Missing required parameter: property", thrown.getMessage()); } @Test - public void testCreateRemoteMessageAllProperties() { + public void testCreateRemoteMessageAllProperties() throws Exception { final var requestBody = new BalanceShardUniqueRequestBody(); requestBody.property = "someProperty"; requestBody.shardUnique = Boolean.TRUE; requestBody.onlyActiveNodes = Boolean.TRUE; requestBody.async = "someAsyncId"; - final var remoteMessage = - BalanceShardUnique.createRemoteMessage("someCollName", requestBody).getProperties(); - assertEquals(6, remoteMessage.size()); - assertEquals("balanceshardunique", remoteMessage.get(QUEUE_OPERATION)); + api.balanceShardUnique("someCollName", requestBody); + verify(mockCommandRunner) + .runCollectionCommand(contextCapturer.capture(), messageCapturer.capture(), anyLong()); + + final ZkNodeProps createdMessage = messageCapturer.getValue(); + final Map remoteMessage = createdMessage.getProperties(); + + assertEquals(4, remoteMessage.size()); assertEquals("someCollName", remoteMessage.get(COLLECTION)); assertEquals("someProperty", remoteMessage.get(PROPERTY_PROP)); assertEquals(Boolean.TRUE, remoteMessage.get(SHARD_UNIQUE)); assertEquals(Boolean.TRUE, remoteMessage.get(ONLY_ACTIVE_NODES)); - assertEquals("someAsyncId", remoteMessage.get(ASYNC)); + + final AdminCmdContext context = contextCapturer.getValue(); + assertEquals(CollectionParams.CollectionAction.BALANCESHARDUNIQUE, context.getAction()); + assertEquals("someAsyncId", context.getAsyncId()); } } diff --git a/solr/core/src/test/org/apache/solr/handler/admin/api/CoreSnapshotAPITest.java b/solr/core/src/test/org/apache/solr/handler/admin/api/CoreSnapshotAPITest.java index 5ada271d324..12b72b2c563 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/api/CoreSnapshotAPITest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/api/CoreSnapshotAPITest.java @@ -34,7 +34,7 @@ public class CoreSnapshotAPITest extends SolrTestCaseJ4 { - private CoreSnapshot coreSnapshotAPI; + private CoreSnapshot api; @BeforeClass public static void initializeCoreAndRequestFactory() throws Exception { @@ -54,7 +54,7 @@ public void setUp() throws Exception { CoreAdminHandler.CoreAdminAsyncTracker coreAdminAsyncTracker = new CoreAdminHandler.CoreAdminAsyncTracker(); - coreSnapshotAPI = + api = new CoreSnapshot(solrQueryRequest, solrQueryResponse, coreContainer, coreAdminAsyncTracker); } @@ -63,7 +63,7 @@ public void setUp() throws Exception { @After public void deleteSnapshots() throws Exception { for (String snapshotName : snapshotsToCleanup) { - coreSnapshotAPI.deleteSnapshot(coreName, snapshotName, null); + api.deleteSnapshot(coreName, snapshotName, null); } snapshotsToCleanup.clear(); @@ -73,8 +73,7 @@ public void deleteSnapshots() throws Exception { public void testCreateSnapshotReturnsValidResponse() throws Exception { final String snapshotName = "my-new-snapshot"; - final CreateCoreSnapshotResponse response = - coreSnapshotAPI.createSnapshot(coreName, snapshotName, null); + final CreateCoreSnapshotResponse response = api.createSnapshot(coreName, snapshotName, null); snapshotsToCleanup.add(snapshotName); assertEquals(coreName, response.core); @@ -91,9 +90,7 @@ public void testReportsErrorWhenCreatingSnapshotForNonexistentCore() { final SolrException solrException = expectThrows( SolrException.class, - () -> { - coreSnapshotAPI.createSnapshot(nonExistentCoreName, "my-new-snapshot", null); - }); + () -> api.createSnapshot(nonExistentCoreName, "my-new-snapshot", null)); assertEquals(400, solrException.code()); assertTrue( "Exception message differed from expected: " + solrException.getMessage(), @@ -106,11 +103,11 @@ public void testListSnapshotsReturnsValidResponse() throws Exception { for (int i = 0; i < 5; i++) { final String snapshotName = snapshotNameBase + i; - coreSnapshotAPI.createSnapshot(coreName, snapshotName, null); + api.createSnapshot(coreName, snapshotName, null); snapshotsToCleanup.add(snapshotName); } - final ListCoreSnapshotsResponse response = coreSnapshotAPI.listSnapshots(coreName); + final ListCoreSnapshotsResponse response = api.listSnapshots(coreName); assertEquals(5, response.snapshots.size()); } @@ -123,7 +120,7 @@ public void testReportsErrorWhenListingSnapshotsForNonexistentCore() { expectThrows( SolrException.class, () -> { - coreSnapshotAPI.listSnapshots(nonExistentCoreName); + api.listSnapshots(nonExistentCoreName); }); assertEquals(400, solrException.code()); assertTrue( @@ -135,15 +132,14 @@ public void testReportsErrorWhenListingSnapshotsForNonexistentCore() { public void testDeleteSnapshotReturnsValidResponse() throws Exception { final String snapshotName = "my-new-snapshot"; - coreSnapshotAPI.createSnapshot(coreName, snapshotName, null); + api.createSnapshot(coreName, snapshotName, null); - final DeleteSnapshotResponse deleteResponse = - coreSnapshotAPI.deleteSnapshot(coreName, snapshotName, null); + final DeleteSnapshotResponse deleteResponse = api.deleteSnapshot(coreName, snapshotName, null); assertEquals(coreName, deleteResponse.coreName); assertEquals(snapshotName, deleteResponse.commitName); - final ListCoreSnapshotsResponse response = coreSnapshotAPI.listSnapshots(coreName); + final ListCoreSnapshotsResponse response = api.listSnapshots(coreName); assertEquals(0, response.snapshots.size()); } @@ -155,9 +151,7 @@ public void testReportsErrorWhenDeletingSnapshotForNonexistentCore() { final SolrException solrException = expectThrows( SolrException.class, - () -> { - coreSnapshotAPI.deleteSnapshot(nonExistentCoreName, "non-existent-snapshot", null); - }); + () -> api.deleteSnapshot(nonExistentCoreName, "non-existent-snapshot", null)); assertEquals(400, solrException.code()); assertTrue( "Exception message differed from expected: " + solrException.getMessage(), diff --git a/solr/core/src/test/org/apache/solr/handler/admin/api/CreateAliasAPITest.java b/solr/core/src/test/org/apache/solr/handler/admin/api/CreateAliasAPITest.java index b4f5e2d2760..bcfdf9ca068 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/api/CreateAliasAPITest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/api/CreateAliasAPITest.java @@ -18,33 +18,43 @@ package org.apache.solr.handler.admin.api; import static org.apache.solr.client.solrj.request.beans.V2ApiConstants.COLLECTIONS; -import static org.apache.solr.cloud.Overseer.QUEUE_OPERATION; -import static org.apache.solr.common.params.CommonAdminParams.ASYNC; import static org.hamcrest.Matchers.containsString; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; import java.util.List; -import org.apache.solr.SolrTestCaseJ4; +import java.util.Map; import org.apache.solr.client.api.model.CategoryRoutedAliasProperties; import org.apache.solr.client.api.model.CreateAliasRequestBody; import org.apache.solr.client.api.model.CreateCollectionRequestBody; import org.apache.solr.client.api.model.RoutedAliasProperties; import org.apache.solr.client.api.model.TimeRoutedAliasProperties; +import org.apache.solr.cloud.api.collections.AdminCmdContext; import org.apache.solr.common.SolrException; +import org.apache.solr.common.cloud.ZkNodeProps; +import org.apache.solr.common.params.CollectionParams; import org.apache.solr.common.params.ModifiableSolrParams; +import org.junit.Before; import org.junit.Test; /** Unit tests for {@link CreateAlias} */ -public class CreateAliasAPITest extends SolrTestCaseJ4 { +public class CreateAliasAPITest extends MockAPITest { + + private CreateAlias api; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + when(mockCoreContainer.isZooKeeperAware()).thenReturn(true); + + api = new CreateAlias(mockCoreContainer, mockQueryRequest, queryResponse); + } @Test public void testReportsErrorIfRequestBodyMissing() { - final SolrException thrown = - expectThrows( - SolrException.class, - () -> { - final var api = new CreateAlias(null, null, null); - api.createAlias(null); - }); + final SolrException thrown = expectThrows(SolrException.class, () -> api.createAlias(null)); assertEquals(400, thrown.code()); assertEquals("Request body is required but missing", thrown.getMessage()); @@ -202,24 +212,29 @@ public void testReportsErrorIfTimeRoutedAliasDoesntSpecifyAllRequiredParameters( } @Test - public void testRemoteMessageCreationForTraditionalAlias() { + public void testRemoteMessageCreationForTraditionalAlias() throws Exception { final var requestBody = new CreateAliasRequestBody(); requestBody.name = "someAliasName"; requestBody.collections = List.of("validColl1", "validColl2"); requestBody.async = "someAsyncId"; - final var remoteMessage = - CreateAlias.createRemoteMessageForTraditionalAlias(requestBody).getProperties(); + api.createAlias(requestBody); + verify(mockCommandRunner) + .runCollectionCommand(contextCapturer.capture(), messageCapturer.capture(), anyLong()); - assertEquals(4, remoteMessage.size()); - assertEquals("createalias", remoteMessage.get(QUEUE_OPERATION)); + final ZkNodeProps createdMessage = messageCapturer.getValue(); + final Map remoteMessage = createdMessage.getProperties(); + assertEquals(2, remoteMessage.size()); assertEquals("someAliasName", remoteMessage.get("name")); assertEquals("validColl1,validColl2", remoteMessage.get(COLLECTIONS)); - assertEquals("someAsyncId", remoteMessage.get(ASYNC)); + + final AdminCmdContext context = contextCapturer.getValue(); + assertEquals(CollectionParams.CollectionAction.CREATEALIAS, context.getAction()); + assertEquals("someAsyncId", context.getAsyncId()); } @Test - public void testRemoteMessageCreationForCategoryRoutedAlias() { + public void testRemoteMessageCreationForCategoryRoutedAlias() throws Exception { final var requestBody = new CreateAliasRequestBody(); requestBody.name = "someAliasName"; final var categoryRouter = new CategoryRoutedAliasProperties(); @@ -230,25 +245,31 @@ public void testRemoteMessageCreationForCategoryRoutedAlias() { createParams.config = "someConfig"; requestBody.collCreationParameters = createParams; - final var remoteMessage = - CreateAlias.createRemoteMessageForRoutedAlias(requestBody).getProperties(); + api.createAlias(requestBody); + verify(mockCommandRunner) + .runCollectionCommand(contextCapturer.capture(), messageCapturer.capture(), anyLong()); - assertEquals(6, remoteMessage.size()); - assertEquals("createalias", remoteMessage.get(QUEUE_OPERATION)); + final ZkNodeProps createdMessage = messageCapturer.getValue(); + final Map remoteMessage = createdMessage.getProperties(); + assertEquals(5, remoteMessage.size()); assertEquals("someAliasName", remoteMessage.get("name")); assertEquals("category", remoteMessage.get("router.name")); assertEquals("someField", remoteMessage.get("router.field")); assertEquals(3, remoteMessage.get("create-collection.numShards")); assertEquals("someConfig", remoteMessage.get("create-collection.collection.configName")); + + final AdminCmdContext context = contextCapturer.getValue(); + assertEquals(CollectionParams.CollectionAction.CREATEALIAS, context.getAction()); + assertNull(context.getAsyncId()); } @Test - public void testRemoteMessageCreationForTimeRoutedAlias() { + public void testRemoteMessageCreationForTimeRoutedAlias() throws Exception { final var requestBody = new CreateAliasRequestBody(); requestBody.name = "someAliasName"; final var timeRouter = new TimeRoutedAliasProperties(); timeRouter.field = "someField"; - timeRouter.start = "NOW"; + timeRouter.start = "NOW/HOUR"; timeRouter.interval = "+1MONTH"; timeRouter.maxFutureMs = 123456L; requestBody.routers = List.of(timeRouter); @@ -257,28 +278,34 @@ public void testRemoteMessageCreationForTimeRoutedAlias() { createParams.config = "someConfig"; requestBody.collCreationParameters = createParams; - final var remoteMessage = - CreateAlias.createRemoteMessageForRoutedAlias(requestBody).getProperties(); + api.createAlias(requestBody); + verify(mockCommandRunner) + .runCollectionCommand(contextCapturer.capture(), messageCapturer.capture(), anyLong()); - assertEquals(9, remoteMessage.size()); - assertEquals("createalias", remoteMessage.get(QUEUE_OPERATION)); + final ZkNodeProps createdMessage = messageCapturer.getValue(); + final Map remoteMessage = createdMessage.getProperties(); + assertEquals(8, remoteMessage.size()); assertEquals("someAliasName", remoteMessage.get("name")); assertEquals("time", remoteMessage.get("router.name")); assertEquals("someField", remoteMessage.get("router.field")); - assertEquals("NOW", remoteMessage.get("router.start")); + assertEquals("NOW/HOUR", remoteMessage.get("router.start")); assertEquals("+1MONTH", remoteMessage.get("router.interval")); assertEquals(Long.valueOf(123456L), remoteMessage.get("router.maxFutureMs")); assertEquals(3, remoteMessage.get("create-collection.numShards")); assertEquals("someConfig", remoteMessage.get("create-collection.collection.configName")); + + final AdminCmdContext context = contextCapturer.getValue(); + assertEquals(CollectionParams.CollectionAction.CREATEALIAS, context.getAction()); + assertNull(context.getAsyncId()); } @Test - public void testRemoteMessageCreationForMultiDimensionalRoutedAlias() { + public void testRemoteMessageCreationForMultiDimensionalRoutedAlias() throws Exception { final var requestBody = new CreateAliasRequestBody(); requestBody.name = "someAliasName"; final var timeRouter = new TimeRoutedAliasProperties(); timeRouter.field = "someField"; - timeRouter.start = "NOW"; + timeRouter.start = "NOW/HOUR"; timeRouter.interval = "+1MONTH"; timeRouter.maxFutureMs = 123456L; final var categoryRouter = new CategoryRoutedAliasProperties(); @@ -289,21 +316,27 @@ public void testRemoteMessageCreationForMultiDimensionalRoutedAlias() { createParams.config = "someConfig"; requestBody.collCreationParameters = createParams; - final var remoteMessage = - CreateAlias.createRemoteMessageForRoutedAlias(requestBody).getProperties(); + api.createAlias(requestBody); + verify(mockCommandRunner) + .runCollectionCommand(contextCapturer.capture(), messageCapturer.capture(), anyLong()); - assertEquals(11, remoteMessage.size()); - assertEquals("createalias", remoteMessage.get(QUEUE_OPERATION)); + final ZkNodeProps createdMessage = messageCapturer.getValue(); + final Map remoteMessage = createdMessage.getProperties(); + assertEquals(10, remoteMessage.size()); assertEquals("someAliasName", remoteMessage.get("name")); assertEquals("time", remoteMessage.get("router.0.name")); assertEquals("someField", remoteMessage.get("router.0.field")); - assertEquals("NOW", remoteMessage.get("router.0.start")); + assertEquals("NOW/HOUR", remoteMessage.get("router.0.start")); assertEquals("+1MONTH", remoteMessage.get("router.0.interval")); assertEquals(Long.valueOf(123456L), remoteMessage.get("router.0.maxFutureMs")); assertEquals("category", remoteMessage.get("router.1.name")); assertEquals("someField", remoteMessage.get("router.1.field")); assertEquals(3, remoteMessage.get("create-collection.numShards")); assertEquals("someConfig", remoteMessage.get("create-collection.collection.configName")); + + final AdminCmdContext context = contextCapturer.getValue(); + assertEquals(CollectionParams.CollectionAction.CREATEALIAS, context.getAction()); + assertNull(context.getAsyncId()); } private CreateAliasRequestBody requestBodyWithProvidedRouter(RoutedAliasProperties router) { @@ -325,7 +358,7 @@ public void testConvertsV1ParamsForMultiDimensionalAliasToV2RequestBody() { v1Params.add("name", "someAliasName"); v1Params.add("router.name", "Dimensional[time,category]"); v1Params.add("router.0.field", "someField"); - v1Params.add("router.0.start", "NOW"); + v1Params.add("router.0.start", "NOW/HOUR"); v1Params.add("router.0.interval", "+1MONTH"); v1Params.add("router.0.maxFutureMs", "123456"); v1Params.add("router.1.field", "someOtherField"); @@ -342,7 +375,7 @@ public void testConvertsV1ParamsForMultiDimensionalAliasToV2RequestBody() { requestBody.routers.get(0) instanceof TimeRoutedAliasProperties); final var timeRouter = (TimeRoutedAliasProperties) requestBody.routers.get(0); assertEquals("someField", timeRouter.field); - assertEquals("NOW", timeRouter.start); + assertEquals("NOW/HOUR", timeRouter.start); assertEquals("+1MONTH", timeRouter.interval); assertEquals(Long.valueOf(123456L), timeRouter.maxFutureMs); assertTrue( diff --git a/solr/core/src/test/org/apache/solr/handler/admin/api/CreateCollectionAPITest.java b/solr/core/src/test/org/apache/solr/handler/admin/api/CreateCollectionAPITest.java index 80a2200131a..6fdbaa74419 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/api/CreateCollectionAPITest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/api/CreateCollectionAPITest.java @@ -17,7 +17,6 @@ package org.apache.solr.handler.admin.api; -import static org.apache.solr.cloud.Overseer.QUEUE_OPERATION; import static org.apache.solr.cloud.api.collections.CollectionHandlingUtils.CREATE_NODE_SET; import static org.apache.solr.cloud.api.collections.CollectionHandlingUtils.NUM_SLICES; import static org.apache.solr.common.cloud.DocCollection.CollectionStateProps.SHARDS; @@ -30,31 +29,47 @@ import static org.apache.solr.common.params.CollectionAdminParams.PULL_REPLICAS; import static org.apache.solr.common.params.CollectionAdminParams.REPLICATION_FACTOR; import static org.apache.solr.common.params.CollectionAdminParams.TLOG_REPLICAS; -import static org.apache.solr.common.params.CommonAdminParams.ASYNC; import static org.apache.solr.common.params.CommonAdminParams.WAIT_FOR_FINAL_STATE; import static org.apache.solr.common.params.CoreAdminParams.NAME; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import java.nio.charset.StandardCharsets; import java.util.List; import java.util.Map; -import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.client.api.model.CreateCollectionRequestBody; import org.apache.solr.client.api.model.CreateCollectionRouterProperties; +import org.apache.solr.cloud.api.collections.AdminCmdContext; import org.apache.solr.common.SolrException; +import org.apache.solr.common.cloud.ZkNodeProps; +import org.apache.solr.common.params.CollectionParams; import org.apache.solr.common.params.ModifiableSolrParams; +import org.junit.Before; import org.junit.Test; /** Unit tests for {@link CreateCollection}. */ -public class CreateCollectionAPITest extends SolrTestCaseJ4 { +public class CreateCollectionAPITest extends MockAPITest { + + private CreateCollection api; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + when(mockCoreContainer.isZooKeeperAware()).thenReturn(true); + + api = new CreateCollection(mockCoreContainer, mockQueryRequest, queryResponse); + when(mockSolrZkClient.getData(eq("properties.json"), any(), any())) + .thenReturn("{}".getBytes(StandardCharsets.UTF_8)); + } @Test public void testReportsErrorIfRequestBodyMissing() { final SolrException thrown = - expectThrows( - SolrException.class, - () -> { - final var api = new CreateCollection(null, null, null); - api.createCollection(null); - }); + expectThrows(SolrException.class, () -> api.createCollection(null)); assertEquals(400, thrown.code()); assertEquals("Request body is missing but required", thrown.getMessage()); @@ -71,11 +86,7 @@ public void testReportsErrorIfReplicationFactorAndNrtReplicasConflict() { requestBody.nrtReplicas = 321; final SolrException thrown = - expectThrows( - SolrException.class, - () -> { - CreateCollection.validateRequestBody(requestBody); - }); + expectThrows(SolrException.class, () -> CreateCollection.validateRequestBody(requestBody)); assertEquals(400, thrown.code()); assertEquals( @@ -90,11 +101,7 @@ public void testReportsErrorIfCollectionNameInvalid() { requestBody.config = "someConfig"; final SolrException thrown = - expectThrows( - SolrException.class, - () -> { - CreateCollection.validateRequestBody(requestBody); - }); + expectThrows(SolrException.class, () -> CreateCollection.validateRequestBody(requestBody)); assertEquals(400, thrown.code()); assertTrue( @@ -112,11 +119,7 @@ public void testReportsErrorIfShardNamesInvalid() { requestBody.shardNames = List.of("good-name", "bad;name"); final SolrException thrown = - expectThrows( - SolrException.class, - () -> { - CreateCollection.validateRequestBody(requestBody); - }); + expectThrows(SolrException.class, () -> CreateCollection.validateRequestBody(requestBody)); assertEquals(400, thrown.code()); assertTrue( @@ -125,7 +128,7 @@ public void testReportsErrorIfShardNamesInvalid() { } @Test - public void testCreateRemoteMessageAllProperties() { + public void testCreateRemoteMessageAllProperties() throws Exception { final var requestBody = new CreateCollectionRequestBody(); requestBody.name = "someName"; requestBody.replicationFactor = 123; @@ -145,9 +148,13 @@ public void testCreateRemoteMessageAllProperties() { requestBody.nodeSet = List.of("node1", "node2"); requestBody.shuffleNodes = false; - final var remoteMessage = CreateCollection.createRemoteMessage(requestBody).getProperties(); + api.createCollection(requestBody); + verify(mockCommandRunner) + .runCollectionCommand(contextCapturer.capture(), messageCapturer.capture(), anyLong()); - assertEquals("create", remoteMessage.get(QUEUE_OPERATION)); + final ZkNodeProps createdMessage = messageCapturer.getValue(); + final Map remoteMessage = createdMessage.getProperties(); + assertEquals(17, remoteMessage.size()); assertEquals("true", remoteMessage.get("fromApi")); assertEquals("someName", remoteMessage.get(NAME)); assertEquals(123, remoteMessage.get(REPLICATION_FACTOR)); @@ -161,24 +168,36 @@ public void testCreateRemoteMessageAllProperties() { assertEquals(true, remoteMessage.get(PER_REPLICA_STATE)); assertEquals("someAliasName", remoteMessage.get(ALIAS)); assertEquals("propValue", remoteMessage.get("property.propName")); - assertEquals("someAsyncId", remoteMessage.get(ASYNC)); assertEquals("someRouterName", remoteMessage.get("router.name")); assertEquals("someField", remoteMessage.get("router.field")); assertEquals("node1,node2", remoteMessage.get(CREATE_NODE_SET)); assertEquals(false, remoteMessage.get(CREATE_NODE_SET_SHUFFLE_PARAM)); + + final AdminCmdContext context = contextCapturer.getValue(); + assertEquals(CollectionParams.CollectionAction.CREATE, context.getAction()); + assertEquals("someAsyncId", context.getAsyncId()); } @Test - public void testNoReplicaCreationMessage() { + public void testNoReplicaCreationMessage() throws Exception { final var requestBody = new CreateCollectionRequestBody(); requestBody.name = "someName"; requestBody.createReplicas = false; + requestBody.async = "someAsyncId"; - final var remoteMessage = CreateCollection.createRemoteMessage(requestBody).getProperties(); + api.createCollection(requestBody); + verify(mockCommandRunner) + .runCollectionCommand(contextCapturer.capture(), messageCapturer.capture(), anyLong()); + + final ZkNodeProps createdMessage = messageCapturer.getValue(); + final Map remoteMessage = createdMessage.getProperties(); - assertEquals("create", remoteMessage.get(QUEUE_OPERATION)); assertEquals("someName", remoteMessage.get(NAME)); assertEquals("EMPTY", remoteMessage.get(CREATE_NODE_SET)); + + final AdminCmdContext context = contextCapturer.getValue(); + assertEquals(CollectionParams.CollectionAction.CREATE, context.getAction()); + assertEquals("someAsyncId", context.getAsyncId()); } @Test diff --git a/solr/core/src/test/org/apache/solr/handler/admin/api/CreateCollectionSnapshotAPITest.java b/solr/core/src/test/org/apache/solr/handler/admin/api/CreateCollectionSnapshotAPITest.java index 27efa7bc0ef..f3e1e8f8fdb 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/api/CreateCollectionSnapshotAPITest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/api/CreateCollectionSnapshotAPITest.java @@ -16,48 +16,79 @@ */ package org.apache.solr.handler.admin.api; -import static org.apache.solr.cloud.Overseer.QUEUE_OPERATION; import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP; import static org.apache.solr.common.params.CollectionAdminParams.FOLLOW_ALIASES; -import static org.apache.solr.common.params.CommonAdminParams.ASYNC; import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; import java.util.Map; -import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.client.api.model.CreateCollectionSnapshotRequestBody; +import org.apache.solr.cloud.api.collections.AdminCmdContext; import org.apache.solr.common.cloud.ZkNodeProps; +import org.apache.solr.common.params.CollectionParams; import org.apache.solr.common.params.CoreAdminParams; +import org.junit.Before; import org.junit.Test; +import org.mockito.Mockito; -public class CreateCollectionSnapshotAPITest extends SolrTestCaseJ4 { +public class CreateCollectionSnapshotAPITest extends MockAPITest { + + private CreateCollectionSnapshot api; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + when(mockCoreContainer.isZooKeeperAware()).thenReturn(true); + + api = new CreateCollectionSnapshot(mockCoreContainer, mockQueryRequest, queryResponse); + } @Test - public void testConstructsValidOverseerMessage() { - final ZkNodeProps messageOne = - CreateCollectionSnapshot.createRemoteMessage("myCollName", false, "mySnapshotName", null); + public void testConstructsValidOverseerMessage() throws Exception { + when(mockClusterState.hasCollection("myCollName")).thenReturn(true); + when(mockSolrZkClient.exists(anyString())).thenReturn(false); + + CreateCollectionSnapshotRequestBody body = new CreateCollectionSnapshotRequestBody(); + body.followAliases = false; + api.createCollectionSnapshot("myCollName", "mySnapshotName", body); + verify(mockCommandRunner) + .runCollectionCommand(contextCapturer.capture(), messageCapturer.capture(), anyLong()); + final ZkNodeProps messageOne = messageCapturer.getValue(); final Map rawMessageOne = messageOne.getProperties(); - assertEquals(4, rawMessageOne.size()); + assertEquals(3, rawMessageOne.size()); assertThat( rawMessageOne.keySet(), - containsInAnyOrder( - QUEUE_OPERATION, COLLECTION_PROP, CoreAdminParams.COMMIT_NAME, FOLLOW_ALIASES)); - assertEquals("createsnapshot", rawMessageOne.get(QUEUE_OPERATION)); + containsInAnyOrder(COLLECTION_PROP, CoreAdminParams.COMMIT_NAME, FOLLOW_ALIASES)); assertEquals("myCollName", rawMessageOne.get(COLLECTION_PROP)); assertEquals("mySnapshotName", rawMessageOne.get(CoreAdminParams.COMMIT_NAME)); assertEquals(false, rawMessageOne.get(FOLLOW_ALIASES)); - final ZkNodeProps messageTwo = - CreateCollectionSnapshot.createRemoteMessage( - "myCollName", true, "mySnapshotName", "myAsyncId"); + AdminCmdContext context = contextCapturer.getValue(); + assertEquals(CollectionParams.CollectionAction.CREATESNAPSHOT, context.getAction()); + assertNull(context.getAsyncId()); + + body.followAliases = true; + body.async = "testId"; + Mockito.clearInvocations(mockCommandRunner); + api.createCollectionSnapshot("myCollName", "mySnapshotName", body); + verify(mockCommandRunner) + .runCollectionCommand(contextCapturer.capture(), messageCapturer.capture(), anyLong()); + final ZkNodeProps messageTwo = messageCapturer.getValue(); final Map rawMessageTwo = messageTwo.getProperties(); - assertEquals(5, rawMessageTwo.size()); + assertEquals(3, rawMessageTwo.size()); assertThat( rawMessageTwo.keySet(), - containsInAnyOrder( - QUEUE_OPERATION, COLLECTION_PROP, CoreAdminParams.COMMIT_NAME, FOLLOW_ALIASES, ASYNC)); - assertEquals("createsnapshot", rawMessageTwo.get(QUEUE_OPERATION)); + containsInAnyOrder(COLLECTION_PROP, CoreAdminParams.COMMIT_NAME, FOLLOW_ALIASES)); assertEquals("myCollName", rawMessageTwo.get(COLLECTION_PROP)); assertEquals("mySnapshotName", rawMessageTwo.get(CoreAdminParams.COMMIT_NAME)); assertEquals(true, rawMessageTwo.get(FOLLOW_ALIASES)); - assertEquals("myAsyncId", rawMessageTwo.get(ASYNC)); + + context = contextCapturer.getValue(); + assertEquals(CollectionParams.CollectionAction.CREATESNAPSHOT, context.getAction()); + assertEquals("testId", context.getAsyncId()); } } diff --git a/solr/core/src/test/org/apache/solr/handler/admin/api/CreateReplicaAPITest.java b/solr/core/src/test/org/apache/solr/handler/admin/api/CreateReplicaAPITest.java index 2abe5d13397..eebf54dc464 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/api/CreateReplicaAPITest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/api/CreateReplicaAPITest.java @@ -17,7 +17,6 @@ package org.apache.solr.handler.admin.api; -import static org.apache.solr.cloud.Overseer.QUEUE_OPERATION; import static org.apache.solr.common.cloud.ZkStateReader.NRT_REPLICAS; import static org.apache.solr.common.cloud.ZkStateReader.PULL_REPLICAS; import static org.apache.solr.common.cloud.ZkStateReader.REPLICA_TYPE; @@ -35,26 +34,41 @@ import static org.apache.solr.common.params.CoreAdminParams.NODE; import static org.apache.solr.common.params.CoreAdminParams.ULOG_DIR; import static org.apache.solr.common.params.ShardParams._ROUTE_; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; import java.util.List; import java.util.Map; -import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.client.api.model.CreateReplicaRequestBody; +import org.apache.solr.cloud.api.collections.AdminCmdContext; import org.apache.solr.common.SolrException; +import org.apache.solr.common.cloud.ZkNodeProps; +import org.apache.solr.common.params.CollectionParams; import org.apache.solr.common.params.ModifiableSolrParams; +import org.junit.Before; import org.junit.Test; /** Unit tests for {@link CreateReplica} */ -public class CreateReplicaAPITest extends SolrTestCaseJ4 { +public class CreateReplicaAPITest extends MockAPITest { + + private CreateReplica api; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + when(mockCoreContainer.isZooKeeperAware()).thenReturn(true); + + api = new CreateReplica(mockCoreContainer, mockQueryRequest, queryResponse); + } + @Test public void testReportsErrorIfRequestBodyMissing() { final SolrException thrown = expectThrows( - SolrException.class, - () -> { - final var api = new CreateReplica(null, null, null); - api.createReplica("someCollName", "someShardName", null); - }); + SolrException.class, () -> api.createReplica("someCollName", "someShardName", null)); assertEquals(400, thrown.code()); assertEquals("Required request-body is missing", thrown.getMessage()); @@ -64,12 +78,7 @@ public void testReportsErrorIfRequestBodyMissing() { public void testReportsErrorIfCollectionNameMissing() { final var requestBody = new CreateReplicaRequestBody(); final SolrException thrown = - expectThrows( - SolrException.class, - () -> { - final var api = new CreateReplica(null, null, null); - api.createReplica(null, "shardName", requestBody); - }); + expectThrows(SolrException.class, () -> api.createReplica(null, "shardName", requestBody)); assertEquals(400, thrown.code()); assertEquals("Missing required parameter: collection", thrown.getMessage()); @@ -80,18 +89,14 @@ public void testReportsErrorIfShardNameMissing() { final var requestBody = new CreateReplicaRequestBody(); final SolrException thrown = expectThrows( - SolrException.class, - () -> { - final var api = new CreateReplica(null, null, null); - api.createReplica("someCollectionName", null, requestBody); - }); + SolrException.class, () -> api.createReplica("someCollectionName", null, requestBody)); assertEquals(400, thrown.code()); assertEquals("Missing required parameter: shard", thrown.getMessage()); } @Test - public void testCreateRemoteMessageAllProperties() { + public void testCreateRemoteMessageAllProperties() throws Exception { final var requestBody = new CreateReplicaRequestBody(); requestBody.name = "someName"; requestBody.type = "NRT"; @@ -110,12 +115,14 @@ public void testCreateRemoteMessageAllProperties() { requestBody.async = "someAsyncId"; requestBody.properties = Map.of("propName1", "propVal1", "propName2", "propVal2"); - final var remoteMessage = - CreateReplica.createRemoteMessage("someCollectionName", "someShardName", requestBody) - .getProperties(); + when(mockClusterState.hasCollection(eq("someCollectionName"))).thenReturn(true); + api.createReplica("someCollectionName", "someShardName", requestBody); + verify(mockCommandRunner) + .runCollectionCommand(contextCapturer.capture(), messageCapturer.capture(), anyLong()); - assertEquals(20, remoteMessage.size()); - assertEquals("addreplica", remoteMessage.get(QUEUE_OPERATION)); + final ZkNodeProps createdMessage = messageCapturer.getValue(); + final Map remoteMessage = createdMessage.getProperties(); + assertEquals(18, remoteMessage.size()); assertEquals("someCollectionName", remoteMessage.get(COLLECTION)); assertEquals("someShardName", remoteMessage.get(SHARD_ID_PROP)); assertEquals("someName", remoteMessage.get(NAME)); @@ -132,9 +139,12 @@ public void testCreateRemoteMessageAllProperties() { assertEquals(Boolean.TRUE, remoteMessage.get(SKIP_NODE_ASSIGNMENT)); assertEquals(true, remoteMessage.get(WAIT_FOR_FINAL_STATE)); assertEquals(true, remoteMessage.get(FOLLOW_ALIASES)); - assertEquals("someAsyncId", remoteMessage.get(ASYNC)); assertEquals("propVal1", remoteMessage.get("property.propName1")); assertEquals("propVal2", remoteMessage.get("property.propName2")); + + final AdminCmdContext context = contextCapturer.getValue(); + assertEquals(CollectionParams.CollectionAction.ADDREPLICA, context.getAction()); + assertEquals("someAsyncId", context.getAsyncId()); } @Test diff --git a/solr/core/src/test/org/apache/solr/handler/admin/api/CreateShardAPITest.java b/solr/core/src/test/org/apache/solr/handler/admin/api/CreateShardAPITest.java index a8bc3edf658..b17a3ee59ed 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/api/CreateShardAPITest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/api/CreateShardAPITest.java @@ -17,7 +17,6 @@ package org.apache.solr.handler.admin.api; -import static org.apache.solr.cloud.Overseer.QUEUE_OPERATION; import static org.apache.solr.common.cloud.ZkStateReader.NRT_REPLICAS; import static org.apache.solr.common.cloud.ZkStateReader.PULL_REPLICAS; import static org.apache.solr.common.cloud.ZkStateReader.REPLICATION_FACTOR; @@ -29,27 +28,44 @@ import static org.apache.solr.common.params.CommonAdminParams.ASYNC; import static org.apache.solr.common.params.CommonAdminParams.WAIT_FOR_FINAL_STATE; import static org.hamcrest.Matchers.containsString; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; import java.util.List; import java.util.Map; -import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.client.api.model.CreateShardRequestBody; +import org.apache.solr.cloud.api.collections.AdminCmdContext; import org.apache.solr.common.SolrException; +import org.apache.solr.common.cloud.DocCollection; +import org.apache.solr.common.cloud.ImplicitDocRouter; +import org.apache.solr.common.cloud.ZkNodeProps; +import org.apache.solr.common.params.CollectionParams; +import org.apache.solr.common.params.CommonParams; import org.apache.solr.common.params.ModifiableSolrParams; +import org.junit.Before; import org.junit.Test; /** Unit tests for {@link CreateShard} */ -public class CreateShardAPITest extends SolrTestCaseJ4 { +public class CreateShardAPITest extends MockAPITest { + + private CreateShard api; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + when(mockCoreContainer.isZooKeeperAware()).thenReturn(true); + + api = new CreateShard(mockCoreContainer, mockQueryRequest, queryResponse); + } @Test public void testReportsErrorIfRequestBodyMissing() { final SolrException thrown = - expectThrows( - SolrException.class, - () -> { - final var api = new CreateShard(null, null, null); - api.createShard("someCollName", null); - }); + expectThrows(SolrException.class, () -> api.createShard("someCollName", null)); assertEquals(400, thrown.code()); assertEquals("Required request-body is missing", thrown.getMessage()); @@ -60,12 +76,7 @@ public void testReportsErrorIfCollectionNameMissing() { final var requestBody = new CreateShardRequestBody(); requestBody.shardName = "someShardName"; final SolrException thrown = - expectThrows( - SolrException.class, - () -> { - final var api = new CreateShard(null, null, null); - api.createShard(null, requestBody); - }); + expectThrows(SolrException.class, () -> api.createShard(null, requestBody)); assertEquals(400, thrown.code()); assertEquals("Missing required parameter: collection", thrown.getMessage()); @@ -76,12 +87,7 @@ public void testReportsErrorIfShardNameMissing() { final var requestBody = new CreateShardRequestBody(); requestBody.shardName = null; final SolrException thrown = - expectThrows( - SolrException.class, - () -> { - final var api = new CreateShard(null, null, null); - api.createShard("someCollectionName", requestBody); - }); + expectThrows(SolrException.class, () -> api.createShard("someCollectionName", requestBody)); assertEquals(400, thrown.code()); assertEquals("Missing required parameter: shard", thrown.getMessage()); @@ -92,19 +98,14 @@ public void testReportsErrorIfShardNameIsInvalid() { final var requestBody = new CreateShardRequestBody(); requestBody.shardName = "invalid$shard@name"; final SolrException thrown = - expectThrows( - SolrException.class, - () -> { - final var api = new CreateShard(null, null, null); - api.createShard("someCollectionName", requestBody); - }); + expectThrows(SolrException.class, () -> api.createShard("someCollectionName", requestBody)); assertEquals(400, thrown.code()); assertThat(thrown.getMessage(), containsString("Invalid shard: [invalid$shard@name]")); } @Test - public void testCreateRemoteMessageAllProperties() { + public void testCreateRemoteMessageAllProperties() throws Exception { final var requestBody = new CreateShardRequestBody(); requestBody.shardName = "someShardName"; requestBody.replicationFactor = 123; @@ -118,11 +119,20 @@ public void testCreateRemoteMessageAllProperties() { requestBody.async = "someAsyncId"; requestBody.properties = Map.of("propName1", "propVal1", "propName2", "propVal2"); - final var remoteMessage = - CreateShard.createRemoteMessage("someCollectionName", requestBody).getProperties(); + DocCollection mockCollection = mock(DocCollection.class); + when(mockClusterState.hasCollection(eq("someCollectionName"))).thenReturn(true); + when(mockClusterState.getCollection(eq("someCollectionName"))).thenReturn(mockCollection); + when(mockCollection.get(DocCollection.CollectionStateProps.DOC_ROUTER)) + .thenReturn(Map.of(CommonParams.NAME, ImplicitDocRouter.NAME)); - assertEquals(13, remoteMessage.size()); - assertEquals("createshard", remoteMessage.get(QUEUE_OPERATION)); + api.createShard("someCollectionName", requestBody); + verify(mockCommandRunner) + .runCollectionCommand(contextCapturer.capture(), messageCapturer.capture(), anyLong()); + + final ZkNodeProps createdMessage = messageCapturer.getValue(); + final Map remoteMessage = createdMessage.getProperties(); + + assertEquals(11, remoteMessage.size()); assertEquals("someCollectionName", remoteMessage.get(COLLECTION)); assertEquals("someShardName", remoteMessage.get(SHARD_ID_PROP)); assertEquals(123, remoteMessage.get(REPLICATION_FACTOR)); @@ -132,9 +142,12 @@ public void testCreateRemoteMessageAllProperties() { assertEquals("node1,node2", remoteMessage.get(CREATE_NODE_SET_PARAM)); assertEquals(true, remoteMessage.get(WAIT_FOR_FINAL_STATE)); assertEquals(true, remoteMessage.get(FOLLOW_ALIASES)); - assertEquals("someAsyncId", remoteMessage.get(ASYNC)); assertEquals("propVal1", remoteMessage.get("property.propName1")); assertEquals("propVal2", remoteMessage.get("property.propName2")); + + final AdminCmdContext context = contextCapturer.getValue(); + assertEquals(CollectionParams.CollectionAction.CREATESHARD, context.getAction()); + assertEquals("someAsyncId", context.getAsyncId()); } @Test diff --git a/solr/core/src/test/org/apache/solr/handler/admin/api/DeleteAliasAPITest.java b/solr/core/src/test/org/apache/solr/handler/admin/api/DeleteAliasAPITest.java index dc93c29b487..fc50d5b33e5 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/api/DeleteAliasAPITest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/api/DeleteAliasAPITest.java @@ -17,28 +17,109 @@ package org.apache.solr.handler.admin.api; -import static org.apache.solr.cloud.Overseer.QUEUE_OPERATION; -import static org.apache.solr.common.params.CommonAdminParams.ASYNC; -import static org.apache.solr.common.params.CoreAdminParams.NAME; - -import java.util.Map; -import org.apache.solr.SolrTestCaseJ4; +import java.lang.invoke.MethodHandles; +import java.time.Duration; +import org.apache.solr.client.api.model.SubResponseAccumulatingJerseyResponse; +import org.apache.solr.client.solrj.impl.CloudSolrClient; +import org.apache.solr.client.solrj.impl.ZkClientClusterStateProvider; +import org.apache.solr.client.solrj.request.AliasesApi; +import org.apache.solr.client.solrj.request.CollectionAdminRequest; +import org.apache.solr.client.solrj.response.RequestStatusState; +import org.apache.solr.cloud.SolrCloudTestCase; +import org.apache.solr.common.cloud.ZkStateReader; +import org.junit.BeforeClass; import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** Integration tests for {@link DeleteAlias} using the V2 API */ +public class DeleteAliasAPITest extends SolrCloudTestCase { + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + @BeforeClass + public static void setupCluster() throws Exception { + configureCluster(1) + .addConfig( + "conf1", TEST_PATH().resolve("configsets").resolve("cloud-dynamic").resolve("conf")) + .configure(); + } + + @Test + public void testDeleteAlias() throws Exception { + CloudSolrClient cloudClient = cluster.getSolrClient(); + String collectionName = "deletealiastest_coll"; + String aliasName = "deletealiastest_alias"; + + // Create a collection + CollectionAdminRequest.createCollection(collectionName, "conf1", 1, 1).process(cloudClient); + cluster.waitForActiveCollection(collectionName, 1, 1); + + // Create an alias pointing to the collection + CollectionAdminRequest.createAlias(aliasName, collectionName).process(cloudClient); + + // Verify the alias exists + var clusterStateProvider = cloudClient.getClusterStateProvider(); + assertEquals( + "Alias should exist before deletion", + collectionName, + clusterStateProvider.resolveSimpleAlias(aliasName)); -/** Unit tests for {@link DeleteAlias}. */ -public class DeleteAliasAPITest extends SolrTestCaseJ4 { + // Delete the alias using the V2 API + var request = new AliasesApi.DeleteAlias(aliasName); + SubResponseAccumulatingJerseyResponse response = request.process(cloudClient); + assertNotNull(response); + assertNull("Expected request to not fail", response.failedSubResponsesByNodeName); + + // Verify the alias is gone + ZkStateReader.AliasesManager aliasesManager = + ((ZkClientClusterStateProvider) clusterStateProvider) + .getZkStateReader() + .getAliasesManager(); + aliasesManager.update(); + assertFalse( + "Alias should not exist after deletion", aliasesManager.getAliases().hasAlias(aliasName)); + } @Test - public void testConstructsValidRemoteMessage() { - Map props = DeleteAlias.createRemoteMessage("aliasName", null).getProperties(); - assertEquals(2, props.size()); - assertEquals("deletealias", props.get(QUEUE_OPERATION)); - assertEquals("aliasName", props.get(NAME)); - - props = DeleteAlias.createRemoteMessage("aliasName", "asyncId").getProperties(); - assertEquals(3, props.size()); - assertEquals("deletealias", props.get(QUEUE_OPERATION)); - assertEquals("aliasName", props.get(NAME)); - assertEquals("asyncId", props.get(ASYNC)); + public void testDeleteAliasAsync() throws Exception { + CloudSolrClient cloudClient = cluster.getSolrClient(); + String collectionName = "deletealiastest_coll_async"; + String aliasName = "deletealiastest_alias_async"; + + // Create a collection + CollectionAdminRequest.createCollection(collectionName, "conf1", 1, 1).process(cloudClient); + cluster.waitForActiveCollection(collectionName, 1, 1); + + // Create an alias pointing to the collection + CollectionAdminRequest.createAlias(aliasName, collectionName).process(cloudClient); + + // Verify the alias exists + var clusterStateProvider = cloudClient.getClusterStateProvider(); + + // Delete the alias using the V2 API with async + String asyncId = "deleteAlias001"; + var request = new AliasesApi.DeleteAlias(aliasName); + request.setAsync(asyncId); + var response = request.process(cloudClient); + assertNotNull(response); + assertNull("Expected request start to not fail", response.failedSubResponsesByNodeName); + + // Wait for the async request to complete + CollectionAdminRequest.RequestStatusResponse rsp = + waitForAsyncClusterRequest(asyncId, Duration.ofSeconds(5)); + + assertEquals( + "Expected async request to complete successfully", + RequestStatusState.COMPLETED, + rsp.getRequestStatus()); + + // Verify the alias is gone + ZkStateReader.AliasesManager aliasesManager = + ((ZkClientClusterStateProvider) clusterStateProvider) + .getZkStateReader() + .getAliasesManager(); + aliasesManager.update(); + assertFalse( + "Alias should not exist after deletion", aliasesManager.getAliases().hasAlias(aliasName)); } } diff --git a/solr/core/src/test/org/apache/solr/handler/admin/api/DeleteCollectionAPITest.java b/solr/core/src/test/org/apache/solr/handler/admin/api/DeleteCollectionAPITest.java index fafc0a67c00..e4a2109c6fc 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/api/DeleteCollectionAPITest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/api/DeleteCollectionAPITest.java @@ -17,44 +17,69 @@ package org.apache.solr.handler.admin.api; -import static org.apache.solr.cloud.Overseer.QUEUE_OPERATION; import static org.apache.solr.common.params.CollectionAdminParams.FOLLOW_ALIASES; import static org.apache.solr.common.params.CollectionParams.NAME; -import static org.apache.solr.common.params.CommonAdminParams.ASYNC; -import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.hasEntry; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; import java.util.Map; -import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.cloud.api.collections.AdminCmdContext; import org.apache.solr.common.cloud.ZkNodeProps; +import org.apache.solr.common.params.CollectionParams; +import org.junit.Before; import org.junit.Test; +import org.mockito.Mockito; /** Unit tests for {@link DeleteCollection} */ -public class DeleteCollectionAPITest extends SolrTestCaseJ4 { +public class DeleteCollectionAPITest extends MockAPITest { + + private DeleteCollection api; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + when(mockCoreContainer.isZooKeeperAware()).thenReturn(true); + + api = new DeleteCollection(mockCoreContainer, mockQueryRequest, queryResponse); + } @Test - public void testConstructsValidOverseerMessage() { + public void testConstructsValidOverseerMessage() throws Exception { // Only required properties provided { - final ZkNodeProps message = DeleteCollection.createRemoteMessage("someCollName", null, null); + api.deleteCollection("someCollName", null, null); + verify(mockCommandRunner) + .runCollectionCommand(contextCapturer.capture(), messageCapturer.capture(), anyLong()); + + final ZkNodeProps message = messageCapturer.getValue(); final Map rawMessage = message.getProperties(); - assertEquals(2, rawMessage.size()); - assertThat(rawMessage.keySet(), containsInAnyOrder(QUEUE_OPERATION, NAME)); - assertEquals("delete", rawMessage.get(QUEUE_OPERATION)); - assertEquals("someCollName", rawMessage.get(NAME)); + assertEquals(1, rawMessage.size()); + assertThat(rawMessage, hasEntry(NAME, "someCollName")); + + AdminCmdContext context = contextCapturer.getValue(); + assertEquals(CollectionParams.CollectionAction.DELETE, context.getAction()); + assertNull(context.getAsyncId()); } - // Optional properties ('followAliases' and 'async') also provided + // Optional property 'followAliases' also provided { - final ZkNodeProps message = - DeleteCollection.createRemoteMessage("someCollName", Boolean.TRUE, "someAsyncId"); + Mockito.clearInvocations(mockCommandRunner); + api.deleteCollection("someCollName", true, "test"); + verify(mockCommandRunner) + .runCollectionCommand(contextCapturer.capture(), messageCapturer.capture(), anyLong()); + + final ZkNodeProps message = messageCapturer.getValue(); final Map rawMessage = message.getProperties(); - assertEquals(4, rawMessage.size()); - assertThat( - rawMessage.keySet(), containsInAnyOrder(QUEUE_OPERATION, NAME, ASYNC, FOLLOW_ALIASES)); - assertEquals("delete", rawMessage.get(QUEUE_OPERATION)); - assertEquals("someCollName", rawMessage.get(NAME)); - assertEquals(Boolean.TRUE, rawMessage.get(FOLLOW_ALIASES)); - assertEquals("someAsyncId", rawMessage.get(ASYNC)); + assertEquals(2, rawMessage.size()); + assertThat(rawMessage, hasEntry(NAME, "someCollName")); + assertThat(rawMessage, hasEntry(FOLLOW_ALIASES, Boolean.TRUE)); + + AdminCmdContext context = contextCapturer.getValue(); + assertEquals(CollectionParams.CollectionAction.DELETE, context.getAction()); + assertEquals("test", context.getAsyncId()); } } } diff --git a/solr/core/src/test/org/apache/solr/handler/admin/api/DeleteCollectionBackupAPITest.java b/solr/core/src/test/org/apache/solr/handler/admin/api/DeleteCollectionBackupAPITest.java index 7d083d1c0cf..f6a12d13752 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/api/DeleteCollectionBackupAPITest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/api/DeleteCollectionBackupAPITest.java @@ -17,22 +17,53 @@ package org.apache.solr.handler.admin.api; -import static org.apache.solr.cloud.Overseer.QUEUE_OPERATION; -import static org.apache.solr.common.params.CommonAdminParams.ASYNC; import static org.apache.solr.common.params.CommonParams.NAME; import static org.apache.solr.common.params.CoreAdminParams.BACKUP_ID; import static org.apache.solr.common.params.CoreAdminParams.BACKUP_LOCATION; import static org.apache.solr.common.params.CoreAdminParams.BACKUP_PURGE_UNUSED; import static org.apache.solr.common.params.CoreAdminParams.BACKUP_REPOSITORY; import static org.apache.solr.common.params.CoreAdminParams.MAX_NUM_BACKUP_POINTS; - -import org.apache.solr.SolrTestCaseJ4; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.net.URI; +import java.util.Map; import org.apache.solr.client.api.model.PurgeUnusedFilesRequestBody; +import org.apache.solr.cloud.api.collections.AdminCmdContext; import org.apache.solr.common.SolrException; +import org.apache.solr.common.cloud.ZkNodeProps; +import org.apache.solr.common.params.CollectionParams; +import org.apache.solr.core.backup.repository.BackupRepository; +import org.junit.Before; import org.junit.Test; /** Unit tests for {@link DeleteCollectionBackup} */ -public class DeleteCollectionBackupAPITest extends SolrTestCaseJ4 { +public class DeleteCollectionBackupAPITest extends MockAPITest { + + private DeleteCollectionBackup api; + private BackupRepository mockBackupRepository; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + when(mockCoreContainer.isZooKeeperAware()).thenReturn(true); + + mockBackupRepository = mock(BackupRepository.class); + when(mockCoreContainer.newBackupRepository(eq("someRepository"))) + .thenReturn(mockBackupRepository); + when(mockCoreContainer.newBackupRepository(null)).thenReturn(mockBackupRepository); + when(mockBackupRepository.getBackupLocation(eq("someLocation"))).thenReturn("someLocation"); + URI uri = new URI("someLocation"); + when(mockBackupRepository.createDirectoryURI(eq("someLocation"))).thenReturn(uri); + when(mockBackupRepository.exists(eq(uri))).thenReturn(true); + + api = new DeleteCollectionBackup(mockCoreContainer, mockQueryRequest, queryResponse); + } + @Test public void testReportsErrorIfBackupNameMissing() { // Single delete @@ -40,11 +71,9 @@ public void testReportsErrorIfBackupNameMissing() { final SolrException thrown = expectThrows( SolrException.class, - () -> { - final var api = new DeleteCollectionBackup(null, null, null); - api.deleteSingleBackupById( - null, "someBackupId", "someLocation", "someRepository", "someAsyncId"); - }); + () -> + api.deleteSingleBackupById( + null, "someBackupId", "someLocation", "someRepository", "someAsyncId")); assertEquals(400, thrown.code()); assertEquals("Missing required parameter: name", thrown.getMessage()); @@ -55,11 +84,9 @@ public void testReportsErrorIfBackupNameMissing() { final SolrException thrown = expectThrows( SolrException.class, - () -> { - final var api = new DeleteCollectionBackup(null, null, null); - api.deleteMultipleBackupsByRecency( - null, 123, "someLocation", "someRepository", "someAsyncId"); - }); + () -> + api.deleteMultipleBackupsByRecency( + null, 123, "someLocation", "someRepository", "someAsyncId")); assertEquals(400, thrown.code()); assertEquals("Missing required parameter: name", thrown.getMessage()); @@ -73,11 +100,7 @@ public void testReportsErrorIfBackupNameMissing() { requestBody.async = "someAsyncId"; final SolrException thrown = expectThrows( - SolrException.class, - () -> { - final var api = new DeleteCollectionBackup(null, null, null); - api.garbageCollectUnusedBackupFiles(null, requestBody); - }); + SolrException.class, () -> api.garbageCollectUnusedBackupFiles(null, requestBody)); assertEquals(400, thrown.code()); assertEquals("Missing required parameter: name", thrown.getMessage()); @@ -89,11 +112,9 @@ public void testDeletionByIdReportsErrorIfIdMissing() { final SolrException thrown = expectThrows( SolrException.class, - () -> { - final var api = new DeleteCollectionBackup(null, null, null); - api.deleteSingleBackupById( - "someBackupName", null, "someLocation", "someRepository", "someAsyncId"); - }); + () -> + api.deleteSingleBackupById( + "someBackupName", null, "someLocation", "someRepository", "someAsyncId")); assertEquals(400, thrown.code()); assertEquals("Missing required parameter: backupId", thrown.getMessage()); @@ -104,11 +125,9 @@ public void testMultiVersionDeletionReportsErrorIfRetainParamMissing() { final SolrException thrown = expectThrows( SolrException.class, - () -> { - final var api = new DeleteCollectionBackup(null, null, null); - api.deleteMultipleBackupsByRecency( - "someBackupName", null, "someLocation", "someRepository", "someAsyncId"); - }); + () -> + api.deleteMultipleBackupsByRecency( + "someBackupName", null, "someLocation", "someRepository", "someAsyncId")); assertEquals(400, thrown.code()); assertEquals("Missing required parameter: retainLatest", thrown.getMessage()); @@ -117,39 +136,63 @@ public void testMultiVersionDeletionReportsErrorIfRetainParamMissing() { // The message created in this test isn't valid in practice, since it contains mutually-exclusive // parameters, but that doesn't matter for the purposes of this test. @Test - public void testCreateRemoteMessageAllParams() { - final var remoteMessage = - DeleteCollectionBackup.createRemoteMessage( - "someBackupName", - "someBackupId", - 123, - true, - "someLocation", - "someRepository", - "someAsyncId") - .getProperties(); - - assertEquals(8, remoteMessage.size()); - assertEquals("deletebackup", remoteMessage.get(QUEUE_OPERATION)); + public void testCreateRemoteMessageSingle() throws Exception { + api.deleteSingleBackupById( + "someBackupName", "someBackupId", "someLocation", "someRepository", "someAsyncId"); + verify(mockCommandRunner) + .runCollectionCommand(contextCapturer.capture(), messageCapturer.capture(), anyLong()); + + final ZkNodeProps createdMessage = messageCapturer.getValue(); + final Map remoteMessage = createdMessage.getProperties(); + assertEquals(4, remoteMessage.size()); assertEquals("someBackupName", remoteMessage.get(NAME)); assertEquals("someBackupId", remoteMessage.get(BACKUP_ID)); - assertEquals(Integer.valueOf(123), remoteMessage.get(MAX_NUM_BACKUP_POINTS)); - assertEquals(Boolean.TRUE, remoteMessage.get(BACKUP_PURGE_UNUSED)); assertEquals("someLocation", remoteMessage.get(BACKUP_LOCATION)); assertEquals("someRepository", remoteMessage.get(BACKUP_REPOSITORY)); - assertEquals("someAsyncId", remoteMessage.get(ASYNC)); + + final AdminCmdContext context = contextCapturer.getValue(); + assertEquals(CollectionParams.CollectionAction.DELETEBACKUP, context.getAction()); + assertEquals("someAsyncId", context.getAsyncId()); } @Test - public void testCreateRemoteMessageOnlyRequiredParams() { - final var remoteMessage = - DeleteCollectionBackup.createRemoteMessage( - "someBackupName", "someBackupId", null, null, null, null, null) - .getProperties(); - - assertEquals(3, remoteMessage.size()); - assertEquals("deletebackup", remoteMessage.get(QUEUE_OPERATION)); + public void testCreateRemoteMessageMultiple() throws Exception { + api.deleteMultipleBackupsByRecency("someBackupName", 2, "someLocation", "someRepository", null); + verify(mockCommandRunner) + .runCollectionCommand(contextCapturer.capture(), messageCapturer.capture(), anyLong()); + + final ZkNodeProps createdMessage = messageCapturer.getValue(); + final Map remoteMessage = createdMessage.getProperties(); + assertEquals(4, remoteMessage.size()); assertEquals("someBackupName", remoteMessage.get(NAME)); - assertEquals("someBackupId", remoteMessage.get(BACKUP_ID)); + assertEquals(2, remoteMessage.get(MAX_NUM_BACKUP_POINTS)); + assertEquals("someLocation", remoteMessage.get(BACKUP_LOCATION)); + assertEquals("someRepository", remoteMessage.get(BACKUP_REPOSITORY)); + + final AdminCmdContext context = contextCapturer.getValue(); + assertEquals(CollectionParams.CollectionAction.DELETEBACKUP, context.getAction()); + assertNull(context.getAsyncId()); + } + + @Test + public void testCreateRemoteMessageGarbageCollect() throws Exception { + PurgeUnusedFilesRequestBody body = new PurgeUnusedFilesRequestBody(); + body.location = "someLocation"; + body.repositoryName = "someRepository"; + api.garbageCollectUnusedBackupFiles("someBackupName", body); + verify(mockCommandRunner) + .runCollectionCommand(contextCapturer.capture(), messageCapturer.capture(), anyLong()); + + final ZkNodeProps createdMessage = messageCapturer.getValue(); + final Map remoteMessage = createdMessage.getProperties(); + assertEquals(4, remoteMessage.size()); + assertEquals("someBackupName", remoteMessage.get(NAME)); + assertEquals(Boolean.TRUE, remoteMessage.get(BACKUP_PURGE_UNUSED)); + assertEquals("someLocation", remoteMessage.get(BACKUP_LOCATION)); + assertEquals("someRepository", remoteMessage.get(BACKUP_REPOSITORY)); + + final AdminCmdContext context = contextCapturer.getValue(); + assertEquals(CollectionParams.CollectionAction.DELETEBACKUP, context.getAction()); + assertNull(context.getAsyncId()); } } diff --git a/solr/core/src/test/org/apache/solr/handler/admin/api/DeleteCollectionSnapshotAPITest.java b/solr/core/src/test/org/apache/solr/handler/admin/api/DeleteCollectionSnapshotAPITest.java index 0ed8e42c12c..a7d82077576 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/api/DeleteCollectionSnapshotAPITest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/api/DeleteCollectionSnapshotAPITest.java @@ -16,48 +16,74 @@ */ package org.apache.solr.handler.admin.api; -import static org.apache.solr.cloud.Overseer.QUEUE_OPERATION; import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP; import static org.apache.solr.common.params.CollectionAdminParams.FOLLOW_ALIASES; -import static org.apache.solr.common.params.CommonAdminParams.ASYNC; import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; import java.util.Map; -import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.cloud.api.collections.AdminCmdContext; import org.apache.solr.common.cloud.ZkNodeProps; +import org.apache.solr.common.params.CollectionParams; import org.apache.solr.common.params.CoreAdminParams; +import org.junit.Before; import org.junit.Test; +import org.mockito.Mockito; -public class DeleteCollectionSnapshotAPITest extends SolrTestCaseJ4 { +public class DeleteCollectionSnapshotAPITest extends MockAPITest { + + private DeleteCollectionSnapshot api; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + when(mockCoreContainer.isZooKeeperAware()).thenReturn(true); + + api = new DeleteCollectionSnapshot(mockCoreContainer, mockQueryRequest, queryResponse); + } @Test - public void testConstructsValidOverseerMessage() { - final ZkNodeProps messageOne = - DeleteCollectionSnapshot.createRemoteMessage("myCollName", false, "mySnapshotName", null); + public void testConstructsValidOverseerMessage() throws Exception { + when(mockClusterState.hasCollection("myCollName")).thenReturn(true); + when(mockSolrZkClient.exists(anyString())).thenReturn(false); + + api.deleteCollectionSnapshot("myCollName", "mySnapshotName", false, null); + verify(mockCommandRunner) + .runCollectionCommand(contextCapturer.capture(), messageCapturer.capture(), anyLong()); + final ZkNodeProps messageOne = messageCapturer.getValue(); final Map rawMessageOne = messageOne.getProperties(); - assertEquals(4, rawMessageOne.size()); + assertEquals(3, rawMessageOne.size()); assertThat( rawMessageOne.keySet(), - containsInAnyOrder( - QUEUE_OPERATION, COLLECTION_PROP, CoreAdminParams.COMMIT_NAME, FOLLOW_ALIASES)); - assertEquals("deletesnapshot", rawMessageOne.get(QUEUE_OPERATION)); + containsInAnyOrder(COLLECTION_PROP, CoreAdminParams.COMMIT_NAME, FOLLOW_ALIASES)); assertEquals("myCollName", rawMessageOne.get(COLLECTION_PROP)); assertEquals("mySnapshotName", rawMessageOne.get(CoreAdminParams.COMMIT_NAME)); assertEquals(false, rawMessageOne.get(FOLLOW_ALIASES)); - final ZkNodeProps messageTwo = - DeleteCollectionSnapshot.createRemoteMessage( - "myCollName", true, "mySnapshotName", "myAsyncId"); + AdminCmdContext context = contextCapturer.getValue(); + assertEquals(CollectionParams.CollectionAction.DELETESNAPSHOT, context.getAction()); + assertNull(context.getAsyncId()); + + Mockito.clearInvocations(mockCommandRunner); + api.deleteCollectionSnapshot("myCollName", "mySnapshotName", true, "testId"); + verify(mockCommandRunner) + .runCollectionCommand(contextCapturer.capture(), messageCapturer.capture(), anyLong()); + final ZkNodeProps messageTwo = messageCapturer.getValue(); final Map rawMessageTwo = messageTwo.getProperties(); - assertEquals(5, rawMessageTwo.size()); + assertEquals(3, rawMessageTwo.size()); assertThat( rawMessageTwo.keySet(), - containsInAnyOrder( - QUEUE_OPERATION, COLLECTION_PROP, CoreAdminParams.COMMIT_NAME, FOLLOW_ALIASES, ASYNC)); - assertEquals("deletesnapshot", rawMessageTwo.get(QUEUE_OPERATION)); + containsInAnyOrder(COLLECTION_PROP, CoreAdminParams.COMMIT_NAME, FOLLOW_ALIASES)); assertEquals("myCollName", rawMessageTwo.get(COLLECTION_PROP)); assertEquals("mySnapshotName", rawMessageTwo.get(CoreAdminParams.COMMIT_NAME)); assertEquals(true, rawMessageTwo.get(FOLLOW_ALIASES)); - assertEquals("myAsyncId", rawMessageTwo.get(ASYNC)); + + context = contextCapturer.getValue(); + assertEquals(CollectionParams.CollectionAction.DELETESNAPSHOT, context.getAction()); + assertEquals("testId", context.getAsyncId()); } } diff --git a/solr/core/src/test/org/apache/solr/handler/admin/api/DeleteNodeAPITest.java b/solr/core/src/test/org/apache/solr/handler/admin/api/DeleteNodeAPITest.java index 74b6b3e927b..8339e632cde 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/api/DeleteNodeAPITest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/api/DeleteNodeAPITest.java @@ -16,59 +16,171 @@ */ package org.apache.solr.handler.admin.api; -import static org.mockito.Mockito.mock; - -import java.util.Map; -import org.apache.solr.SolrTestCaseJ4; -import org.apache.solr.client.api.model.DeleteNodeRequestBody; -import org.apache.solr.common.SolrException; -import org.apache.solr.common.cloud.ZkNodeProps; -import org.apache.solr.common.params.ModifiableSolrParams; +import java.lang.invoke.MethodHandles; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Set; +import org.apache.solr.client.api.model.SubResponseAccumulatingJerseyResponse; +import org.apache.solr.client.solrj.impl.CloudSolrClient; +import org.apache.solr.client.solrj.request.CollectionAdminRequest; +import org.apache.solr.client.solrj.request.NodeApi; +import org.apache.solr.client.solrj.response.RequestStatusState; +import org.apache.solr.cloud.SolrCloudTestCase; +import org.apache.solr.common.cloud.DocCollection; +import org.apache.solr.common.cloud.Replica; +import org.apache.solr.common.cloud.Slice; +import org.apache.solr.common.util.StrUtils; +import org.junit.After; import org.junit.BeforeClass; import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -/** Unit tests for {@link DeleteNode} */ -public class DeleteNodeAPITest extends SolrTestCaseJ4 { +/** Integration tests for {@link DeleteNode} using the V2 API */ +public class DeleteNodeAPITest extends SolrCloudTestCase { + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); @BeforeClass - public static void ensureWorkingMockito() { - assumeWorkingMockito(); + public static void setupCluster() throws Exception { + configureCluster(6) + .addConfig( + "conf1", TEST_PATH().resolve("configsets").resolve("cloud-dynamic").resolve("conf")) + .configure(); } - @Test - public void testV1InvocationThrowsErrorsIfRequiredParametersMissing() { - final var api = mock(DeleteNode.class); - final SolrException e = - expectThrows( - SolrException.class, - () -> { - DeleteNode.invokeUsingV1Inputs(api, new ModifiableSolrParams()); - }); - assertEquals("Missing required parameter: node", e.getMessage()); + @After + public void clearCollections() throws Exception { + cluster.deleteAllCollections(); } @Test - public void testValidOverseerMessageIsCreated() { - final var requestBody = new DeleteNodeRequestBody(); - requestBody.async = "async"; - final ZkNodeProps createdMessage = - DeleteNode.createRemoteMessage("nodeNameToDelete", requestBody); - final Map createdMessageProps = createdMessage.getProperties(); - assertEquals(3, createdMessageProps.size()); - assertEquals("nodeNameToDelete", createdMessageProps.get("node")); - assertEquals("async", createdMessageProps.get("async")); - assertEquals("deletenode", createdMessageProps.get("operation")); + public void testDeleteNode() throws Exception { + CloudSolrClient cloudClient = cluster.getSolrClient(); + String coll = "deletenodetest_coll"; + Set liveNodes = cloudClient.getClusterStateProvider().getLiveNodes(); + ArrayList l = new ArrayList<>(liveNodes); + Collections.shuffle(l, random()); + CollectionAdminRequest.Create create = + pickRandom( + CollectionAdminRequest.createCollection(coll, "conf1", 5, 2, 0, 0), + CollectionAdminRequest.createCollection(coll, "conf1", 5, 1, 1, 0), + CollectionAdminRequest.createCollection(coll, "conf1", 5, 0, 1, 1), + // check RF=1 + CollectionAdminRequest.createCollection(coll, "conf1", 5, 1, 0, 0), + CollectionAdminRequest.createCollection(coll, "conf1", 5, 0, 1, 0)); + create.setCreateNodeSet(StrUtils.join(l, ',')); + cloudClient.request(create); + String nodeToBeDecommissioned = l.get(0); + + // check what replicas are on the node, and whether the call should fail + boolean shouldFail = false; + DocCollection docColl = cloudClient.getClusterStateProvider().getCollection(coll); + log.info("#### DocCollection: {}", docColl); + List replicas = docColl.getReplicasOnNode(nodeToBeDecommissioned); + for (Replica replica : replicas) { + String shard = replica.getShard(); + Slice slice = docColl.getSlice(shard); + boolean hasOtherNonPullReplicas = false; + for (Replica r : slice.getReplicas()) { + if (!r.getName().equals(replica.getName()) + && !r.getNodeName().equals(nodeToBeDecommissioned) + && r.getType().leaderEligible) { + hasOtherNonPullReplicas = true; + break; + } + } + if (!hasOtherNonPullReplicas) { + shouldFail = true; + break; + } + } + + var request = new NodeApi.DeleteNode(nodeToBeDecommissioned); + SubResponseAccumulatingJerseyResponse response = request.process(cloudClient); + + if (log.isInfoEnabled()) { + log.info( + "####### DocCollection after: {}", + cloudClient.getClusterStateProvider().getClusterState().getCollection(coll)); + } + + if (shouldFail) { + assertNotNull( + "Expected request to fail, there should be failures sent back", + response.failedSubResponsesByNodeName); + } else { + assertNull( + "Expected request to not fail, there should be no failures sent back", + response.failedSubResponsesByNodeName); + } } @Test - public void testRequestBodyCanBeOmitted() throws Exception { - final ZkNodeProps createdMessage = DeleteNode.createRemoteMessage("nodeNameToDelete", null); - final Map createdMessageProps = createdMessage.getProperties(); - assertEquals(2, createdMessageProps.size()); - assertEquals("nodeNameToDelete", createdMessageProps.get("node")); - assertEquals("deletenode", createdMessageProps.get("operation")); - assertFalse( - "Expected message to not contain value for async: " + createdMessageProps.get("async"), - createdMessageProps.containsKey("async")); + public void testDeleteNodeAsync() throws Exception { + CloudSolrClient cloudClient = cluster.getSolrClient(); + String coll = "deletenodetest_coll_async"; + Set liveNodes = cloudClient.getClusterStateProvider().getLiveNodes(); + ArrayList l = new ArrayList<>(liveNodes); + Collections.shuffle(l, random()); + CollectionAdminRequest.Create create = + pickRandom( + CollectionAdminRequest.createCollection(coll, "conf1", 5, 2, 0, 0), + CollectionAdminRequest.createCollection(coll, "conf1", 5, 1, 1, 0), + CollectionAdminRequest.createCollection(coll, "conf1", 5, 0, 1, 1), + // check RF=1 + CollectionAdminRequest.createCollection(coll, "conf1", 5, 1, 0, 0), + CollectionAdminRequest.createCollection(coll, "conf1", 5, 0, 1, 0)); + create.setCreateNodeSet(StrUtils.join(l, ',')); + cloudClient.request(create); + String nodeToBeDecommissioned = l.get(0); + + // check what replicas are on the node, and whether the call should fail + boolean shouldFail = false; + DocCollection docColl = cloudClient.getClusterStateProvider().getCollection(coll); + log.info("#### DocCollection: {}", docColl); + List replicas = docColl.getReplicasOnNode(nodeToBeDecommissioned); + for (Replica replica : replicas) { + String shard = replica.getShard(); + Slice slice = docColl.getSlice(shard); + boolean hasOtherNonPullReplicas = false; + for (Replica r : slice.getReplicas()) { + if (!r.getName().equals(replica.getName()) + && !r.getNodeName().equals(nodeToBeDecommissioned) + && r.getType().leaderEligible) { + hasOtherNonPullReplicas = true; + break; + } + } + if (!hasOtherNonPullReplicas) { + shouldFail = true; + break; + } + } + + String asyncId = "003"; + var request = new NodeApi.DeleteNode(nodeToBeDecommissioned); + request.setAsync(asyncId); + SubResponseAccumulatingJerseyResponse response = request.process(cloudClient); + assertNull( + "Expected request to not fail, any failures will be returned in the async status response", + response.failedSubResponsesByNodeName); + + // Wait for the async request to complete + CollectionAdminRequest.RequestStatusResponse rsp = + waitForAsyncClusterRequest(asyncId, Duration.ofSeconds(5)); + + if (log.isInfoEnabled()) { + log.info( + "####### DocCollection after: {}", + cloudClient.getClusterStateProvider().getClusterState().getCollection(coll)); + } + + if (shouldFail) { + assertSame(String.valueOf(rsp), RequestStatusState.FAILED, rsp.getRequestStatus()); + } else { + assertNotSame(String.valueOf(rsp), RequestStatusState.FAILED, rsp.getRequestStatus()); + } } } diff --git a/solr/core/src/test/org/apache/solr/handler/admin/api/DeleteReplicaAPITest.java b/solr/core/src/test/org/apache/solr/handler/admin/api/DeleteReplicaAPITest.java index d0dca0bce8e..710c9673d34 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/api/DeleteReplicaAPITest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/api/DeleteReplicaAPITest.java @@ -17,34 +17,50 @@ package org.apache.solr.handler.admin.api; -import static org.apache.solr.cloud.Overseer.QUEUE_OPERATION; import static org.apache.solr.cloud.api.collections.CollectionHandlingUtils.ONLY_IF_DOWN; import static org.apache.solr.common.cloud.ZkStateReader.SHARD_ID_PROP; import static org.apache.solr.common.params.CollectionAdminParams.COLLECTION; import static org.apache.solr.common.params.CollectionAdminParams.COUNT_PROP; import static org.apache.solr.common.params.CollectionAdminParams.FOLLOW_ALIASES; import static org.apache.solr.common.params.CollectionAdminParams.REPLICA; -import static org.apache.solr.common.params.CommonAdminParams.ASYNC; import static org.apache.solr.common.params.CoreAdminParams.DELETE_DATA_DIR; import static org.apache.solr.common.params.CoreAdminParams.DELETE_INDEX; import static org.apache.solr.common.params.CoreAdminParams.DELETE_INSTANCE_DIR; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; -import org.apache.solr.SolrTestCaseJ4; +import java.util.Map; +import org.apache.solr.client.api.model.ScaleCollectionRequestBody; +import org.apache.solr.cloud.api.collections.AdminCmdContext; import org.apache.solr.common.SolrException; +import org.apache.solr.common.cloud.ZkNodeProps; +import org.apache.solr.common.params.CollectionParams; +import org.junit.Before; import org.junit.Test; /** Unit tests for {@link DeleteReplica} */ -public class DeleteReplicaAPITest extends SolrTestCaseJ4 { +public class DeleteReplicaAPITest extends MockAPITest { + + private DeleteReplica api; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + when(mockCoreContainer.isZooKeeperAware()).thenReturn(true); + + api = new DeleteReplica(mockCoreContainer, mockQueryRequest, queryResponse); + } + @Test public void testReportsErrorIfCollectionNameMissing() { final SolrException thrown = expectThrows( SolrException.class, - () -> { - final var api = new DeleteReplica(null, null, null); - api.deleteReplicaByName( - null, "someShard", "someReplica", null, null, null, null, null, null); - }); + () -> + api.deleteReplicaByName( + null, "someShard", "someReplica", null, null, null, null, null, null)); assertEquals(400, thrown.code()); assertEquals("Missing required parameter: collection", thrown.getMessage()); @@ -55,11 +71,9 @@ public void testReportsErrorIfShardNameMissing() { final SolrException thrown = expectThrows( SolrException.class, - () -> { - final var api = new DeleteReplica(null, null, null); - api.deleteReplicaByName( - "someCollection", null, "someReplica", null, null, null, null, null, null); - }); + () -> + api.deleteReplicaByName( + "someCollection", null, "someReplica", null, null, null, null, null, null)); assertEquals(400, thrown.code()); assertEquals("Missing required parameter: shard", thrown.getMessage()); @@ -70,66 +84,112 @@ public void testReportsErrorIfReplicaNameMissingWhenDeletingByName() { final SolrException thrown = expectThrows( SolrException.class, - () -> { - final var api = new DeleteReplica(null, null, null); - api.deleteReplicaByName( - "someCollection", "someShard", null, null, null, null, null, null, null); - }); + () -> + api.deleteReplicaByName( + "someCollection", "someShard", null, null, null, null, null, null, null)); assertEquals(400, thrown.code()); assertEquals("Missing required parameter: replica", thrown.getMessage()); } @Test - public void testCreateRemoteMessageAllProperties() { - final var remoteMessage = - DeleteReplica.createRemoteMessage( - "someCollName", - "someShardName", - "someReplicaName", - 123, - true, - false, - true, - false, - true, - "someAsyncId") - .getProperties(); - - assertEquals(11, remoteMessage.size()); - assertEquals("deletereplica", remoteMessage.get(QUEUE_OPERATION)); + public void testCreateRemoteMessageByName() throws Exception { + api.deleteReplicaByName( + "someCollName", + "someShardName", + "someReplicaName", + true, + false, + true, + false, + true, + "someAsyncId"); + verify(mockCommandRunner) + .runCollectionCommand(contextCapturer.capture(), messageCapturer.capture(), anyLong()); + + final ZkNodeProps message = messageCapturer.getValue(); + final Map remoteMessage = message.getProperties(); + assertEquals(8, remoteMessage.size()); assertEquals("someCollName", remoteMessage.get(COLLECTION)); assertEquals("someShardName", remoteMessage.get(SHARD_ID_PROP)); assertEquals("someReplicaName", remoteMessage.get(REPLICA)); - assertEquals(Integer.valueOf(123), remoteMessage.get(COUNT_PROP)); assertEquals(Boolean.TRUE, remoteMessage.get(FOLLOW_ALIASES)); assertEquals(Boolean.FALSE, remoteMessage.get(DELETE_INSTANCE_DIR)); assertEquals(Boolean.TRUE, remoteMessage.get(DELETE_DATA_DIR)); assertEquals(Boolean.FALSE, remoteMessage.get(DELETE_INDEX)); assertEquals(Boolean.TRUE, remoteMessage.get(ONLY_IF_DOWN)); - assertEquals("someAsyncId", remoteMessage.get(ASYNC)); + + AdminCmdContext context = contextCapturer.getValue(); + assertEquals(CollectionParams.CollectionAction.DELETEREPLICA, context.getAction()); + assertEquals("someAsyncId", context.getAsyncId()); + } + + @Test + public void testCreateRemoteMessageByCount() throws Exception { + api.deleteReplicasByCount( + "someCollName", "someShardName", 123, true, false, true, false, true, "someAsyncId"); + verify(mockCommandRunner) + .runCollectionCommand(contextCapturer.capture(), messageCapturer.capture(), anyLong()); + + final ZkNodeProps message = messageCapturer.getValue(); + final Map remoteMessage = message.getProperties(); + assertEquals(8, remoteMessage.size()); + assertEquals("someCollName", remoteMessage.get(COLLECTION)); + assertEquals("someShardName", remoteMessage.get(SHARD_ID_PROP)); + assertEquals(123, remoteMessage.get(COUNT_PROP)); + assertEquals(Boolean.TRUE, remoteMessage.get(FOLLOW_ALIASES)); + assertEquals(Boolean.FALSE, remoteMessage.get(DELETE_INSTANCE_DIR)); + assertEquals(Boolean.TRUE, remoteMessage.get(DELETE_DATA_DIR)); + assertEquals(Boolean.FALSE, remoteMessage.get(DELETE_INDEX)); + assertEquals(Boolean.TRUE, remoteMessage.get(ONLY_IF_DOWN)); + + AdminCmdContext context = contextCapturer.getValue(); + assertEquals(CollectionParams.CollectionAction.DELETEREPLICA, context.getAction()); + assertEquals("someAsyncId", context.getAsyncId()); + } + + @Test + public void testCreateRemoteMessageByCountAllShards() throws Exception { + ScaleCollectionRequestBody body = new ScaleCollectionRequestBody(); + body.numToDelete = 123; + body.followAliases = true; + body.deleteIndex = true; + body.onlyIfDown = false; + body.async = "someAsyncId"; + api.deleteReplicasByCountAllShards("someCollName", body); + verify(mockCommandRunner) + .runCollectionCommand(contextCapturer.capture(), messageCapturer.capture(), anyLong()); + + final ZkNodeProps message = messageCapturer.getValue(); + final Map remoteMessage = message.getProperties(); + assertEquals(5, remoteMessage.size()); + assertEquals("someCollName", remoteMessage.get(COLLECTION)); + assertEquals(123, remoteMessage.get(COUNT_PROP)); + assertEquals(Boolean.TRUE, remoteMessage.get(FOLLOW_ALIASES)); + assertEquals(Boolean.TRUE, remoteMessage.get(DELETE_INDEX)); + assertEquals(Boolean.FALSE, remoteMessage.get(ONLY_IF_DOWN)); + + AdminCmdContext context = contextCapturer.getValue(); + assertEquals(CollectionParams.CollectionAction.DELETEREPLICA, context.getAction()); + assertEquals("someAsyncId", context.getAsyncId()); } @Test - public void testMissingValuesExcludedFromRemoteMessage() { - final var remoteMessage = - DeleteReplica.createRemoteMessage( - "someCollName", - "someShardName", - "someReplicaName", - null, - null, - null, - null, - null, - null, - null) - .getProperties(); - - assertEquals(4, remoteMessage.size()); - assertEquals("deletereplica", remoteMessage.get(QUEUE_OPERATION)); + public void testMissingValuesExcludedFromRemoteMessage() throws Exception { + api.deleteReplicaByName( + "someCollName", "someShardName", "someReplicaName", null, null, null, null, null, null); + verify(mockCommandRunner) + .runCollectionCommand(contextCapturer.capture(), messageCapturer.capture(), anyLong()); + + final ZkNodeProps message = messageCapturer.getValue(); + final Map remoteMessage = message.getProperties(); + assertEquals(3, remoteMessage.size()); assertEquals("someCollName", remoteMessage.get(COLLECTION)); assertEquals("someShardName", remoteMessage.get(SHARD_ID_PROP)); assertEquals("someReplicaName", remoteMessage.get(REPLICA)); + + AdminCmdContext context = contextCapturer.getValue(); + assertEquals(CollectionParams.CollectionAction.DELETEREPLICA, context.getAction()); + assertNull("AsyncId should be null", context.getAsyncId()); } } diff --git a/solr/core/src/test/org/apache/solr/handler/admin/api/DeleteReplicaPropertyAPITest.java b/solr/core/src/test/org/apache/solr/handler/admin/api/DeleteReplicaPropertyAPITest.java index 19c85b85481..b5562f52118 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/api/DeleteReplicaPropertyAPITest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/api/DeleteReplicaPropertyAPITest.java @@ -17,20 +17,22 @@ package org.apache.solr.handler.admin.api; -import static org.apache.solr.cloud.Overseer.QUEUE_OPERATION; import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP; import static org.apache.solr.common.cloud.ZkStateReader.PROPERTY_PROP; import static org.apache.solr.common.cloud.ZkStateReader.REPLICA_PROP; import static org.apache.solr.common.cloud.ZkStateReader.SHARD_ID_PROP; +import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; import java.util.Map; -import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.cloud.api.collections.AdminCmdContext; import org.apache.solr.common.SolrException; import org.apache.solr.common.cloud.ZkNodeProps; +import org.apache.solr.common.params.CollectionParams; import org.apache.solr.common.params.ModifiableSolrParams; -import org.junit.BeforeClass; +import org.junit.Before; import org.junit.Test; /** @@ -40,16 +42,21 @@ * here focus primarily on how the v1 code invokes the v2 API and how the v2 API crafts its * overseer/Distributed State Processing RPC message. */ -public class DeleteReplicaPropertyAPITest extends SolrTestCaseJ4 { +public class DeleteReplicaPropertyAPITest extends MockAPITest { - @BeforeClass - public static void ensureWorkingMockito() { - assumeWorkingMockito(); + private DeleteReplicaProperty api; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + when(mockCoreContainer.isZooKeeperAware()).thenReturn(true); + + api = new DeleteReplicaProperty(mockCoreContainer, mockQueryRequest, queryResponse); } @Test public void testV1InvocationThrowsErrorsIfRequiredParametersMissing() { - final var api = mock(DeleteReplicaProperty.class); final var allParams = new ModifiableSolrParams(); allParams.add(COLLECTION_PROP, "someColl"); allParams.add(SHARD_ID_PROP, "someShard"); @@ -62,9 +69,7 @@ public void testV1InvocationThrowsErrorsIfRequiredParametersMissing() { final SolrException e = expectThrows( SolrException.class, - () -> { - DeleteReplicaProperty.invokeUsingV1Inputs(api, noCollectionParams); - }); + () -> DeleteReplicaProperty.invokeUsingV1Inputs(api, noCollectionParams)); assertEquals("Missing required parameter: " + COLLECTION_PROP, e.getMessage()); } @@ -74,9 +79,7 @@ public void testV1InvocationThrowsErrorsIfRequiredParametersMissing() { final SolrException e = expectThrows( SolrException.class, - () -> { - DeleteReplicaProperty.invokeUsingV1Inputs(api, noShardParams); - }); + () -> DeleteReplicaProperty.invokeUsingV1Inputs(api, noShardParams)); assertEquals("Missing required parameter: " + SHARD_ID_PROP, e.getMessage()); } @@ -98,53 +101,55 @@ public void testV1InvocationThrowsErrorsIfRequiredParametersMissing() { final SolrException e = expectThrows( SolrException.class, - () -> { - DeleteReplicaProperty.invokeUsingV1Inputs(api, noPropertyParams); - }); + () -> DeleteReplicaProperty.invokeUsingV1Inputs(api, noPropertyParams)); assertEquals("Missing required parameter: " + PROPERTY_PROP, e.getMessage()); } } @Test public void testV1InvocationPassesAllProvidedParameters() throws Exception { - final var api = mock(DeleteReplicaProperty.class); + final var mockApi = mock(DeleteReplicaProperty.class); final var allParams = new ModifiableSolrParams(); allParams.add(COLLECTION_PROP, "someColl"); allParams.add(SHARD_ID_PROP, "someShard"); allParams.add(REPLICA_PROP, "someReplica"); allParams.add(PROPERTY_PROP, "somePropName"); - DeleteReplicaProperty.invokeUsingV1Inputs(api, allParams); + DeleteReplicaProperty.invokeUsingV1Inputs(mockApi, allParams); - verify(api).deleteReplicaProperty("someColl", "someShard", "someReplica", "somePropName"); + verify(mockApi).deleteReplicaProperty("someColl", "someShard", "someReplica", "somePropName"); } @Test public void testV1InvocationTrimsPropertyNamePrefixIfProvided() throws Exception { - final var api = mock(DeleteReplicaProperty.class); + final var mockApi = mock(DeleteReplicaProperty.class); final var allParams = new ModifiableSolrParams(); allParams.add(COLLECTION_PROP, "someColl"); allParams.add(SHARD_ID_PROP, "someShard"); allParams.add(REPLICA_PROP, "someReplica"); allParams.add(PROPERTY_PROP, "property.somePropName"); // NOTE THE "property." prefix! - DeleteReplicaProperty.invokeUsingV1Inputs(api, allParams); + DeleteReplicaProperty.invokeUsingV1Inputs(mockApi, allParams); - verify(api).deleteReplicaProperty("someColl", "someShard", "someReplica", "somePropName"); + verify(mockApi).deleteReplicaProperty("someColl", "someShard", "someReplica", "somePropName"); } @Test - public void testRPCMessageCreation() { - final ZkNodeProps message = - DeleteReplicaProperty.createRemoteMessage( - "someColl", "someShard", "someReplica", "somePropName"); + public void testRPCMessageCreation() throws Exception { + api.deleteReplicaProperty("someColl", "someShard", "someReplica", "somePropName"); + verify(mockCommandRunner) + .runCollectionCommand(contextCapturer.capture(), messageCapturer.capture(), anyLong()); + final ZkNodeProps message = messageCapturer.getValue(); final Map props = message.getProperties(); - assertEquals(5, props.size()); - assertEquals("deletereplicaprop", props.get(QUEUE_OPERATION)); + assertEquals(4, props.size()); assertEquals("someColl", props.get(COLLECTION_PROP)); assertEquals("someShard", props.get(SHARD_ID_PROP)); assertEquals("someReplica", props.get(REPLICA_PROP)); assertEquals("somePropName", props.get(PROPERTY_PROP)); + + AdminCmdContext context = contextCapturer.getValue(); + assertEquals(CollectionParams.CollectionAction.DELETEREPLICAPROP, context.getAction()); + assertNull(context.getAsyncId()); } } diff --git a/solr/core/src/test/org/apache/solr/handler/admin/api/DeleteShardAPITest.java b/solr/core/src/test/org/apache/solr/handler/admin/api/DeleteShardAPITest.java index e26653e6d89..fa161880fb2 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/api/DeleteShardAPITest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/api/DeleteShardAPITest.java @@ -17,30 +17,45 @@ package org.apache.solr.handler.admin.api; -import static org.apache.solr.cloud.Overseer.QUEUE_OPERATION; import static org.apache.solr.common.cloud.ZkStateReader.SHARD_ID_PROP; import static org.apache.solr.common.params.CollectionAdminParams.COLLECTION; import static org.apache.solr.common.params.CollectionAdminParams.FOLLOW_ALIASES; -import static org.apache.solr.common.params.CommonAdminParams.ASYNC; import static org.apache.solr.common.params.CoreAdminParams.DELETE_DATA_DIR; import static org.apache.solr.common.params.CoreAdminParams.DELETE_INDEX; import static org.apache.solr.common.params.CoreAdminParams.DELETE_INSTANCE_DIR; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; -import org.apache.solr.SolrTestCaseJ4; +import java.util.Map; +import org.apache.solr.cloud.api.collections.AdminCmdContext; import org.apache.solr.common.SolrException; +import org.apache.solr.common.cloud.ZkNodeProps; +import org.apache.solr.common.params.CollectionParams; +import org.junit.Before; import org.junit.Test; /** Unit tests for {@link DeleteShard} */ -public class DeleteShardAPITest extends SolrTestCaseJ4 { +public class DeleteShardAPITest extends MockAPITest { + + private DeleteShard api; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + when(mockCoreContainer.isZooKeeperAware()).thenReturn(true); + + api = new DeleteShard(mockCoreContainer, mockQueryRequest, queryResponse); + } + @Test public void testReportsErrorIfCollectionNameMissing() { final SolrException thrown = expectThrows( SolrException.class, - () -> { - final var api = new DeleteShard(null, null, null); - api.deleteShard(null, "someShard", null, null, null, null, null); - }); + () -> api.deleteShard(null, "someShard", null, null, null, null, null)); assertEquals(400, thrown.code()); assertEquals("Missing required parameter: collection", thrown.getMessage()); @@ -51,43 +66,51 @@ public void testReportsErrorIfShardNameMissing() { final SolrException thrown = expectThrows( SolrException.class, - () -> { - final var api = new DeleteShard(null, null, null); - api.deleteShard("someCollection", null, null, null, null, null, null); - }); + () -> api.deleteShard("someCollection", null, null, null, null, null, null)); assertEquals(400, thrown.code()); assertEquals("Missing required parameter: shard", thrown.getMessage()); } @Test - public void testCreateRemoteMessageAllProperties() { - final var remoteMessage = - DeleteShard.createRemoteMessage( - "someCollName", "someShardName", true, false, true, false, "someAsyncId") - .getProperties(); - - assertEquals(8, remoteMessage.size()); - assertEquals("deleteshard", remoteMessage.get(QUEUE_OPERATION)); + public void testCreateRemoteMessageAllProperties() throws Exception { + when(mockClusterState.hasCollection(eq("someCollectionName"))).thenReturn(true); + api.deleteShard("someCollName", "someShardName", true, false, true, false, "someAsyncId"); + verify(mockCommandRunner) + .runCollectionCommand(contextCapturer.capture(), messageCapturer.capture(), anyLong()); + + final ZkNodeProps createdMessage = messageCapturer.getValue(); + final Map remoteMessage = createdMessage.getProperties(); + + assertEquals(6, remoteMessage.size()); assertEquals("someCollName", remoteMessage.get(COLLECTION)); assertEquals("someShardName", remoteMessage.get(SHARD_ID_PROP)); assertEquals(Boolean.TRUE, remoteMessage.get(DELETE_INSTANCE_DIR)); assertEquals(Boolean.FALSE, remoteMessage.get(DELETE_DATA_DIR)); assertEquals(Boolean.TRUE, remoteMessage.get(DELETE_INDEX)); assertEquals(Boolean.FALSE, remoteMessage.get(FOLLOW_ALIASES)); - assertEquals("someAsyncId", remoteMessage.get(ASYNC)); + + final AdminCmdContext context = contextCapturer.getValue(); + assertEquals(CollectionParams.CollectionAction.DELETESHARD, context.getAction()); + assertEquals("someAsyncId", context.getAsyncId()); } @Test - public void testMissingValuesExcludedFromRemoteMessage() { - final var remoteMessage = - DeleteShard.createRemoteMessage( - "someCollName", "someShardName", null, null, null, null, null) - .getProperties(); - - assertEquals(3, remoteMessage.size()); - assertEquals("deleteshard", remoteMessage.get(QUEUE_OPERATION)); + public void testMissingValuesExcludedFromRemoteMessage() throws Exception { + when(mockClusterState.hasCollection(eq("someCollectionName"))).thenReturn(true); + api.deleteShard("someCollName", "someShardName", null, null, null, null, null); + verify(mockCommandRunner) + .runCollectionCommand(contextCapturer.capture(), messageCapturer.capture(), anyLong()); + + final ZkNodeProps createdMessage = messageCapturer.getValue(); + final Map remoteMessage = createdMessage.getProperties(); + + assertEquals(2, remoteMessage.size()); assertEquals("someCollName", remoteMessage.get(COLLECTION)); assertEquals("someShardName", remoteMessage.get(SHARD_ID_PROP)); + + final AdminCmdContext context = contextCapturer.getValue(); + assertEquals(CollectionParams.CollectionAction.DELETESHARD, context.getAction()); + assertNull("There should be no asyncId", context.getAsyncId()); } } diff --git a/solr/core/src/test/org/apache/solr/handler/admin/api/ForceLeaderAPITest.java b/solr/core/src/test/org/apache/solr/handler/admin/api/ForceLeaderAPITest.java index be1fce44fb8..5158041282e 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/api/ForceLeaderAPITest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/api/ForceLeaderAPITest.java @@ -17,21 +17,30 @@ package org.apache.solr.handler.admin.api; -import org.apache.solr.SolrTestCaseJ4; +import static org.mockito.Mockito.when; + import org.apache.solr.common.SolrException; +import org.junit.Before; import org.junit.Test; /** Unit tests for {@link ForceLeader} */ -public class ForceLeaderAPITest extends SolrTestCaseJ4 { +public class ForceLeaderAPITest extends MockAPITest { + + private ForceLeader api; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + when(mockCoreContainer.isZooKeeperAware()).thenReturn(true); + + api = new ForceLeader(mockCoreContainer, mockQueryRequest, queryResponse); + } + @Test public void testReportsErrorIfCollectionNameMissing() { final SolrException thrown = - expectThrows( - SolrException.class, - () -> { - final var api = new ForceLeader(null, null, null); - api.forceShardLeader(null, "someShard"); - }); + expectThrows(SolrException.class, () -> api.forceShardLeader(null, "someShard")); assertEquals(400, thrown.code()); assertEquals("Missing required parameter: collection", thrown.getMessage()); @@ -40,12 +49,7 @@ public void testReportsErrorIfCollectionNameMissing() { @Test public void testReportsErrorIfShardNameMissing() { final SolrException thrown = - expectThrows( - SolrException.class, - () -> { - final var api = new ForceLeader(null, null, null); - api.forceShardLeader("someCollection", null); - }); + expectThrows(SolrException.class, () -> api.forceShardLeader("someCollection", null)); assertEquals(400, thrown.code()); assertEquals("Missing required parameter: shard", thrown.getMessage()); diff --git a/solr/core/src/test/org/apache/solr/handler/admin/api/MigrateReplicasAPITest.java b/solr/core/src/test/org/apache/solr/handler/admin/api/MigrateReplicasAPITest.java index 85f0ffc1d3c..3e49723fa3d 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/api/MigrateReplicasAPITest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/api/MigrateReplicasAPITest.java @@ -16,66 +16,33 @@ */ package org.apache.solr.handler.admin.api; -import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyLong; -import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import java.util.Collections; import java.util.Map; -import java.util.Optional; import java.util.Set; -import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.client.api.model.MigrateReplicasRequestBody; -import org.apache.solr.cloud.OverseerSolrResponse; -import org.apache.solr.cloud.ZkController; -import org.apache.solr.cloud.api.collections.DistributedCollectionConfigSetCommandRunner; +import org.apache.solr.cloud.api.collections.AdminCmdContext; import org.apache.solr.common.SolrException; import org.apache.solr.common.cloud.ZkNodeProps; -import org.apache.solr.common.util.NamedList; -import org.apache.solr.core.CoreContainer; -import org.apache.solr.request.SolrQueryRequest; -import org.apache.solr.response.SolrQueryResponse; +import org.apache.solr.common.params.CollectionParams; import org.junit.Before; -import org.junit.BeforeClass; import org.junit.Test; -import org.mockito.ArgumentCaptor; /** Unit tests for {@link ReplaceNode} */ -public class MigrateReplicasAPITest extends SolrTestCaseJ4 { +public class MigrateReplicasAPITest extends MockAPITest { - private CoreContainer mockCoreContainer; - private ZkController mockZkController; - private SolrQueryRequest mockQueryRequest; - private SolrQueryResponse queryResponse; - private MigrateReplicas migrateReplicasAPI; - private DistributedCollectionConfigSetCommandRunner mockCommandRunner; - private ArgumentCaptor messageCapturer; - - @BeforeClass - public static void ensureWorkingMockito() { - assumeWorkingMockito(); - } + private MigrateReplicas api; @Override @Before public void setUp() throws Exception { super.setUp(); - - mockCoreContainer = mock(CoreContainer.class); - mockZkController = mock(ZkController.class); - mockCommandRunner = mock(DistributedCollectionConfigSetCommandRunner.class); - when(mockCoreContainer.getZkController()).thenReturn(mockZkController); - when(mockZkController.getDistributedCommandRunner()).thenReturn(Optional.of(mockCommandRunner)); - when(mockCommandRunner.runCollectionCommand(any(), any(), anyLong())) - .thenReturn(new OverseerSolrResponse(new NamedList<>())); - mockQueryRequest = mock(SolrQueryRequest.class); - queryResponse = new SolrQueryResponse(); - migrateReplicasAPI = new MigrateReplicas(mockCoreContainer, mockQueryRequest, queryResponse); - messageCapturer = ArgumentCaptor.forClass(ZkNodeProps.class); - when(mockCoreContainer.isZooKeeperAware()).thenReturn(true); + + api = new MigrateReplicas(mockCoreContainer, mockQueryRequest, queryResponse); } @Test @@ -83,31 +50,36 @@ public void testCreatesValidOverseerMessage() throws Exception { MigrateReplicasRequestBody requestBody = new MigrateReplicasRequestBody( Set.of("demoSourceNode"), Set.of("demoTargetNode"), false, "async"); - migrateReplicasAPI.migrateReplicas(requestBody); - verify(mockCommandRunner).runCollectionCommand(messageCapturer.capture(), any(), anyLong()); + api.migrateReplicas(requestBody); + verify(mockCommandRunner) + .runCollectionCommand(contextCapturer.capture(), messageCapturer.capture(), anyLong()); final ZkNodeProps createdMessage = messageCapturer.getValue(); final Map createdMessageProps = createdMessage.getProperties(); - assertEquals(5, createdMessageProps.size()); + assertEquals(3, createdMessageProps.size()); assertEquals(Set.of("demoSourceNode"), createdMessageProps.get("sourceNodes")); assertEquals(Set.of("demoTargetNode"), createdMessageProps.get("targetNodes")); assertEquals(false, createdMessageProps.get("waitForFinalState")); - assertEquals("async", createdMessageProps.get("async")); - assertEquals("migrate_replicas", createdMessageProps.get("operation")); + final AdminCmdContext context = contextCapturer.getValue(); + assertEquals(CollectionParams.CollectionAction.MIGRATE_REPLICAS, context.getAction()); + assertEquals("async", context.getAsyncId()); } @Test public void testNoTargetNodes() throws Exception { MigrateReplicasRequestBody requestBody = new MigrateReplicasRequestBody(Set.of("demoSourceNode"), null, null, null); - migrateReplicasAPI.migrateReplicas(requestBody); - verify(mockCommandRunner).runCollectionCommand(messageCapturer.capture(), any(), anyLong()); + api.migrateReplicas(requestBody); + verify(mockCommandRunner) + .runCollectionCommand(contextCapturer.capture(), messageCapturer.capture(), anyLong()); final ZkNodeProps createdMessage = messageCapturer.getValue(); final Map createdMessageProps = createdMessage.getProperties(); - assertEquals(2, createdMessageProps.size()); + assertEquals(1, createdMessageProps.size()); assertEquals(Set.of("demoSourceNode"), createdMessageProps.get("sourceNodes")); - assertEquals("migrate_replicas", createdMessageProps.get("operation")); + final AdminCmdContext context = contextCapturer.getValue(); + assertEquals(CollectionParams.CollectionAction.MIGRATE_REPLICAS, context.getAction()); + assertNull("There should be no asyncId", context.getAsyncId()); } @Test @@ -115,9 +87,9 @@ public void testNoSourceNodesThrowsError() throws Exception { MigrateReplicasRequestBody requestBody1 = new MigrateReplicasRequestBody( Collections.emptySet(), Set.of("demoTargetNode"), null, null); - assertThrows(SolrException.class, () -> migrateReplicasAPI.migrateReplicas(requestBody1)); + assertThrows(SolrException.class, () -> api.migrateReplicas(requestBody1)); MigrateReplicasRequestBody requestBody2 = new MigrateReplicasRequestBody(null, Set.of("demoTargetNode"), null, null); - assertThrows(SolrException.class, () -> migrateReplicasAPI.migrateReplicas(requestBody2)); + assertThrows(SolrException.class, () -> api.migrateReplicas(requestBody2)); } } diff --git a/solr/core/src/test/org/apache/solr/handler/admin/api/MockAPITest.java b/solr/core/src/test/org/apache/solr/handler/admin/api/MockAPITest.java new file mode 100644 index 00000000000..a47d4943137 --- /dev/null +++ b/solr/core/src/test/org/apache/solr/handler/admin/api/MockAPITest.java @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.handler.admin.api; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import io.opentelemetry.api.trace.Span; +import java.util.Optional; +import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.cloud.OverseerSolrResponse; +import org.apache.solr.cloud.ZkController; +import org.apache.solr.cloud.api.collections.AdminCmdContext; +import org.apache.solr.cloud.api.collections.DistributedCollectionConfigSetCommandRunner; +import org.apache.solr.common.cloud.Aliases; +import org.apache.solr.common.cloud.ClusterState; +import org.apache.solr.common.cloud.SolrZkClient; +import org.apache.solr.common.cloud.ZkNodeProps; +import org.apache.solr.common.cloud.ZkStateReader; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.core.CoreContainer; +import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.response.SolrQueryResponse; +import org.junit.Before; +import org.junit.BeforeClass; +import org.mockito.ArgumentCaptor; + +/** + * Abstract test class to setup shared mocks for unit testing v2 API calls that go to the Overseer. + */ +public class MockAPITest extends SolrTestCaseJ4 { + + protected CoreContainer mockCoreContainer; + protected ZkController mockZkController; + protected ClusterState mockClusterState; + protected DistributedCollectionConfigSetCommandRunner mockCommandRunner; + protected SolrZkClient mockSolrZkClient; + protected ZkStateReader mockZkStateReader; + protected SolrQueryRequest mockQueryRequest; + protected SolrQueryResponse queryResponse; + protected ArgumentCaptor messageCapturer; + protected ArgumentCaptor contextCapturer; + + @BeforeClass + public static void ensureWorkingMockito() { + assumeWorkingMockito(); + } + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + + mockCoreContainer = mock(CoreContainer.class); + mockZkController = mock(ZkController.class); + mockClusterState = mock(ClusterState.class); + mockSolrZkClient = mock(SolrZkClient.class); + mockZkStateReader = mock(ZkStateReader.class); + mockCommandRunner = mock(DistributedCollectionConfigSetCommandRunner.class); + when(mockCoreContainer.getZkController()).thenReturn(mockZkController); + when(mockCoreContainer.getAliases()).thenReturn(Aliases.EMPTY); + when(mockZkController.getDistributedCommandRunner()).thenReturn(Optional.of(mockCommandRunner)); + when(mockZkController.getClusterState()).thenReturn(mockClusterState); + when(mockZkController.getZkStateReader()).thenReturn(mockZkStateReader); + when(mockZkController.getZkClient()).thenReturn(mockSolrZkClient); + when(mockCommandRunner.runCollectionCommand(any(), any(), anyLong())) + .thenReturn(new OverseerSolrResponse(new NamedList<>())); + mockQueryRequest = mock(SolrQueryRequest.class); + when(mockQueryRequest.getSpan()).thenReturn(Span.getInvalid()); + queryResponse = new SolrQueryResponse(); + messageCapturer = ArgumentCaptor.forClass(ZkNodeProps.class); + contextCapturer = ArgumentCaptor.forClass(AdminCmdContext.class); + } +} diff --git a/solr/core/src/test/org/apache/solr/handler/admin/api/ReloadCollectionAPITest.java b/solr/core/src/test/org/apache/solr/handler/admin/api/ReloadCollectionAPITest.java index 5900fece7a4..03a8f3e6d7d 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/api/ReloadCollectionAPITest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/api/ReloadCollectionAPITest.java @@ -17,42 +17,61 @@ package org.apache.solr.handler.admin.api; -import static org.apache.solr.cloud.Overseer.QUEUE_OPERATION; -import static org.apache.solr.common.params.CommonAdminParams.ASYNC; import static org.apache.solr.common.params.CoreAdminParams.NAME; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; -import org.apache.solr.SolrTestCaseJ4; +import java.util.Map; import org.apache.solr.client.api.model.ReloadCollectionRequestBody; +import org.apache.solr.cloud.api.collections.AdminCmdContext; import org.apache.solr.common.SolrException; +import org.apache.solr.common.cloud.ZkNodeProps; +import org.apache.solr.common.params.CollectionParams; +import org.junit.Before; import org.junit.Test; /** Unit tests for {@link ReloadCollectionAPI} */ -public class ReloadCollectionAPITest extends SolrTestCaseJ4 { +public class ReloadCollectionAPITest extends MockAPITest { + + private ReloadCollectionAPI api; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + when(mockCoreContainer.isZooKeeperAware()).thenReturn(true); + + api = new ReloadCollectionAPI(mockCoreContainer, mockQueryRequest, queryResponse); + } + @Test public void testReportsErrorIfCollectionNameMissing() { final SolrException thrown = expectThrows( SolrException.class, - () -> { - final var api = new ReloadCollectionAPI(null, null, null); - api.reloadCollection(null, new ReloadCollectionRequestBody()); - }); + () -> api.reloadCollection(null, new ReloadCollectionRequestBody())); assertEquals(400, thrown.code()); assertEquals("Missing required parameter: collection", thrown.getMessage()); } - // TODO message creation @Test - public void testCreateRemoteMessageAllProperties() { + public void testCreateRemoteMessageAllProperties() throws Exception { final var requestBody = new ReloadCollectionRequestBody(); requestBody.async = "someAsyncId"; - final var remoteMessage = - ReloadCollectionAPI.createRemoteMessage("someCollName", requestBody).getProperties(); - assertEquals(3, remoteMessage.size()); - assertEquals("reload", remoteMessage.get(QUEUE_OPERATION)); + api.reloadCollection("someCollName", requestBody); + verify(mockCommandRunner) + .runCollectionCommand(contextCapturer.capture(), messageCapturer.capture(), anyLong()); + + final ZkNodeProps createdMessage = messageCapturer.getValue(); + final Map remoteMessage = createdMessage.getProperties(); + assertEquals(1, remoteMessage.size()); assertEquals("someCollName", remoteMessage.get(NAME)); - assertEquals("someAsyncId", remoteMessage.get(ASYNC)); + + final AdminCmdContext context = contextCapturer.getValue(); + assertEquals(CollectionParams.CollectionAction.RELOAD, context.getAction()); + assertEquals("someAsyncId", context.getAsyncId()); } } diff --git a/solr/core/src/test/org/apache/solr/handler/admin/api/ReloadCoreAPITest.java b/solr/core/src/test/org/apache/solr/handler/admin/api/ReloadCoreAPITest.java index fc73c6781f9..1589f97310a 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/api/ReloadCoreAPITest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/api/ReloadCoreAPITest.java @@ -31,7 +31,7 @@ public class ReloadCoreAPITest extends SolrTestCaseJ4 { - private ReloadCore reloadCoreAPI; + private ReloadCore api; private static final String NON_EXISTENT_CORE = "non_existent_core"; @BeforeClass @@ -49,13 +49,12 @@ public void setUp() throws Exception { CoreContainer coreContainer = h.getCoreContainer(); CoreAdminHandler.CoreAdminAsyncTracker coreAdminAsyncTracker = new CoreAdminHandler.CoreAdminAsyncTracker(); - reloadCoreAPI = - new ReloadCore(solrQueryRequest, solrQueryResponse, coreContainer, coreAdminAsyncTracker); + api = new ReloadCore(solrQueryRequest, solrQueryResponse, coreContainer, coreAdminAsyncTracker); } @Test public void testValidReloadCoreAPIResponse() throws Exception { - SolrJerseyResponse response = reloadCoreAPI.reloadCore(coreName, new ReloadCoreRequestBody()); + SolrJerseyResponse response = api.reloadCore(coreName, new ReloadCoreRequestBody()); assertEquals(0, response.responseHeader.status); } @@ -64,9 +63,7 @@ public void testNonExistentCoreExceptionResponse() { final SolrException solrException = expectThrows( SolrException.class, - () -> { - reloadCoreAPI.reloadCore(NON_EXISTENT_CORE, new ReloadCoreRequestBody()); - }); + () -> api.reloadCore(NON_EXISTENT_CORE, new ReloadCoreRequestBody())); assertEquals(400, solrException.code()); assertTrue(solrException.getMessage().contains("No such core: " + NON_EXISTENT_CORE)); } diff --git a/solr/core/src/test/org/apache/solr/handler/admin/api/ReplaceNodeAPITest.java b/solr/core/src/test/org/apache/solr/handler/admin/api/ReplaceNodeAPITest.java index a325da55e7d..2ac1998ddd4 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/api/ReplaceNodeAPITest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/api/ReplaceNodeAPITest.java @@ -18,110 +18,79 @@ import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyLong; -import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import java.util.Map; -import java.util.Optional; -import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.client.api.model.ReplaceNodeRequestBody; -import org.apache.solr.cloud.OverseerSolrResponse; -import org.apache.solr.cloud.ZkController; -import org.apache.solr.cloud.api.collections.DistributedCollectionConfigSetCommandRunner; +import org.apache.solr.cloud.api.collections.AdminCmdContext; import org.apache.solr.common.cloud.ZkNodeProps; -import org.apache.solr.common.util.NamedList; -import org.apache.solr.core.CoreContainer; -import org.apache.solr.request.SolrQueryRequest; -import org.apache.solr.response.SolrQueryResponse; +import org.apache.solr.common.params.CollectionParams; import org.junit.Before; -import org.junit.BeforeClass; import org.junit.Test; -import org.mockito.ArgumentCaptor; /** Unit tests for {@link ReplaceNode} */ -public class ReplaceNodeAPITest extends SolrTestCaseJ4 { +public class ReplaceNodeAPITest extends MockAPITest { - private CoreContainer mockCoreContainer; - private ZkController mockZkController; - private SolrQueryRequest mockQueryRequest; - private SolrQueryResponse queryResponse; - private ReplaceNode replaceNodeApi; - private DistributedCollectionConfigSetCommandRunner mockCommandRunner; - private ArgumentCaptor messageCapturer; - - @BeforeClass - public static void ensureWorkingMockito() { - assumeWorkingMockito(); - } + private ReplaceNode api; @Override @Before public void setUp() throws Exception { super.setUp(); - - mockCoreContainer = mock(CoreContainer.class); - mockZkController = mock(ZkController.class); - mockCommandRunner = mock(DistributedCollectionConfigSetCommandRunner.class); - when(mockCoreContainer.getZkController()).thenReturn(mockZkController); - when(mockZkController.getDistributedCommandRunner()).thenReturn(Optional.of(mockCommandRunner)); - when(mockCommandRunner.runCollectionCommand(any(), any(), anyLong())) - .thenReturn(new OverseerSolrResponse(new NamedList<>())); - mockQueryRequest = mock(SolrQueryRequest.class); - queryResponse = new SolrQueryResponse(); - replaceNodeApi = new ReplaceNode(mockCoreContainer, mockQueryRequest, queryResponse); - messageCapturer = ArgumentCaptor.forClass(ZkNodeProps.class); - when(mockCoreContainer.isZooKeeperAware()).thenReturn(true); + + api = new ReplaceNode(mockCoreContainer, mockQueryRequest, queryResponse); } @Test public void testCreatesValidOverseerMessage() throws Exception { final var requestBody = new ReplaceNodeRequestBody("demoTargetNode", false, "async"); - replaceNodeApi.replaceNode("demoSourceNode", requestBody); - verify(mockCommandRunner).runCollectionCommand(messageCapturer.capture(), any(), anyLong()); + api.replaceNode("demoSourceNode", requestBody); + verify(mockCommandRunner) + .runCollectionCommand(contextCapturer.capture(), messageCapturer.capture(), anyLong()); final ZkNodeProps createdMessage = messageCapturer.getValue(); final Map createdMessageProps = createdMessage.getProperties(); - assertEquals(5, createdMessageProps.size()); + assertEquals(3, createdMessageProps.size()); assertEquals("demoSourceNode", createdMessageProps.get("sourceNode")); assertEquals("demoTargetNode", createdMessageProps.get("targetNode")); assertEquals(false, createdMessageProps.get("waitForFinalState")); - assertEquals("async", createdMessageProps.get("async")); - assertEquals("replacenode", createdMessageProps.get("operation")); + final AdminCmdContext context = contextCapturer.getValue(); + assertEquals(CollectionParams.CollectionAction.REPLACENODE, context.getAction()); + assertEquals("async", context.getAsyncId()); } @Test public void testRequestBodyCanBeOmittedAltogether() throws Exception { - replaceNodeApi.replaceNode("demoSourceNode", null); - verify(mockCommandRunner).runCollectionCommand(messageCapturer.capture(), any(), anyLong()); + api.replaceNode("demoSourceNode", null); + verify(mockCommandRunner).runCollectionCommand(any(), messageCapturer.capture(), anyLong()); final ZkNodeProps createdMessage = messageCapturer.getValue(); final Map createdMessageProps = createdMessage.getProperties(); - assertEquals(2, createdMessageProps.size()); + assertEquals(1, createdMessageProps.size()); assertEquals("demoSourceNode", createdMessageProps.get("sourceNode")); - assertEquals("replacenode", createdMessageProps.get("operation")); } @Test public void testOptionalValuesNotAddedToRemoteMessageIfNotProvided() throws Exception { final var requestBody = new ReplaceNodeRequestBody("demoTargetNode", null, null); - replaceNodeApi.replaceNode("demoSourceNode", requestBody); - verify(mockCommandRunner).runCollectionCommand(messageCapturer.capture(), any(), anyLong()); + api.replaceNode("demoSourceNode", requestBody); + verify(mockCommandRunner) + .runCollectionCommand(contextCapturer.capture(), messageCapturer.capture(), anyLong()); final ZkNodeProps createdMessage = messageCapturer.getValue(); final Map createdMessageProps = createdMessage.getProperties(); - assertEquals(3, createdMessageProps.size()); + assertEquals(2, createdMessageProps.size()); assertEquals("demoSourceNode", createdMessageProps.get("sourceNode")); assertEquals("demoTargetNode", createdMessageProps.get("targetNode")); - assertEquals("replacenode", createdMessageProps.get("operation")); assertFalse( "Expected message to not contain value for waitForFinalState: " + createdMessageProps.get("waitForFinalState"), createdMessageProps.containsKey("waitForFinalState")); - assertFalse( - "Expected message to not contain value for async: " + createdMessageProps.get("async"), - createdMessageProps.containsKey("async")); + final AdminCmdContext context = contextCapturer.getValue(); + assertEquals(CollectionParams.CollectionAction.REPLACENODE, context.getAction()); + assertNull("asyncId should be null", context.getAsyncId()); } } diff --git a/solr/core/src/test/org/apache/solr/handler/admin/api/RestoreCollectionAPITest.java b/solr/core/src/test/org/apache/solr/handler/admin/api/RestoreCollectionAPITest.java index ce9833771f9..166bb6aa90f 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/api/RestoreCollectionAPITest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/api/RestoreCollectionAPITest.java @@ -17,7 +17,6 @@ package org.apache.solr.handler.admin.api; -import static org.apache.solr.cloud.Overseer.QUEUE_OPERATION; import static org.apache.solr.common.params.CollectionAdminParams.COLLECTION; import static org.apache.solr.common.params.CollectionAdminParams.COLL_CONF; import static org.apache.solr.common.params.CollectionAdminParams.CREATE_NODE_SET_PARAM; @@ -25,40 +24,52 @@ import static org.apache.solr.common.params.CollectionAdminParams.PULL_REPLICAS; import static org.apache.solr.common.params.CollectionAdminParams.REPLICATION_FACTOR; import static org.apache.solr.common.params.CollectionAdminParams.TLOG_REPLICAS; -import static org.apache.solr.common.params.CommonAdminParams.ASYNC; import static org.apache.solr.common.params.CoreAdminParams.BACKUP_ID; import static org.apache.solr.common.params.CoreAdminParams.BACKUP_LOCATION; import static org.apache.solr.common.params.CoreAdminParams.BACKUP_REPOSITORY; import static org.apache.solr.common.params.CoreAdminParams.NAME; import static org.hamcrest.Matchers.containsString; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import java.net.URI; import java.util.List; import java.util.Map; -import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.client.api.model.CreateCollectionRequestBody; import org.apache.solr.client.api.model.RestoreCollectionRequestBody; +import org.apache.solr.cloud.api.collections.AdminCmdContext; import org.apache.solr.common.SolrException; +import org.apache.solr.common.cloud.ZkNodeProps; +import org.apache.solr.common.params.CollectionParams; import org.apache.solr.common.params.ModifiableSolrParams; -import org.apache.solr.common.util.NamedList; -import org.apache.solr.core.CoreContainer; -import org.apache.solr.core.NodeConfig; -import org.apache.solr.request.LocalSolrQueryRequest; -import org.junit.BeforeClass; +import org.apache.solr.core.backup.repository.BackupRepository; +import org.junit.Before; import org.junit.Test; /** Unit tests for {@link RestoreCollection} */ -public class RestoreCollectionAPITest extends SolrTestCaseJ4 { - - private static RestoreCollection restoreCollectionAPI; - - @BeforeClass - public static void setUpApi() { - restoreCollectionAPI = - new RestoreCollection( - new CoreContainer( - new NodeConfig.NodeConfigBuilder("testnode", createTempDir()).build()), - new LocalSolrQueryRequest(null, new NamedList<>()), - null); +public class RestoreCollectionAPITest extends MockAPITest { + + private static RestoreCollection api; + private BackupRepository mockBackupRepository; + + @Before + public void setUp() throws Exception { + super.setUp(); + when(mockCoreContainer.isZooKeeperAware()).thenReturn(true); + + mockBackupRepository = mock(BackupRepository.class); + when(mockCoreContainer.newBackupRepository(eq("someRepository"))) + .thenReturn(mockBackupRepository); + when(mockCoreContainer.newBackupRepository(null)).thenReturn(mockBackupRepository); + when(mockBackupRepository.getBackupLocation(eq("someLocation"))).thenReturn("someLocation"); + URI uri = new URI("someLocation"); + when(mockBackupRepository.createDirectoryURI(eq("someLocation"))).thenReturn(uri); + when(mockBackupRepository.exists(eq(uri))).thenReturn(true); + + api = new RestoreCollection(mockCoreContainer, mockQueryRequest, queryResponse); } @Test @@ -66,11 +77,7 @@ public void testReportsErrorIfBackupNameMissing() { final var requestBody = new RestoreCollectionRequestBody(); requestBody.collection = "someCollection"; final SolrException thrown = - expectThrows( - SolrException.class, - () -> { - restoreCollectionAPI.restoreCollection(null, requestBody); - }); + expectThrows(SolrException.class, () -> api.restoreCollection(null, requestBody)); assertEquals(400, thrown.code()); assertEquals("Required parameter 'backupName' missing", thrown.getMessage()); @@ -79,11 +86,7 @@ public void testReportsErrorIfBackupNameMissing() { @Test public void testReportsErrorIfRequestBodyMissing() { final SolrException thrown = - expectThrows( - SolrException.class, - () -> { - restoreCollectionAPI.restoreCollection("someBackupName", null); - }); + expectThrows(SolrException.class, () -> api.restoreCollection("someBackupName", null)); assertEquals(400, thrown.code()); assertEquals("Missing required request body", thrown.getMessage()); @@ -95,10 +98,7 @@ public void testReportsErrorIfCollectionNameMissing() { final var requestBody = new RestoreCollectionRequestBody(); final SolrException thrown = expectThrows( - SolrException.class, - () -> { - restoreCollectionAPI.restoreCollection("someBackupName", requestBody); - }); + SolrException.class, () -> api.restoreCollection("someBackupName", requestBody)); assertEquals(400, thrown.code()); assertEquals("Required parameter 'collection' missing", thrown.getMessage()); @@ -110,10 +110,7 @@ public void testReportsErrorIfProvidedCollectionNameIsInvalid() { requestBody.collection = "invalid$collection@name"; final SolrException thrown = expectThrows( - SolrException.class, - () -> { - restoreCollectionAPI.restoreCollection("someBackupName", requestBody); - }); + SolrException.class, () -> api.restoreCollection("someBackupName", requestBody)); assertEquals(400, thrown.code()); assertThat( @@ -121,34 +118,41 @@ public void testReportsErrorIfProvidedCollectionNameIsInvalid() { } @Test - public void testCreatesValidRemoteMessageForExistingCollectionRestore() { + public void testCreatesValidRemoteMessageForExistingCollectionRestore() throws Exception { final var requestBody = new RestoreCollectionRequestBody(); requestBody.collection = "someCollectionName"; - requestBody.location = "/some/location/path"; + requestBody.location = "someLocation"; requestBody.backupId = 123; - requestBody.repository = "someRepositoryName"; + requestBody.repository = "someRepository"; requestBody.async = "someAsyncId"; - final var remoteMessage = - restoreCollectionAPI.createRemoteMessage("someBackupName", requestBody).getProperties(); + when(mockClusterState.hasCollection(eq("someCollectionName"))).thenReturn(true); + api.restoreCollection("someBackupName", requestBody); + verify(mockCommandRunner) + .runCollectionCommand(contextCapturer.capture(), messageCapturer.capture(), anyLong()); + + final ZkNodeProps createdMessage = messageCapturer.getValue(); + final Map remoteMessage = createdMessage.getProperties(); - assertEquals(7, remoteMessage.size()); - assertEquals("restore", remoteMessage.get(QUEUE_OPERATION)); + assertEquals(5, remoteMessage.size()); assertEquals("someCollectionName", remoteMessage.get(COLLECTION)); - assertEquals("/some/location/path", remoteMessage.get(BACKUP_LOCATION)); + assertEquals("someLocation", remoteMessage.get(BACKUP_LOCATION)); assertEquals(Integer.valueOf(123), remoteMessage.get(BACKUP_ID)); - assertEquals("someRepositoryName", remoteMessage.get(BACKUP_REPOSITORY)); - assertEquals("someAsyncId", remoteMessage.get(ASYNC)); + assertEquals("someRepository", remoteMessage.get(BACKUP_REPOSITORY)); assertEquals("someBackupName", remoteMessage.get(NAME)); + + AdminCmdContext context = contextCapturer.getValue(); + assertEquals(CollectionParams.CollectionAction.RESTORE, context.getAction()); + assertEquals("someAsyncId", context.getAsyncId()); } @Test - public void testCreatesValidRemoteMessageForNewCollectionRestore() { + public void testCreatesValidRemoteMessageForNewCollectionRestore() throws Exception { final var requestBody = new RestoreCollectionRequestBody(); requestBody.collection = "someCollectionName"; - requestBody.location = "/some/location/path"; + requestBody.location = "someLocation"; requestBody.backupId = 123; - requestBody.repository = "someRepositoryName"; + requestBody.repository = "someRepository"; requestBody.async = "someAsyncId"; final var createParams = new CreateCollectionRequestBody(); requestBody.createCollectionParams = createParams; @@ -159,16 +163,19 @@ public void testCreatesValidRemoteMessageForNewCollectionRestore() { createParams.nodeSet = List.of("node1", "node2"); createParams.properties = Map.of("foo", "bar"); - final var remoteMessage = - restoreCollectionAPI.createRemoteMessage("someBackupName", requestBody).getProperties(); + when(mockClusterState.hasCollection(eq("someCollectionName"))).thenReturn(true); + api.restoreCollection("someBackupName", requestBody); + verify(mockCommandRunner) + .runCollectionCommand(contextCapturer.capture(), messageCapturer.capture(), anyLong()); + + final ZkNodeProps createdMessage = messageCapturer.getValue(); + final Map remoteMessage = createdMessage.getProperties(); - assertEquals(14, remoteMessage.size()); - assertEquals("restore", remoteMessage.get(QUEUE_OPERATION)); + assertEquals(12, remoteMessage.size()); assertEquals("someCollectionName", remoteMessage.get(COLLECTION)); - assertEquals("/some/location/path", remoteMessage.get(BACKUP_LOCATION)); + assertEquals("someLocation", remoteMessage.get(BACKUP_LOCATION)); assertEquals(Integer.valueOf(123), remoteMessage.get(BACKUP_ID)); - assertEquals("someRepositoryName", remoteMessage.get(BACKUP_REPOSITORY)); - assertEquals("someAsyncId", remoteMessage.get(ASYNC)); + assertEquals("someRepository", remoteMessage.get(BACKUP_REPOSITORY)); assertEquals("someBackupName", remoteMessage.get(NAME)); assertEquals("someConfig", remoteMessage.get(COLL_CONF)); assertEquals(Integer.valueOf(123), remoteMessage.get(NRT_REPLICAS)); @@ -177,6 +184,10 @@ public void testCreatesValidRemoteMessageForNewCollectionRestore() { assertEquals(Integer.valueOf(789), remoteMessage.get(PULL_REPLICAS)); assertEquals("node1,node2", remoteMessage.get(CREATE_NODE_SET_PARAM)); assertEquals("bar", remoteMessage.get("property.foo")); + + AdminCmdContext context = contextCapturer.getValue(); + assertEquals(CollectionParams.CollectionAction.RESTORE, context.getAction()); + assertEquals("someAsyncId", context.getAsyncId()); } @Test diff --git a/solr/core/src/test/org/apache/solr/handler/admin/api/SyncShardAPITest.java b/solr/core/src/test/org/apache/solr/handler/admin/api/SyncShardAPITest.java index 23d3f8982a2..e9c605de183 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/api/SyncShardAPITest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/api/SyncShardAPITest.java @@ -17,21 +17,30 @@ package org.apache.solr.handler.admin.api; -import org.apache.solr.SolrTestCaseJ4; +import static org.mockito.Mockito.when; + import org.apache.solr.common.SolrException; +import org.junit.Before; import org.junit.Test; /** Unit tests for {@link SyncShard} */ -public class SyncShardAPITest extends SolrTestCaseJ4 { +public class SyncShardAPITest extends MockAPITest { + + private SyncShard api; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + when(mockCoreContainer.isZooKeeperAware()).thenReturn(true); + + api = new SyncShard(mockCoreContainer, mockQueryRequest, queryResponse); + } + @Test public void testReportsErrorIfCollectionNameMissing() { final SolrException thrown = - expectThrows( - SolrException.class, - () -> { - final var api = new SyncShard(null, null, null); - api.syncShard(null, "someShard"); - }); + expectThrows(SolrException.class, () -> api.syncShard(null, "someShard")); assertEquals(400, thrown.code()); assertEquals("Missing required parameter: collection", thrown.getMessage()); @@ -40,12 +49,7 @@ public void testReportsErrorIfCollectionNameMissing() { @Test public void testReportsErrorIfShardNameMissing() { final SolrException thrown = - expectThrows( - SolrException.class, - () -> { - final var api = new SyncShard(null, null, null); - api.syncShard("someCollection", null); - }); + expectThrows(SolrException.class, () -> api.syncShard("someCollection", null)); assertEquals(400, thrown.code()); assertEquals("Missing required parameter: shard", thrown.getMessage()); diff --git a/solr/core/src/test/org/apache/solr/handler/admin/api/UnloadCoreAPITest.java b/solr/core/src/test/org/apache/solr/handler/admin/api/UnloadCoreAPITest.java index c823f33dea1..c2c8a3b126e 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/api/UnloadCoreAPITest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/api/UnloadCoreAPITest.java @@ -30,7 +30,7 @@ import org.junit.Test; public class UnloadCoreAPITest extends SolrTestCaseJ4 { - private UnloadCore unloadCoreAPI; + private UnloadCore api; private static final String NON_EXISTENT_CORE = "non_existent_core"; @BeforeClass @@ -48,13 +48,12 @@ public void setUp() throws Exception { CoreContainer coreContainer = h.getCoreContainer(); CoreAdminHandler.CoreAdminAsyncTracker coreAdminAsyncTracker = new CoreAdminHandler.CoreAdminAsyncTracker(); - unloadCoreAPI = - new UnloadCore(coreContainer, coreAdminAsyncTracker, solrQueryRequest, solrQueryResponse); + api = new UnloadCore(coreContainer, coreAdminAsyncTracker, solrQueryRequest, solrQueryResponse); } @Test public void testValidUnloadCoreAPIResponse() throws Exception { - SolrJerseyResponse response = unloadCoreAPI.unloadCore(coreName, getUnloadCoreRequestBodyObj()); + SolrJerseyResponse response = api.unloadCore(coreName, getUnloadCoreRequestBodyObj()); assertEquals(0, response.responseHeader.status); } @@ -63,9 +62,7 @@ public void testNonExistentCoreExceptionResponse() { final SolrException solrException = expectThrows( SolrException.class, - () -> { - unloadCoreAPI.unloadCore(NON_EXISTENT_CORE, getUnloadCoreRequestBodyObj()); - }); + () -> api.unloadCore(NON_EXISTENT_CORE, getUnloadCoreRequestBodyObj())); assertEquals(400, solrException.code()); assertTrue(solrException.getMessage().contains("Cannot unload non-existent core")); } diff --git a/solr/core/src/test/org/apache/solr/handler/admin/api/V2CollectionBackupApiTest.java b/solr/core/src/test/org/apache/solr/handler/admin/api/V2CollectionBackupApiTest.java index 514423ba5df..535bf774743 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/api/V2CollectionBackupApiTest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/api/V2CollectionBackupApiTest.java @@ -16,18 +16,43 @@ */ package org.apache.solr.handler.admin.api; -import static org.apache.solr.cloud.Overseer.QUEUE_OPERATION; import static org.apache.solr.common.params.CollectionAdminParams.COPY_FILES_STRATEGY; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; -import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.client.api.model.CreateCollectionBackupRequestBody; +import org.apache.solr.cloud.api.collections.AdminCmdContext; +import org.apache.solr.common.params.CollectionParams; import org.apache.solr.common.params.ModifiableSolrParams; +import org.apache.solr.core.backup.repository.BackupRepository; +import org.junit.Before; import org.junit.Test; /** Unit tests for {@link CreateCollectionBackup} */ -public class V2CollectionBackupApiTest extends SolrTestCaseJ4 { +public class V2CollectionBackupApiTest extends MockAPITest { + + private CreateCollectionBackup api; + private BackupRepository mockBackupRepository; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + when(mockCoreContainer.isZooKeeperAware()).thenReturn(true); + + mockBackupRepository = mock(BackupRepository.class); + when(mockCoreContainer.newBackupRepository("someRepoName")).thenReturn(mockBackupRepository); + when(mockCoreContainer.newBackupRepository(null)).thenReturn(mockBackupRepository); + + api = new CreateCollectionBackup(mockCoreContainer, mockQueryRequest, queryResponse); + } + @Test - public void testCreateRemoteMessageWithAllProperties() { + public void testCreateRemoteMessageWithAllProperties() throws Exception { + final var requestBody = new CreateCollectionBackupRequestBody(); requestBody.location = "/some/location"; requestBody.repository = "someRepoName"; @@ -38,12 +63,18 @@ public void testCreateRemoteMessageWithAllProperties() { requestBody.maxNumBackupPoints = 123; requestBody.async = "someId"; - var message = - CreateCollectionBackup.createRemoteMessage( - "someCollectionName", "someBackupName", requestBody); - var messageProps = message.getProperties(); + when(mockClusterState.hasCollection("someCollectionName")).thenReturn(true); + when(mockBackupRepository.getBackupLocation(requestBody.location)) + .thenReturn(requestBody.location); + when(mockBackupRepository.exists(any())).thenReturn(true); + + api.createCollectionBackup("someCollectionName", "someBackupName", requestBody); + verify(mockCommandRunner) + .runCollectionCommand(contextCapturer.capture(), messageCapturer.capture(), anyLong()); - assertEquals(11, messageProps.size()); + var message = messageCapturer.getValue(); + var messageProps = message.getProperties(); + assertEquals(messageProps.toString(), 9, messageProps.size()); assertEquals("someCollectionName", messageProps.get("collection")); assertEquals("/some/location", messageProps.get("location")); assertEquals("someRepoName", messageProps.get("repository")); @@ -52,30 +83,45 @@ public void testCreateRemoteMessageWithAllProperties() { assertEquals("someSnapshotName", messageProps.get("commitName")); assertEquals(true, messageProps.get("incremental")); assertEquals(123, messageProps.get("maxNumBackupPoints")); - assertEquals("someId", messageProps.get("async")); - assertEquals("backup", messageProps.get(QUEUE_OPERATION)); assertEquals("someBackupName", messageProps.get("name")); + + AdminCmdContext context = contextCapturer.getValue(); + assertEquals(CollectionParams.CollectionAction.BACKUP, context.getAction()); + assertEquals("someId", context.getAsyncId()); } @Test - public void testCreateRemoteMessageOmitsNullValues() { + public void testCreateRemoteMessageOmitsNullValues() throws Exception { final var requestBody = new CreateCollectionBackupRequestBody(); requestBody.location = "/some/location"; - var message = - CreateCollectionBackup.createRemoteMessage( - "someCollectionName", "someBackupName", requestBody); - var messageProps = message.getProperties(); + when(mockClusterState.hasCollection("someCollectionName")).thenReturn(true); + when(mockBackupRepository.getBackupLocation(requestBody.location)) + .thenReturn(requestBody.location); + when(mockBackupRepository.exists(any())).thenReturn(true); - assertEquals(4, messageProps.size()); + api.createCollectionBackup("someCollectionName", "someBackupName", requestBody); + verify(mockCommandRunner) + .runCollectionCommand(contextCapturer.capture(), messageCapturer.capture(), anyLong()); + + var message = messageCapturer.getValue(); + var messageProps = message.getProperties(); + assertEquals(5, messageProps.size()); assertEquals("someCollectionName", messageProps.get("collection")); assertEquals("/some/location", messageProps.get("location")); - assertEquals("backup", messageProps.get(QUEUE_OPERATION)); assertEquals("someBackupName", messageProps.get("name")); + assertEquals(true, messageProps.get("incremental")); + assertEquals("copy-files", messageProps.get("indexBackup")); + + AdminCmdContext context = contextCapturer.getValue(); + assertEquals(CollectionParams.CollectionAction.BACKUP, context.getAction()); + assertNull(context.getAsyncId()); } @Test public void testCanCreateV2RequestBodyFromV1Params() { + when(mockClusterState.hasCollection("demoSourceNode")).thenReturn(true); + final var params = new ModifiableSolrParams(); params.set("collection", "someCollectionName"); params.set("location", "/some/location"); diff --git a/solr/modules/s3-repository/src/test/org/apache/solr/s3/S3IncrementalBackupTest.java b/solr/modules/s3-repository/src/test/org/apache/solr/s3/S3IncrementalBackupTest.java index 80c5207505b..a34fb0ac973 100644 --- a/solr/modules/s3-repository/src/test/org/apache/solr/s3/S3IncrementalBackupTest.java +++ b/solr/modules/s3-repository/src/test/org/apache/solr/s3/S3IncrementalBackupTest.java @@ -22,6 +22,7 @@ import java.lang.invoke.MethodHandles; import org.apache.lucene.tests.util.LuceneTestCase; import org.apache.solr.cloud.api.collections.AbstractIncrementalBackupTest; +import org.apache.solr.util.LogLevel; import org.junit.BeforeClass; import org.junit.ClassRule; import org.slf4j.Logger; @@ -31,6 +32,9 @@ // Backups do checksum validation against a footer value not present in 'SimpleText' @LuceneTestCase.SuppressCodecs({"SimpleText"}) @ThreadLeakLingering(linger = 10) +@LogLevel( + value = + "org.apache.solr.cloud=DEBUG;org.apache.solr.cloud.api.collections=DEBUG;org.apache.solr.cloud.overseer=DEBUG") public class S3IncrementalBackupTest extends AbstractIncrementalBackupTest { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); diff --git a/solr/solrj/src/java/org/apache/solr/common/params/CollectionAdminParams.java b/solr/solrj/src/java/org/apache/solr/common/params/CollectionAdminParams.java index 74a8e5dd869..629e56384aa 100644 --- a/solr/solrj/src/java/org/apache/solr/common/params/CollectionAdminParams.java +++ b/solr/solrj/src/java/org/apache/solr/common/params/CollectionAdminParams.java @@ -124,4 +124,6 @@ public interface CollectionAdminParams { String PROPERTY_PREFIX = "property."; String PER_REPLICA_STATE = CollectionStateProps.PER_REPLICA_STATE; + + String CALLING_LOCK_IDS_HEADER = "callingLockIds"; } diff --git a/solr/solrj/src/java/org/apache/solr/common/params/CollectionParams.java b/solr/solrj/src/java/org/apache/solr/common/params/CollectionParams.java index 6ae82508df4..82d9f80909b 100644 --- a/solr/solrj/src/java/org/apache/solr/common/params/CollectionParams.java +++ b/solr/solrj/src/java/org/apache/solr/common/params/CollectionParams.java @@ -48,7 +48,8 @@ enum LockLevel { REPLICA(3, null), SHARD(2, REPLICA), COLLECTION(1, SHARD), - CLUSTER(0, COLLECTION); + CLUSTER(0, COLLECTION), + BASE(-1, CLUSTER); private final int height; private final LockLevel child; diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudHttp2SolrClientTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudHttp2SolrClientTest.java index e80c60242a7..3531bc0cd08 100644 --- a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudHttp2SolrClientTest.java +++ b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudHttp2SolrClientTest.java @@ -581,7 +581,10 @@ private void queryWithShardsPreferenceRules(CloudSolrClient cloudClient, String // And since all the nodes are hosting cores from all shards, the // distributed query formed by this node will select cores from the // local shards only - QueryResponse qResponse = cloudClient.query(collectionName, qRequest); + QueryResponse qResponse = null; + for (int i = 0; i < 100; i++) { + qResponse = cloudClient.query(collectionName, qRequest); + } Object shardsInfo = qResponse.getResponse().get(ShardParams.SHARDS_INFO); assertNotNull("Unable to obtain " + ShardParams.SHARDS_INFO, shardsInfo); diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/SolrCloudTestCase.java b/solr/test-framework/src/java/org/apache/solr/cloud/SolrCloudTestCase.java index 7f589e48b23..1066d12e893 100644 --- a/solr/test-framework/src/java/org/apache/solr/cloud/SolrCloudTestCase.java +++ b/solr/test-framework/src/java/org/apache/solr/cloud/SolrCloudTestCase.java @@ -26,6 +26,7 @@ import java.lang.annotation.RetentionPolicy; import java.lang.annotation.Target; import java.lang.invoke.MethodHandles; +import java.time.Duration; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -47,7 +48,9 @@ import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.apache.CloudLegacySolrClient; import org.apache.solr.client.solrj.apache.HttpSolrClient; +import org.apache.solr.client.solrj.request.CollectionAdminRequest; import org.apache.solr.client.solrj.request.CoreAdminRequest; +import org.apache.solr.client.solrj.response.RequestStatusState; import org.apache.solr.common.cloud.CollectionStatePredicate; import org.apache.solr.common.cloud.DocCollection; import org.apache.solr.common.cloud.LiveNodesPredicate; @@ -57,7 +60,9 @@ import org.apache.solr.common.cloud.ZkStateReader; import org.apache.solr.common.util.EnvUtils; import org.apache.solr.common.util.NamedList; +import org.apache.solr.common.util.TimeSource; import org.apache.solr.embedded.JettySolrRunner; +import org.apache.solr.util.TimeOut; import org.junit.AfterClass; import org.junit.Before; import org.slf4j.Logger; @@ -415,6 +420,25 @@ protected static CoreStatusResponse.SingleCoreData getCoreStatus(Replica replica } } + protected CollectionAdminRequest.RequestStatusResponse waitForAsyncClusterRequest( + String asyncId, Duration timeout) + throws SolrServerException, IOException, InterruptedException { + CollectionAdminRequest.RequestStatus requestStatus = + CollectionAdminRequest.requestStatus(asyncId); + CollectionAdminRequest.RequestStatusResponse rsp = null; + TimeOut timeoutCheck = new TimeOut(timeout, TimeSource.NANO_TIME); + while (!timeoutCheck.hasTimedOut()) { + rsp = requestStatus.process(cluster.getSolrClient()); + if (rsp.getRequestStatus() == RequestStatusState.FAILED + || rsp.getRequestStatus() == RequestStatusState.COMPLETED) { + return rsp; + } + Thread.sleep(50); + } + fail("Async request " + asyncId + " did not complete within duration: " + timeout.toString()); + return rsp; + } + @SuppressWarnings({"rawtypes"}) protected NamedList waitForResponse( Predicate predicate, diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/api/collections/AbstractIncrementalBackupTest.java b/solr/test-framework/src/java/org/apache/solr/cloud/api/collections/AbstractIncrementalBackupTest.java index 0df483d8b35..79bb122a8f8 100644 --- a/solr/test-framework/src/java/org/apache/solr/cloud/api/collections/AbstractIncrementalBackupTest.java +++ b/solr/test-framework/src/java/org/apache/solr/cloud/api/collections/AbstractIncrementalBackupTest.java @@ -71,6 +71,7 @@ import org.apache.solr.core.backup.ShardBackupMetadata; import org.apache.solr.core.backup.repository.BackupRepository; import org.apache.solr.embedded.JettySolrRunner; +import org.apache.solr.util.LogLevel; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; @@ -84,6 +85,9 @@ *

For a similar test harness for snapshot backup/restoration see {@link * AbstractCloudBackupRestoreTestCase} */ +@LogLevel( + value = + "org.apache.solr.cloud=DEBUG;org.apache.solr.cloud.api.collections=DEBUG;org.apache.solr.cloud.overseer=DEBUG") public abstract class AbstractIncrementalBackupTest extends SolrCloudTestCase { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); @@ -127,6 +131,9 @@ public void setTestSuffix(String testSuffix) { public abstract String getBackupLocation(); @Test + @LogLevel( + value = + "org.apache.solr.cloud=DEBUG;org.apache.solr.cloud.api.collections=DEBUG;org.apache.solr.cloud.overseer=DEBUG") public void testSimple() throws Exception { setTestSuffix("testbackupincsimple"); final String backupCollectionName = getCollectionName(); @@ -193,6 +200,9 @@ public void testSimple() throws Exception { } @Test + @LogLevel( + value = + "org.apache.solr.cloud=DEBUG;org.apache.solr.cloud.api.collections=DEBUG;org.apache.solr.cloud.overseer=DEBUG") public void testRestoreToOriginalCollection() throws Exception { setTestSuffix("testbackuprestoretooriginal"); final String backupCollectionName = getCollectionName(); @@ -355,6 +365,9 @@ public void testBackupIncremental() throws Exception { } @Test + @LogLevel( + value = + "org.apache.solr.cloud=DEBUG;org.apache.solr.cloud.api.collections=DEBUG;org.apache.solr.cloud.overseer=DEBUG") public void testSkipConfigset() throws Exception { setTestSuffix("testskipconfigset"); final String backupCollectionName = getCollectionName();