SERVER-78918 Make shardCollection command shard authoritative

Author: PolPinol

buildscripts/resmokeconfig/fully_disabled_feature_flags.yml

@@ -11,7 +11,6 @@
 # its targeted tests should enable it.
 - featureFlagAlwaysCreateConfigTransactionsPartialIndexOnStepUp
 - featureFlagUpdateDocumentShardKeyUsingTransactionApi
-- featureFlagAuthoritativeShardCollection
 - featureFlagAllMongodsAreSharded
 - featureFlagGRPC
 - featureFlagReplicaSetEndpoint

jstests/sharding/update_replace_id.js

@@ -65,15 +65,31 @@ function setUpData() {
 }
 
 function runReplacementUpdateTestsForHashedShardKey() {
-    // Make sure the chunk containing key {_id: -100} is on shard0.
+    // Make sure the chunk containing key {_id: -100} is on shard0; the chunk is migrated twice to
+    // ensure that the shard CSR has also a not-UNKOWN version. This will allow direct writes
+    // performed later in this test to work because the filtering metadata are set.
+    assert.commandWorked(mongosDB.adminCommand({
+        moveChunk: mongosColl.getFullName(),
+        find: {_id: -100},
+        to: st.shard1.shardName,
+        _waitForDelete: true
+    }));
     assert.commandWorked(mongosDB.adminCommand({
         moveChunk: mongosColl.getFullName(),
         find: {_id: -100},
         to: st.shard0.shardName,
         _waitForDelete: true
     }));
 
-    // Make sure the chunk containing key {_id: -101} is on shard1.
+    // Make sure the chunk containing key {_id: 101} is on shard1; the chunk is migrated twice to
+    // ensure that the shard CSR has also a not-UNKOWN version. This will allow direct writes
+    // performed later in this test to work because the filtering metadata are set.
+    assert.commandWorked(mongosDB.adminCommand({
+        moveChunk: mongosColl.getFullName(),
+        find: {_id: 101},
+        to: st.shard0.shardName,
+        _waitForDelete: true
+    }));
     assert.commandWorked(mongosDB.adminCommand({
         moveChunk: mongosColl.getFullName(),
         find: {_id: 101},

src/mongo/db/commands/set_feature_compatibility_version_command.cpp

@@ -644,8 +644,6 @@ class SetFeatureCompatibilityVersionCommand : public BasicCommand {
         // TODO (SERVER-77915): Remove once 8.0 (trackUnshardedCollections) becomes lastLTS.
         if (isDowngrading &&
             (feature_flags::gAuthoritativeShardCollection
-                 .isDisabledOnTargetFCVButEnabledOnOriginalFCV(requestedVersion, originalVersion) ||
-             feature_flags::gTrackUnshardedCollectionsOnShardingCatalog
                  .isDisabledOnTargetFCVButEnabledOnOriginalFCV(requestedVersion,
                                                                originalVersion))) {
             ShardingDDLCoordinatorService::getService(opCtx)
@@ -1618,7 +1616,7 @@ class SetFeatureCompatibilityVersionCommand : public BasicCommand {
             feature_flags::gAuthoritativeShardCollection.isEnabledOnVersion(requestedVersion)) {
             ShardingDDLCoordinatorService::getService(opCtx)
                 ->waitForCoordinatorsOfGivenTypeToComplete(
-                    opCtx, DDLCoordinatorTypeEnum::kCreateCollectionPre71Compatible);
+                    opCtx, DDLCoordinatorTypeEnum::kCreateCollectionPre73Compatible);
         }
         _maybeRemoveOldAuditConfig(opCtx, requestedVersion);
 

src/mongo/db/s/create_collection_coordinator.cpp

@@ -88,6 +88,7 @@ class CreateCollectionResponseProvider {
     virtual ~CreateCollectionResponseProvider() {}
 };
 
+// TODO (SERVER-79304): Remove once 8.0 becomes last LTS.
 class CreateCollectionCoordinatorLegacy
     : public RecoverableShardingDDLCoordinator<CreateCollectionCoordinatorDocumentLegacy,
                                                CreateCollectionCoordinatorPhaseLegacyEnum>,
@@ -185,33 +186,54 @@ class CreateCollectionCoordinator
         return CreateCollectionCoordinatorPhase_serializer(phase);
     }
 
+    bool _mustAlwaysMakeProgress() override;
+
     ExecutorFuture<void> _runImpl(std::shared_ptr<executor::ScopedTaskExecutor> executor,
                                   const CancellationToken& token) noexcept override;
 
     ExecutorFuture<void> _cleanupOnAbort(std::shared_ptr<executor::ScopedTaskExecutor> executor,
                                          const CancellationToken& token,
                                          const Status& status) noexcept override;
 
-    /**
-     * Acquires critical sections on all shards.
-     */
-    void _acquireCriticalSectionsOnParticipants(
-        OperationContext* opCtx,
-        std::shared_ptr<executor::ScopedTaskExecutor> executor,
-        const CancellationToken& token);
+    // Check the command arguments passed and validate that the collection has not been tracked from
+    // another request.
+    void _checkPreconditions();
 
-    /**
-     * Releases critical sections on all shards.
-     */
-    void _releaseCriticalSectionsOnParticipants(
-        OperationContext* opCtx,
-        std::shared_ptr<executor::ScopedTaskExecutor> executor,
-        const CancellationToken& token);
+    // Enter to the critical section on the coordinator for the namespace and its buckets namespace.
+    // Only blocks writes.
+    void _enterWriteCriticalSectionOnCoordinator();
 
-    /**
-     * Functions to encapsulate each phase
-     */
-    void _checkPreconditions(std::shared_ptr<executor::ScopedTaskExecutor> executor);
+    // Translate the request parameters and persist them in the coordinator document.
+    void _translateRequestParameters();
+
+    // Ensure that the collection is created locally and build the shard key index if necessary.
+    void _createCollectionOnCoordinator();
+
+    // Enter to the critical section on all the shards. Blocks writes and reads.
+    void _enterCriticalSection(const std::shared_ptr<executor::ScopedTaskExecutor>& executor,
+                               const CancellationToken& token);
+
+    // Broadcast create collection to the other shards.
+    void _createCollectionOnParticipants(
+        const std::shared_ptr<executor::ScopedTaskExecutor>& executor);
+
+    // Commits the create collection operation to the sharding catalog within a transaction. After
+    // that, it clears the filtering metadata on the primary shard.
+    void _commitOnShardingCatalog(const std::shared_ptr<executor::ScopedTaskExecutor>& executor);
+
+    // Exit from the critical section on all the shards, unblocking reads and writes. On the
+    // participant shards, it is set to clear the filtering metadata after exiting the critical
+    // section.
+    void _exitCriticalSection(const std::shared_ptr<executor::ScopedTaskExecutor>& executor,
+                              const CancellationToken& token);
+
+    // Exit critical sections on participant shards.
+    void _exitCriticalSectionsOnParticipants(OperationContext* opCtx,
+                                             bool throwIfReasonDiffers,
+                                             std::shared_ptr<executor::ScopedTaskExecutor> executor,
+                                             const CancellationToken& token);
+
+    bool _validateCreateCollectionAlreadyCommitted(OperationContext* opCtx);
 
     mongo::ShardsvrCreateCollectionRequest _request;
 
@@ -220,13 +242,11 @@ class CreateCollectionCoordinator
     // Set on successful completion of the coordinator
     boost::optional<CreateCollectionResponse> _result;
 
-    // The fields below are only populated if the coordinator enters in the branch where the
-    // collection is not already sharded (i.e., they will not be present on early return)
-
-    boost::optional<UUID> _collectionUUID;
-    std::unique_ptr<InitialSplitPolicy> _splitPolicy;
-    boost::optional<InitialSplitPolicy::ShardCollectionConfig> _initialChunks;
+    // The fields below are only populated for the first execution, they will not be present if it
+    // is not the first run.
+    boost::optional<UUID> _uuid;
     boost::optional<bool> _collectionEmpty;
+    boost::optional<InitialSplitPolicy::ShardCollectionConfig> _initialChunks;
 };
 
 }  // namespace mongo

src/mongo/db/s/create_collection_coordinator.h

@@ -38,22 +38,26 @@ imports:
     - "mongo/s/request_types/sharded_ddl_commands.idl"
 
 enums:
+    # TODO (SERVER-79304): Remove once 8.0 becomes last LTS.
     CreateCollectionCoordinatorPhaseLegacy:
         description: "Current create collection coordinator's operation state."
         type: string
         values:
             kUnset: "unset"
             kCommit: "commit"
+    
     CreateCollectionCoordinatorPhase:
         description: "Current create collection coordinator's operation state."
         type: string
         values:
             kUnset: "unset"
-            kAcquireCSOnCoordinator: "acquireCSOnCoordinator"
-            kTranslateRequest: "translateRequest"
-            kAcquireCSOnParticipants: "acquireCSOnParticipants"
-            kCommit: "commit"
-            kReleaseAllCS: "releaseAllCS"
+            kEnterWriteCriticalSectionOnCoordinator: "enterWriteCriticalSectionOnCoordinator"
+            kTranslateRequestParameters: "translateRequestParameters"
+            kCreateCollectionOnCoordinator: "createCollectionOnCoordinator"
+            kEnterCriticalSection: "enterCriticalSection"
+            kCreateCollectionOnParticipants: "createCollectionOnParticipants"
+            kCommitOnShardingCatalog: "commitOnShardingCatalog"
+            kExitCriticalSection: "exitCriticalSection"
 
 structs:
     TranslatedRequestParams:
@@ -74,8 +78,9 @@ structs:
                 description: "Collation value from the targeted unsharded collection if this exists; otherwise, simple-collation"
                 optional: false
 
+    # TODO (SERVER-79304): Remove once 8.0 becomes last LTS.
     CreateCollectionCoordinatorDocumentLegacy:
-        description: "Object with neccessary fields to create a collection"
+        description: "Object with necessary fields to create a collection"
         generate_comparison_operators: false
         strict: false
         chained_structs:
@@ -90,6 +95,7 @@ structs:
                 type: TranslatedRequestParams
                 description: "The field is populated only once the kTranslateRequest phase is completed"
                 optional: true
+
     CreateCollectionCoordinatorDocument:
         description: "Object with neccessary fields to create a collection"
         generate_comparison_operators: false
@@ -104,5 +110,9 @@ structs:
                 default: kUnset
             translatedRequestParams:
                 type: TranslatedRequestParams
-                description: "The field is populated only once the kTranslateRequest phase is completed"
+                description: "The field is populated only once the kTranslateRequest phase is completed."
+                optional: true
+            shardIds:
+                type: array<shard_id>
+                description: "The list of shards involved in the create collection operation."
                 optional: true

src/mongo/db/s/create_collection_coordinator_document.idl

@@ -49,7 +49,8 @@ enums:
             kDropCollection: "dropCollection_V2"
             kRenameCollection: "renameCollection_V2"
             kCreateCollection: "createCollection_V4"
-            kCreateCollectionPre71Compatible: "createCollection_V3"
+            # TODO SERVER-79304: Remove once 8.0 becomes last LTS
+            kCreateCollectionPre73Compatible: "createCollection_V3"
             # TODO SERVER-79064: Remove once 8.0 becomes last LTS
             kRefineCollectionShardKeyPre71Compatible: "refineCollectionShardKey"
             kRefineCollectionShardKey: "refineCollectionShardKey_V2"

src/mongo/db/s/sharding_ddl_coordinator.idl

@@ -112,7 +112,7 @@ std::shared_ptr<ShardingDDLCoordinator> constructShardingDDLCoordinatorInstance(
             return std::make_shared<RefineCollectionShardKeyCoordinatorPre71Compatible>(
                 service, std::move(initialState));
             break;
-        case DDLCoordinatorTypeEnum::kCreateCollectionPre71Compatible:
+        case DDLCoordinatorTypeEnum::kCreateCollectionPre73Compatible:
             return std::make_shared<CreateCollectionCoordinatorLegacy>(service,
                                                                        std::move(initialState));
             break;

src/mongo/db/s/sharding_ddl_coordinator_service.cpp

@@ -184,8 +184,8 @@ class ShardsvrCreateCollectionCommand final : public TypedCommand<ShardsvrCreate
                     requestToForward.setTimeseries(std::move(timeseriesOptions));
                 }
 
+                // TODO (SERVER-79304): Remove once 8.0 becomes last LTS.
                 FixedFCVRegion fixedFcvRegion{opCtx};
-
                 auto coordinatorDoc = [&] {
                     if (feature_flags::gAuthoritativeShardCollection.isEnabled(
                             (*fixedFcvRegion).acquireFCVSnapshot())) {
@@ -197,7 +197,7 @@ class ShardsvrCreateCollectionCommand final : public TypedCommand<ShardsvrCreate
                         return doc.toBSON();
                     } else {
                         const DDLCoordinatorTypeEnum coordType =
-                            DDLCoordinatorTypeEnum::kCreateCollectionPre71Compatible;
+                            DDLCoordinatorTypeEnum::kCreateCollectionPre73Compatible;
                         auto doc = CreateCollectionCoordinatorDocumentLegacy();
                         doc.setShardingDDLCoordinatorMetadata({{ns(), coordType}});
                         doc.setShardsvrCreateCollectionRequest(requestToForward);

src/mongo/db/s/shardsvr_create_collection_command.cpp

@@ -82,6 +82,7 @@ feature_flags:
     default: true
     version: 7.1
     shouldBeFCVGated: true
+  # TODO (SERVER-79304): Remove once 8.0 becomes last LTS.
   featureFlagAuthoritativeShardCollection:
     description: "Feature flag for enabling the authoritative version of the shardCollection DDL operation."
     cpp_varname: feature_flags::gAuthoritativeShardCollection