SERVER-98399 Time-series collections with mixed-schema data should fail validation if only the top-level mixed-schema flag is set (#34071)

GitOrigin-RevId: 1ac214a2f627476862e7803f4b6d0b89a5758665

Author: joanbm-mongodb

jstests/noPassthrough/query/timeseries/timeseries_validate_mixed_schema_bucket_legacy_flag.js

@@ -0,0 +1,92 @@
+/**
+ * Tests validating a time-series collection with mixed schema buckets when the mixed-schema flag
+ * in the top-level catalog metadata, but not in the collection options (WiredTiger config string).
+ *
+ * This replicates the scenario where a time series collection has been created in MongoDB <5.2,
+ * (i.e. before SERVER-60565) then later upgraded to MongoDB <6.0.17 (i.e. before SERVER-91195),
+ * then all the way up to the current version (see SERVER-98399 for more details).
+ *
+ * As having the mixed-schema flag set only in the top-level catalog metadata is a precarious state,
+ * as its value may be lost, direct manipulation of mixed-schema buckets is prevented, and
+ * validation will flag the collection as needing manual intervention for SERVER-91194.
+ */
+(function() {
+"use strict";
+
+load("jstests/libs/fail_point_util.js");  // For configureFailPoint
+
+const conn = MongoRunner.runMongod();
+const testDB = conn.getDB(jsTestName());
+
+const collName = "ts";
+
+// Create a time-series collection containing a mixed-schema bucket
+assert.commandWorked(testDB.runCommand({drop: collName}));
+assert.commandWorked(
+    testDB.createCollection(collName, {timeseries: {timeField: 't', metaField: 'm'}}));
+const coll = testDB[collName];
+const bucketsColl = testDB["system.buckets." + collName];
+
+const bucket = {
+    _id: ObjectId("65a6eb806ffc9fa4280ecac4"),
+    control: {
+        version: NumberInt(1),
+        min: {
+            _id: ObjectId("65a6eba7e6d2e848e08c3750"),
+            t: ISODate("2024-01-16T20:48:00Z"),
+            a: 1,
+        },
+        max: {
+            _id: ObjectId("65a6eba7e6d2e848e08c3751"),
+            t: ISODate("2024-01-16T20:48:39.448Z"),
+            a: "a",
+        },
+    },
+    meta: 0,
+    data: {
+        _id: {
+            0: ObjectId("65a6eba7e6d2e848e08c3750"),
+            1: ObjectId("65a6eba7e6d2e848e08c3751"),
+        },
+        t: {
+            0: ISODate("2024-01-16T20:48:39.448Z"),
+            1: ISODate("2024-01-16T20:48:39.448Z"),
+        },
+        a: {
+            0: "a",
+            1: 1,
+        },
+    },
+};
+
+assert.commandWorked(
+    testDB.runCommand({collMod: collName, timeseriesBucketsMayHaveMixedSchemaData: true}));
+assert.commandWorked(bucketsColl.insert(bucket));
+
+// Set the mixed-schema flag only set on the top-level catalog metadata field
+// (md.timeseriesBucketsMayHaveMixedSchemaData), but not on the collection options
+// (inside md.options.storageEngine.wiredTiger.configString).
+const fpsimulateLegacyTimeseriesMixedSchemaFlag =
+    configureFailPoint(conn, "simulateLegacyTimeseriesMixedSchemaFlag");
+assert.commandWorked(
+    testDB.runCommand({collMod: collName, timeseriesBucketsMayHaveMixedSchemaData: true}));
+fpsimulateLegacyTimeseriesMixedSchemaFlag.off();
+
+const bucketsCatalogEntry = bucketsColl.aggregate([{$listCatalog: {}}]).toArray()[0];
+const wtConfigStr = bucketsCatalogEntry.md.options.storageEngine?.wiredTiger?.configString ?? '';
+assert.eq(true, bucketsCatalogEntry.md.timeseriesBucketsMayHaveMixedSchemaData);
+assert(!wtConfigStr.includes("timeseriesBucketsMayHaveMixedSchemaData"));
+
+// Validation of the collection returns the error asking for SERVER-91194 manual intervention
+const res = assert.commandWorked(coll.validate());
+assert(!res.valid);
+assert.eq(res.warnings.length, 0);
+assert.gt(res.errors.length, 0, "Validation should return at least one error.");
+assert.containsPrefix(
+    "Detected a time-series bucket with mixed schema data",
+    res.errors,
+    "Validation of mixed schema buckets when they are not allowed should return an error stating such");
+
+assert.commandWorked(bucketsColl.deleteOne({_id: bucket._id}));  // Clean up for shutdown validation
+MongoRunner.stopMongod(conn);
+})();

src/mongo/db/catalog/SConscript

@@ -484,6 +484,7 @@ env.Library(
         'collection_options',
         'index_catalog',
         'index_key_validate',
+        'storage_engine_collection_options_flags_parser',
         'throttle_cursor',
         'validate_state',
     ],

src/mongo/db/catalog/collection_impl.cpp

@@ -80,6 +80,10 @@ namespace {
 MONGO_FAIL_POINT_DEFINE(allowSettingMalformedCollectionValidators);
 
 MONGO_FAIL_POINT_DEFINE(skipCappedDeletes);
+// Simulate the behavior of mixed-schema flag of MongoDB versions without SERVER-91195:
+// Only set the legacy time-series mixed-schema flag at the top level of the catalog,
+// and clear the new durable flag which is stored inside the collection options.
+MONGO_FAIL_POINT_DEFINE(simulateLegacyTimeseriesMixedSchemaFlag);
 
 Status checkValidatorCanBeUsedOnNs(const BSONObj& validator,
                                    const NamespaceString& nss,
@@ -866,7 +870,7 @@ void CollectionImpl::setTimeseriesBucketsMayHaveMixedSchemaData(OperationContext
             md.options.storageEngine = setFlagToStorageEngineBson(
                 md.options.storageEngine,
                 backwards_compatible_collection_options::kTimeseriesBucketsMayHaveMixedSchemaData,
-                *setting);
+                simulateLegacyTimeseriesMixedSchemaFlag.shouldFail() ? boost::none : setting);
         }
 
         // Also update legacy parameter for compatibility when downgrading to older sub-versions

src/mongo/db/catalog/validate_adaptor.cpp

@@ -37,11 +37,13 @@
 #include "mongo/bson/bsonobj.h"
 #include "mongo/bson/util/bson_extract.h"
 #include "mongo/bson/util/bsoncolumn.h"
+#include "mongo/db/catalog/backwards_compatible_collection_options_util.h"
 #include "mongo/db/catalog/clustered_collection_util.h"
 #include "mongo/db/catalog/collection.h"
 #include "mongo/db/catalog/column_index_consistency.h"
 #include "mongo/db/catalog/index_catalog.h"
 #include "mongo/db/catalog/index_consistency.h"
+#include "mongo/db/catalog/storage_engine_collection_options_flags_parser.h"
 #include "mongo/db/catalog/throttle_cursor.h"
 #include "mongo/db/concurrency/exception_util.h"
 #include "mongo/db/curop.h"
@@ -821,8 +823,16 @@ void ValidateAdaptor::traverseRecordStore(OperationContext* opCtx,
                         results->valid = false;
                     } else if (containsMixedSchemaDataResponse.isOK() &&
                                containsMixedSchemaDataResponse.getValue()) {
-                        bool mixedSchemaAllowed =
-                            coll->getTimeseriesBucketsMayHaveMixedSchemaData().get();
+                        // Only allow mixed-schema data if the new durable mixed-schema flag
+                        // introduced by SERVER-91195 (in `options.storageEngine`) is set.
+                        // Checking against `coll->getTimeseriesBucketsMayHaveMixedSchemaData()` is
+                        // unsafe, since only the legacy mixed-schema flag may be set, which can be
+                        // later lost on collection cloning due to SERVER-91194.
+                        bool mixedSchemaAllowed = getFlagFromStorageEngineBson(
+                                                      coll->getCollectionOptions().storageEngine,
+                                                      backwards_compatible_collection_options::
+                                                          kTimeseriesBucketsMayHaveMixedSchemaData)
+                                                      .value_or(false);
                         if (mixedSchemaAllowed && !bucketMixedSchemaDataWarning) {
                             bucketMixedSchemaDataWarning = true;
                             LOGV2_WARNING(8469901,