SERVER-97775: Write rolled back oplog entry to rollback file for debugging (#31736)

GitOrigin-RevId: 132bad3

Author: ruchitharajaghatta

jstests/replsets/rollback_writes_oplogs_to_file.js

@@ -0,0 +1,56 @@
+/**
+ * Tests that a rollback directory is created for oplog entries during replica set rollback, and
+ * verifies that the rolled back oplogs are written to that directory.
+ *
+ * @tags: [requires_persistence, requires_fcv_81]
+ */
+
+import {RollbackTest} from "jstests/replsets/libs/rollback_test.js";
+
+function runRollbackOplogsTest(shouldCreateRollbackFiles) {
+    jsTestLog("Testing createRollbackDataFiles = " + shouldCreateRollbackFiles);
+    const rollbackTest = new RollbackTest(jsTestName());
+    const rollbackNode = rollbackTest.getPrimary();
+    const secondTermPrimary = rollbackTest.getSecondary();
+    assert.commandWorked(rollbackNode.getDB("admin").adminCommand(
+        {setParameter: 1, createRollbackDataFiles: shouldCreateRollbackFiles}));
+    assert.commandWorked(secondTermPrimary.getDB("admin").adminCommand(
+        {setParameter: 1, createRollbackDataFiles: shouldCreateRollbackFiles}));
+
+    const dbName = "test";
+    const collName = "rollbackColl";
+
+    // Isolate the rollbackNode (current primary node) and insert documents (which will be rolled
+    // back).
+    rollbackTest.transitionToRollbackOperations();
+    assert.commandWorked(rollbackNode.getDB(dbName)[collName].insert({"a": 1}));
+    assert.commandWorked(rollbackNode.getDB(dbName)[collName].insert({"a": 2}));
+    const rst = rollbackTest.getTestFixture();
+    const oplogsToRollback = rst.findOplog(rollbackNode, {}).toArray();
+
+    // Elect the previous secondary as the new primary.
+    rollbackTest.transitionToSyncSourceOperationsBeforeRollback();
+    assert.commandWorked(secondTermPrimary.getDB(dbName)[collName].insert({"b": 1}));
+
+    // Reconnect the isolated node and rollback should start.
+    rollbackTest.transitionToSyncSourceOperationsDuringRollback();
+    // Transition back to steady state.
+    rollbackTest.transitionToSteadyStateOperations();
+
+    // Check for rollback files.
+    const rollbackNodePath = rst.getDbPath(rollbackNode);
+    const oplogRollbackDir = rollbackNodePath + "/rollback/local.oplog.rs";
+    assert.eq(pathExists(oplogRollbackDir), shouldCreateRollbackFiles, oplogRollbackDir);
+    if (shouldCreateRollbackFiles) {
+        const listRollbackFiles = listFiles(oplogRollbackDir);
+        let oplogsRolledBack = [];
+        for (let i = 0; i < listRollbackFiles.length; i++) {
+            oplogsRolledBack = oplogsRolledBack.concat(_readDumpFile(listRollbackFiles[i].name));
+        }
+        assert.contains(oplogsToRollback[0], oplogsRolledBack);
+        assert.contains(oplogsToRollback[1], oplogsRolledBack);
+    }
+    rst.stopSet();
+}
+runRollbackOplogsTest(true);
+runRollbackOplogsTest(false);

src/mongo/db/repl/BUILD.bazel

@@ -780,6 +780,7 @@ mongo_cc_library(
         ":oplog_entry",
         ":optime",  # TODO(SERVER-93876): Remove.
         "//src/mongo:base",
+        "//src/mongo/db/storage:remove_saver",
         "//src/mongo/util:fail_point",  # TODO(SERVER-93876): Remove.
     ],
 )

src/mongo/db/repl/roll_back_local_operations.cpp

@@ -38,6 +38,7 @@
 #include "mongo/bson/timestamp.h"
 #include "mongo/db/repl/oplog_entry.h"
 #include "mongo/db/repl/roll_back_local_operations.h"
+#include "mongo/db/storage/remove_saver.h"
 #include "mongo/logv2/log.h"
 #include "mongo/logv2/log_attr.h"
 #include "mongo/logv2/log_component.h"
@@ -76,6 +77,9 @@ long long getTerm(const OplogInterface::Iterator::Value& oplogValue) {
 }
 }  // namespace
 
+static constexpr auto kRollbackRemoveSaverType = "rollback";
+static constexpr auto kRollbackRemoveSaverWhy = "removed";
+
 RollBackLocalOperations::RollBackLocalOperations(const OplogInterface& localOplog,
                                                  const RollbackOperationFn& rollbackOperation)
 
@@ -99,7 +103,7 @@ RollBackLocalOperations::RollbackCommonPoint::RollbackCommonPoint(BSONObj oplogB
 }
 
 StatusWith<RollBackLocalOperations::RollbackCommonPoint> RollBackLocalOperations::onRemoteOperation(
-    const BSONObj& operation) {
+    const BSONObj& operation, RemoveSaver& removeSaver, bool shouldCreateDataFiles) {
     if (_scanned == 0) {
         auto result = _localOplogIterator->next();
         if (!result.isOK()) {
@@ -118,6 +122,9 @@ StatusWith<RollBackLocalOperations::RollbackCommonPoint> RollBackLocalOperations
                     2,
                     "Local oplog entry to roll back",
                     "oplogEntry"_attr = redact(_localOplogValue.first));
+        if (shouldCreateDataFiles) {
+            fassert(9777500, removeSaver.goingToDelete(_localOplogValue.first));
+        }
         auto status = _rollbackOperation(_localOplogValue.first);
         if (!status.isOK()) {
             invariant(ErrorCodes::NoSuchKey != status.code());
@@ -163,7 +170,8 @@ StatusWith<RollBackLocalOperations::RollbackCommonPoint> RollBackLocalOperations
 StatusWith<RollBackLocalOperations::RollbackCommonPoint> syncRollBackLocalOperations(
     const OplogInterface& localOplog,
     const OplogInterface& remoteOplog,
-    const RollBackLocalOperations::RollbackOperationFn& rollbackOperation) {
+    const RollBackLocalOperations::RollbackOperationFn& rollbackOperation,
+    bool shouldCreateDataFiles) {
 
     std::unique_ptr<OplogInterface::Iterator> remoteIterator;
 
@@ -187,10 +195,12 @@ StatusWith<RollBackLocalOperations::RollbackCommonPoint> syncRollBackLocalOperat
 
     RollBackLocalOperations finder(localOplog, rollbackOperation);
     Timestamp theirTime;
+    RemoveSaver removeSaver(kRollbackRemoveSaverType, "local.oplog.rs", kRollbackRemoveSaverWhy);
     while (remoteResult.isOK()) {
         BSONObj theirObj = remoteResult.getValue().first;
         theirTime = theirObj["ts"].timestamp();
-        auto result = finder.onRemoteOperation(theirObj);
+
+        auto result = finder.onRemoteOperation(theirObj, removeSaver, shouldCreateDataFiles);
         if (result.isOK()) {
             return result.getValue();
         } else if (result.getStatus().code() != ErrorCodes::NoSuchKey) {

src/mongo/db/repl/roll_back_local_operations.h

@@ -37,6 +37,7 @@
 #include "mongo/db/record_id.h"
 #include "mongo/db/repl/oplog_interface.h"
 #include "mongo/db/repl/optime.h"
+#include "mongo/db/storage/remove_saver.h"
 #include "mongo/util/fail_point.h"
 #include "mongo/util/time_support.h"
 
@@ -101,7 +102,9 @@ class RollBackLocalOperations {
      * Returns ErrorCodes::NoSuchKey if common point has not been found and
      * additional operations have to be read from the remote oplog.
      */
-    StatusWith<RollbackCommonPoint> onRemoteOperation(const BSONObj& operation);
+    StatusWith<RollbackCommonPoint> onRemoteOperation(const BSONObj& operation,
+                                                      RemoveSaver& removeSaver,
+                                                      bool shouldCreateDataFiles);
 
 private:
     std::unique_ptr<OplogInterface::Iterator> _localOplogIterator;
@@ -122,7 +125,8 @@ class RollBackLocalOperations {
 StatusWith<RollBackLocalOperations::RollbackCommonPoint> syncRollBackLocalOperations(
     const OplogInterface& localOplog,
     const OplogInterface& remoteOplog,
-    const RollBackLocalOperations::RollbackOperationFn& rollbackOperation);
+    const RollBackLocalOperations::RollbackOperationFn& rollbackOperation,
+    bool shouldCreateDataFiles);
 
 }  // namespace repl
 }  // namespace mongo