SERVER-32721 Explain output should indicate when a backup plan is used

jstests/core/explain_use_backup_plan.js

@@ -0,0 +1,30 @@
+// Test that the explain will use backup plan if the original winning plan ran out of memory in the
+// "executionStats" mode
+// This test was designed to reproduce SERVER-32721"
+
+load("jstests/libs/analyze_plan.js");
+
+"use strict";
+
+db.foo.drop();
+let bulk = db.foo.initializeUnorderedBulkOp();
+
+for (let i = 0; i < 100000; ++i) {
+    bulk.insert({_id: i, x: i, y: i});
+}
+
+bulk.execute();
+db.foo.ensureIndex({x: 1});
+
+// Configure log level and lower the sort bytes limit.
+db.setLogLevel(5, "query");
+db.adminCommand({setParameter: 1, internalQueryExecMaxBlockingSortBytes: 100});
+
+const test1 = db.foo.find({x: {$gte: 90}}).sort({_id: 1}).explain(true);
+const test2 = db.foo.find({x: {$gte: 90000}}).sort({_id: 1}).explain(true);
+// This query will not use the backup plan, hence it generates only two stages: winningPlan and
+// rejectedPlans
+assert.eq(backupPlanUsed(test1), false, "test1 did not use a backup plan");
+// This query will use backup plan, the exaplin output for this query will generate three
+// stages: winningPlan, rejectedPlans and originalWinningPlan
+assert(backupPlanUsed(test2), "backup plan invoked in test2");

jstests/libs/analyze_plan.js

@@ -287,3 +287,7 @@ function assertCoveredQueryAndCount({collection, query, project, count}) {
            "Winning plan for count was not covered: " + tojson(explain.queryPlanner.winningPlan));
     assertExplainCount({explainResults: explain, expectedCount: count});
 }
+
+function backupPlanUsed(root) {
+    return root.queryPlanner.backupPlanUsed;
+}

src/mongo/db/exec/multi_plan.cpp

@@ -71,6 +71,7 @@ MultiPlanStage::MultiPlanStage(OperationContext* opCtx,
       _query(cq),
       _bestPlanIdx(kNoSuchPlan),
       _backupPlanIdx(kNoSuchPlan),
+      _originalWinningPlanIdx(kNoSuchPlan),
       _failure(false),
       _failureCount(0),
       _statusMemberId(WorkingSet::INVALID_ID) {
@@ -128,6 +129,7 @@ PlanStage::StageState MultiPlanStage::doWork(WorkingSetID* out) {
         // cached plan runner to fall back on a different solution
         // if the best solution fails. Alternatively we could try to
         // defer cache insertion to be after the first produced result.
+        _originalWinningPlanIdx = _bestPlanIdx;
 
         _collection->infoCache()->getPlanCache()->remove(*_query).transitional_ignore();
 
@@ -452,6 +454,12 @@ void MultiPlanStage::doInvalidate(OperationContext* opCtx,
 bool MultiPlanStage::hasBackupPlan() const {
     return kNoSuchPlan != _backupPlanIdx;
 }
+int MultiPlanStage::backupPlanIdx() const {
+    return _backupPlanIdx;
+}
+int MultiPlanStage::originalWinningPlanIdx() const {
+    return _originalWinningPlanIdx;
+}
 
 bool MultiPlanStage::bestPlanChosen() const {
     return kNoSuchPlan != _bestPlanIdx;

src/mongo/db/exec/multi_plan.h

@@ -129,9 +129,21 @@ class MultiPlanStage final : public PlanStage {
     /** Return true if a best plan has been chosen  */
     bool bestPlanChosen() const;
 
-    /** Return the index of the best plan chosen, for testing */
+    /*
+    * Return the index of the best plan chosen
+    */
     int bestPlanIdx() const;
 
+    /**
+     * Return the index of the backup plan chosen 
+     */
+    int backupPlanIdx() const;
+
+    /**
+     * Return the index of the original winning plan chosen
+     */
+    int originalWinningPlanIdx() const;
+
     /**
      * Returns the QuerySolution for the best plan, or NULL if no best plan
      *
@@ -198,10 +210,14 @@ class MultiPlanStage final : public PlanStage {
     // uses -1 / kNoSuchPlan when best plan is not (yet) known
     int _bestPlanIdx;
 
-    // index into _candidates, of the backup plan for sort
+    // index into _candidates, of the backup of the plan competition
     // uses -1 / kNoSuchPlan when best plan is not (yet) known
     int _backupPlanIdx;
 
+    // Index into _candidates, of the original winner of the plan which can be used if a blocking plan .fails
+    // This is set to 'kNoSuchPlan' if there is no backup plan, or when it is not (yet) known
+    int _originalWinningPlanIdx;
+
     // Set if this MultiPlanStage cannot continue, and the query must fail. This can happen in
     // two ways. The first is that all candidate plans fail. Note that one plan can fail
     // during normal execution of the plan competition.  Here is an example:

src/mongo/db/query/explain.cpp

@@ -62,6 +62,7 @@ using std::string;
 using std::unique_ptr;
 using std::vector;
 
+bool backupPlanUsed = false;
 /**
  * Traverse the tree rooted at 'root', and add all tree nodes into the list 'flattened'.
  */
@@ -280,6 +281,30 @@ std::vector<std::unique_ptr<PlanStageStats>> getRejectedPlansTrialStats(PlanExec
     return res;
 }
 
+/**
+ * Gather the PlanStages when a backup plan is used
+ */
+std::vector<std::unique_ptr<PlanStageStats>> getRejectedPlansTrialStatsForAllPlanExecution(PlanExecutor* exec) {
+    const auto mps = getMultiPlanStage(exec->getRootStage());
+    std::vector<std::unique_ptr<PlanStageStats>> res;
+
+    if (backupPlanUsed) {
+        // Get the stats from the trial period for all the plans.
+        if (mps) {
+            const auto mpsStats = mps->getStats();
+            for (size_t i = 0; i < mpsStats->children.size(); ++i) {
+                if (i != static_cast<size_t>(mps->originalWinningPlanIdx())) {
+                    res.emplace_back(std::move(mpsStats->children[i]));
+                }
+            }
+        }
+    } else {
+        res = getRejectedPlansTrialStats(exec);
+    }
+
+    return res;
+}
+
 /**
  * Get PlanExecutor's winning plan stats tree.
  */
@@ -289,6 +314,18 @@ unique_ptr<PlanStageStats> getWinningPlanStatsTree(const PlanExecutor* exec) {
                : std::move(exec->getRootStage()->getStats());
 }
 
+
+/**
+ * Returns the root of the roginal winning plan used by 'exec'.
+ * This might be different than the final winning plan in the case that the MultiPlanStage selected a 
+ * blocking plan which failed, and fell back to a non-blocking plan instead.
+ * If there is no MultiPlanStage in the tree, returns the root stage of 'exec'
+ */
+unique_ptr<PlanStageStats> getOriginalWinningPlanStatsTree(const PlanExecutor* exec) {
+    MultiPlanStage* mps = getMultiPlanStage(exec->getRootStage());
+    return mps ? std::move(mps->getStats()->children[mps->originalWinningPlanIdx()])
+               : std::move(exec->getRootStage()->getStats());
+}
 }  // namespace
 
 namespace mongo {
@@ -644,6 +681,17 @@ void Explain::generatePlannerInfo(PlanExecutor* exec,
     BSONObjBuilder plannerBob(out->subobjStart("queryPlanner"));
 
     plannerBob.append("plannerVersion", QueryPlanner::kPlannerVersion);
+
+    const auto mps = getMultiPlanStage(exec->getRootStage());
+    // const bool backupPlanUsed = ((mps->originalWinningPlanIdx()) > -1); 
+    backupPlanUsed = ((mps->originalWinningPlanIdx()) > -1); 
+
+    if (backupPlanUsed) {
+        plannerBob.append("backupPlanUsed", true);
+    } else {
+        plannerBob.append("backupPlanUsed", false);
+    }
+
     plannerBob.append("namespace", exec->nss().ns());
 
     // Find whether there is an index filter set for the query shape. The 'indexFilterSet'
@@ -688,6 +736,16 @@ void Explain::generatePlannerInfo(PlanExecutor* exec,
     }
     allPlansBob.doneFast();
 
+    if (backupPlanUsed) {
+        // Generate array of original winning plan
+        BSONObjBuilder originalWinningPlanBob(plannerBob.subobjStart("originalWinningPlan"));
+        const auto originalWinnerStats = getOriginalWinningPlanStatsTree(exec);
+        statsToBSON(*originalWinnerStats.get(),
+                    &originalWinningPlanBob,
+                    ExplainOptions::Verbosity::kQueryPlanner);
+        originalWinningPlanBob.doneFast();
+    }
+
     plannerBob.doneFast();
 }
 
@@ -798,7 +856,7 @@ void Explain::generateExecutionInfo(PlanExecutor* exec,
             planBob.doneFast();
         }
 
-        const vector<unique_ptr<PlanStageStats>> rejectedStats = getRejectedPlansTrialStats(exec);
+        const vector<unique_ptr<PlanStageStats>> rejectedStats = getRejectedPlansTrialStatsForAllPlanExecution(exec);
         for (size_t i = 0; i < rejectedStats.size(); ++i) {
             BSONObjBuilder planBob(allPlansBob.subobjStart());
             generateSinglePlanExecutionInfo(
@@ -855,8 +913,9 @@ void Explain::explainStages(PlanExecutor* exec,
                             const Collection* collection,
                             ExplainOptions::Verbosity verbosity,
                             BSONObjBuilder* out) {
+
     auto winningPlanTrialStats = Explain::getWinningPlanTrialStats(exec);
-
+   
     Status executePlanStatus = Status::OK();
 
     // If we need execution stats, then run the plan in order to gather the stats.