Bug#37162636 Ndb : GCP_SAVE_REF handling + recoverability [noclose]

Backport to 7.6

As described in the bug, there are some situations around graceful
shutdown + node failures which could lead to a cluster becoming
unrecoverable without manual intervention.

The causes of these situations can be addressed individually.

In this patch the generic GCI info propagation mechanism (CopyGCI)
is modified to reject propagating any set of GCI info which does
not describe the ability to automatically recover a cluster via
SR.

Specifically, it is essential that for a given restorable GCI,
there must be at least one node in every nodegroup also restorable
to that GCI.

Checking this in the Master before propagating it avoids it
becoming disk durable anywhere, so that in any cases where it
may be about-to-be violated, the problem does not propagate.

This could lead to a (faster) cluster shutdown, but the
cluster would then be automatically recoverable via SR.

Change-Id: I840267ef15873c47856ad31378052675efe008e8

Author: frazerclement

storage/ndb/src/kernel/blocks/dbdih/Dbdih.hpp

@@ -1523,6 +1523,7 @@ class Dbdih: public SimulatedBlock {
   void copyTabReq_complete(Signal* signal, TabRecordPtr tabPtr);
 
   void gcpcommitreqLab(Signal *);
+  bool checkAllNgsRepresented(Signal *, const NdbNodeBitmask *nodes);
   void validateCopyGci(Signal *);
   void upgradeAlignCopyGci();
   void copyGciLab(Signal *, CopyGCIReq::CopyReason reason);

storage/ndb/src/kernel/blocks/dbdih/DbdihMain.cpp

@@ -17797,25 +17797,57 @@ void Dbdih::execDIHNDBTAMPER(Signal* signal)
 /*****************************************************************************/
 /* **********     FILE HANDLING MODULE                           *************/
 /*****************************************************************************/
+bool Dbdih::checkAllNgsRepresented(Signal *signal,
+                                   const NdbNodeBitmask *nodes) {
+  jam();
+
+  /**
+   * CheckNodeGroups will examine our bitmap of nodes
+   * If any nodegroup is entirely missing then the result
+   * will be Lose.
+   * If all nodegroups are present then the result will
+   * be Win or Partitioning (we don't care which)
+   */
+  CheckNodeGroups *cng = (CheckNodeGroups *)&signal->theData[0];
+  cng->requestType = CheckNodeGroups::Direct | CheckNodeGroups::ArbitCheck;
+  cng->mask.assign(*nodes);
+  execCHECKNODEGROUPSREQ(signal);
+
+  return (cng->output != CheckNodeGroups::Lose);
+}
+
 void Dbdih::validateCopyGci(Signal *signal) {
   jam();
   /**
    * Before we (Master) copy our GCI info to all other
    * nodes, let's check it for sanity
    */
   bool newestRestorableGCIIsMax = true;
+  bool recoverableNodesAreSufficientForSR = true;
+
   const Uint32 newestRestorableGCI = SYSFILE->newestRestorableGCI;
 
+  /* Build bitmap of recoverable nodes */
+  NdbNodeBitmask recoverableNodes;
   for (Uint32 i = 0; i < MAX_NDB_NODES; i++) {
     const Uint32 nodeLastCompletedGci = SYSFILE->lastCompletedGCI[i];
 
+    if (nodeLastCompletedGci == newestRestorableGCI) {
+      recoverableNodes.set(i);
+    }
+
     if (nodeLastCompletedGci > newestRestorableGCI) {
       jam();
       newestRestorableGCIIsMax = false;
     }
   }
 
-  if (unlikely(!newestRestorableGCIIsMax)) {
+  /* Check overall system is recoverable */
+  recoverableNodesAreSufficientForSR =
+      checkAllNgsRepresented(signal, &recoverableNodes);
+
+  if (unlikely(
+          !(newestRestorableGCIIsMax && recoverableNodesAreSufficientForSR))) {
     jam();
     g_eventLogger->error("DIH : newestRestorableGCI %u", newestRestorableGCI);
     for (Uint32 i = 0; i < MAX_NDB_NODES; i++) {
@@ -17835,8 +17867,20 @@ void Dbdih::validateCopyGci(Signal *signal) {
       g_eventLogger->error(
           "DIH : Invalid CopyGCIREQ attempted, newestRestorableGCI is not max");
     }
-
+    if (!recoverableNodesAreSufficientForSR) {
+      jam();
+      /**
+       * Require that every nodegroup has at least one representative
+       * which is restorable to the newestRestorableGci number.
+       * Otherwise System Restart with all nodes present will not have
+       * sufficient 'log' to be recoverable
+       */
+      g_eventLogger->error(
+          "DIH : Invalid CopyGCIREQ attempted, recoverable nodes are not "
+          "sufficient for SR");
+    }
     ndbrequire(newestRestorableGCIIsMax);
+    ndbrequire(recoverableNodesAreSufficientForSR);
   }
 }