Bug#37140331 MySQL NDB Cluster is crushing with Signal 8 error (Floating Point Exception) 1/2

Problem:
When a query on ndbinfo table FRAG_MEM_USE or FRAG_OPERATIONS
is performed, a DBINFO_SCANREQ signal is sent to LQH. during
the handling of that signal, LQH asks TUP about fragStats
of all tables defined in the cluster.
If in 'parallel' with the ndbinfo scan a create (or drop)
table operation is on going, there could be a discrepancy
between the view LQH has of new table fragments and TUP view
about those fragments, that can lead to a crash in both TUP/ACC
or LQH.

In particular, if DBINFO_SCANREQ finds the new table with status
ADD_TABLE_ONGOING it could be impossible to TUP/ACC to get the
status of the new table fragments since at that point, fragments
information in TUP/ACC is not yet updated.
In similar way, during drop table if the status of the target table
in LQH is DROP_TABLE_* or PREP_DROP_* there could be differences
between the view that LQH and TUP/ACC have of the fragments of
that table.

This patch add 2 new tests to testNdbinfo suite where tables
FRAG_MEM_USE and FRAG_OPERATIONS are scanned (intensively) in
parallel with create/drop table operations running in different
threads.

Without a fix to the discrepancy in fragments in LQH and ACC/TUP
these tests should crash data nodes in different ways on both ACC
or TUP (it depends on the 'phase' of create/drop table at the
moment that SCANREQ of frags table arrives to LQH)

Note: due to some asserts in the code, if ndb(mt)d is compiled in
debug mode data node can crash earlier in those assert instead
of in the problematic places in LQH and ACC/TUP.

Change-Id: I60d1954b9ef6a3b1324fe3b668860d1b1d680a98

Author: vinc13e

storage/ndb/src/kernel/blocks/ERROR_codes.txt

@@ -24,8 +24,8 @@
 Next QMGR 950
 Next NDBCNTR 1030
 Next NDBFS 2002
-Next DBACC 3006
-Next DBTUP 4039
+Next DBACC 3007
+Next DBTUP 4040
 Next DBLQH 5113
 Next DBDICT 6227
 Next DBDIH 7251

storage/ndb/src/kernel/blocks/dbacc/DbaccMain.cpp

@@ -543,6 +543,16 @@ void Dbacc::set_tup_fragptr(Uint32 fragptr, Uint32 tup_fragptr) {
 void Dbacc::execACCFRAGREQ(Signal *signal) {
   const AccFragReq *const req = (AccFragReq *)&signal->theData[0];
   jamEntry();
+  if (ERROR_INSERTED(3006)) {
+    jam();
+    // Delay each GSN_ACCFRAGREQ only once
+    if (signal->senderBlockRef() != reference()) {
+      jam();
+      sendSignalWithDelay(reference(), GSN_ACCFRAGREQ, signal, 100,
+                          signal->getLength());
+      return;
+    }
+  }
   if (ERROR_INSERTED(3001)) {
     jam();
     addFragRefuse(signal, 1);

storage/ndb/src/kernel/blocks/dbtup/DbtupMeta.cpp

@@ -2363,6 +2363,12 @@ void Dbtup::drop_fragment_fsremove_done(Signal *signal, TablerecPtr tabPtr,
     signal->theData[0] = ZREL_FRAG;
     signal->theData[1] = tabPtr.i;
     signal->theData[2] = logfile_group_id;
+    if (ERROR_INSERTED(4039)) {
+      jam();
+      // Delay fragment release
+      sendSignalWithDelay(cownref, GSN_CONTINUEB, signal, 1000, 3);
+      return;
+    }
     sendSignal(cownref, GSN_CONTINUEB, signal, 3, JBB);
   } else {
     jam();

storage/ndb/test/ndbapi/testNdbinfo.cpp

@@ -470,6 +470,132 @@ int runRestarter(NDBT_Context *ctx, NDBT_Step *step) {
   return result;
 }
 
+int runCreateDropTableUntilStopped(NDBT_Context *ctx, NDBT_Step *step) {
+  Ndb *pNdb = GETNDB(step);
+  NdbRestarter res;
+  const char *tableName = ctx->getProperty("tableName", (char *)NULL);
+
+  NdbDictionary::Dictionary *pDict = pNdb->getDictionary();
+  Uint32 stepNum = step->getStepNo();
+  BaseString tabName(tableName);
+  tabName.appfmt("_%i", stepNum);
+
+  NdbDictionary::Table tab(tabName.c_str());
+  {
+    NdbDictionary::Column col("a");
+    col.setType(NdbDictionary::Column::Unsigned);
+    col.setPrimaryKey(true);
+    tab.addColumn(col);
+  }
+  {
+    NdbDictionary::Column col("b");
+    col.setType(NdbDictionary::Column::Unsigned);
+    col.setNullable(false);
+    tab.addColumn(col);
+  }
+
+  while (!ctx->isTestStopped()) {
+    /*
+     * 3006: Error Insert to delay handling of ACCFRAGREQ (and indirectly
+     * TUPFRAGREQ).
+     * 4039: Error Insert to delay fragment release in TUP.
+     */
+    int error = rand() % 2 ? 3006 : 4039;
+    if (res.insertErrorInAllNodes(error) != 0) {
+      g_err << "Failed to insertError " << error << endl;
+      return NDBT_FAILED;
+    }
+
+    if (pDict->createTable(tab) != 0) {
+      NdbError err = pDict->getNdbError();
+      g_err << "Failed to create table (" << tabName.c_str() << ") " << err
+            << endl;
+      /**
+       * if error is:
+       * 701: System busy with other schema operation, or
+       * 721: Schema object with given name already exists
+       * test can continue
+       */
+      if (err.code != 701 && err.code != 721) {
+        return NDBT_FAILED;
+      }
+    }
+
+    /**
+     * if error is:
+     * 701: System busy with other schema operation, or
+     * 723: No such table existed
+     * test can continue
+     */
+    if (pDict->dropTable(tabName.c_str()) != 0) {
+      NdbError err = pDict->getNdbError();
+      g_err << "Failed to drop table (" << tabName.c_str() << ") " << err
+            << endl;
+      if (err.code != 701 && err.code != 723) {
+        return NDBT_FAILED;
+      }
+    }
+    NdbSleep_MilliSleep(20);
+    if (res.insertErrorInAllNodes(0) != 0) {
+      g_err << "Failed to clear Error " << error << endl;
+      return NDBT_FAILED;
+    }
+  }
+  return NDBT_OK;
+}
+
+int runScanNdbInfoTable(NDBT_Context *ctx, NDBT_Step *step) {
+  int result = NDBT_OK;
+  int loops = ctx->getNumLoops();
+  const char *tableName = ctx->getProperty("infoTableName", (char *)NULL);
+
+  NdbInfo ndbinfo(&ctx->m_cluster_connection, "ndbinfo/");
+  if (!ndbinfo.init()) {
+    g_err << "ndbinfo.init failed" << endl;
+    ctx->stopTest();
+    return NDBT_FAILED;
+  }
+
+  const NdbInfo::Table *table;
+  if (ndbinfo.openTable(tableName, &table) != 0) {
+    g_err << "Failed to openTable " << tableName << endl;
+    ctx->stopTest();
+    return NDBT_FAILED;
+  }
+
+  while (loops-- && !ctx->isTestStopped()) {
+    NdbInfoScanOperation *scanOp = nullptr;
+    if (ndbinfo.createScanOperation(table, &scanOp)) {
+      g_err << "createScanOperation failed" << endl;
+      ndbinfo.releaseScanOperation(scanOp);
+      result = NDBT_FAILED;
+      break;
+    }
+
+    if (scanOp->readTuples() != 0) {
+      g_err << "scanOp->readTuples failed" << endl;
+      ndbinfo.releaseScanOperation(scanOp);
+      result = NDBT_FAILED;
+      break;
+    }
+
+    if (scanOp->execute() != 0) {
+      g_err << "scanOp->execute failed" << endl;
+      ndbinfo.releaseScanOperation(scanOp);
+      result = NDBT_FAILED;
+      break;
+    }
+    while (scanOp->nextResult() == 1) {
+    }
+
+    ndbinfo.releaseScanOperation(scanOp);
+  }
+
+  ndbinfo.closeTable(table);
+  ctx->stopTest();
+  return result;
+}
+
 NDBT_TESTSUITE(testNdbinfo);
 TESTCASE("NodeRestart", "Scan NdbInfo tables while restarting nodes") {
   STEP(runRestarter);
@@ -504,6 +630,23 @@ TESTCASE("TestTable",
          "of rows which will depend on how many TUP blocks are configured") {
   STEP(runTestTable);
 }
+TESTCASE("ScanFragOperationsDuringCreateDropTable",
+         "Check that scanning of ndbinfo/frag_operations table is robust"
+         "to CREATE/DROP table operation running in parallel with the scan") {
+  TC_PROPERTY("tableName", "tmp_table");
+  TC_PROPERTY("infoTableName", "ndbinfo/frag_operations");
+  STEPS(runCreateDropTableUntilStopped, 1);
+  STEPS(runScanNdbInfoTable, 16);
+}
+TESTCASE("ScanFragMemUseDuringCreateDropTable",
+         "Check that scanning of ndbinfo/frag_mem_use table is robust"
+         "to CREATE/DROP table operation running in parallel with the scan") {
+  TC_PROPERTY("tableName", "tmp_table");
+  TC_PROPERTY("infoTableName", "ndbinfo/frag_mem_use");
+  STEPS(runCreateDropTableUntilStopped, 1);
+  STEPS(runScanNdbInfoTable, 16);
+}
+
 NDBT_TESTSUITE_END(testNdbinfo)
 
 int main(int argc, const char **argv) {

storage/ndb/test/run-test/daily-devel--07-tests.txt

@@ -364,3 +364,11 @@ max-time: 480
 cmd: testSystemRestart
 args: -n GCPSaveLagLcpSR T1
 max-time: 240
+
+cmd: testNdbinfo
+args: -n ScanFragOperationsDuringCreateDropTable -l 10000 T1
+max-time: 180
+
+cmd: testNdbinfo
+args: -n ScanFragMemUseDuringCreateDropTable -l 10000 T1
+max-time: 180