Apply minor cleanup to cputopology and Isomalloc-Sync

Author: evan-charmworks

src/conv-core/cputopology.C

@@ -69,10 +69,6 @@ typedef struct _nodeTopoMsg {
   int *nodes;
 } nodeTopoMsg;
 
-typedef struct _topoDoneMsg { // used for empty reduction to indicate all PEs have topo info
-  char core[CmiMsgHeaderSizeBytes];
-} topoDoneMsg;
-
 // nodeIDs[pe] is the node number of processor pe
 class CpuTopology {
 public:
@@ -203,21 +199,21 @@ static std::atomic<bool> cpuTopoSyncCommThreadDone{};
 #endif
 
 #if CMK_SMP && !CMK_SMP_NO_COMMTHD
-static void cpuTopoSyncWaitCommThread()
+static void cpuTopoSyncWaitCommThread(std::atomic<bool> & done)
 {
   do
     CommunicationServerThread(5);
-  while (!cpuTopoSyncCommThreadDone.load());
+  while (!done.load());
 
   CommunicationServerThread(5);
 }
 #endif
 
-static void cpuTopoSyncWait()
+static void cpuTopoSyncWait(std::atomic<bool> & done)
 {
   do
     CsdSchedulePoll();
-  while (!cpuTopoSyncHandlerDone.load());
+  while (!done.load());
 
   CsdSchedulePoll();
 }
@@ -493,7 +489,7 @@ extern "C" void LrtsInitCpuTopo(char **argv)
 #if CMK_SMP && !CMK_SMP_NO_COMMTHD
   if (CmiInCommThread())
   {
-    cpuTopoSyncWaitCommThread();
+    cpuTopoSyncWaitCommThread(cpuTopoSyncCommThreadDone);
   }
   else
 #endif
@@ -511,7 +507,7 @@ extern "C" void LrtsInitCpuTopo(char **argv)
     proc->nodeID = 0;
     CmiReduce(msg, sizeof(hostnameMsg)+sizeof(_procInfo), combineMessage);
 
-    cpuTopoSyncWait();
+    cpuTopoSyncWait(cpuTopoSyncHandlerDone);
 
     if (CmiMyRank() == 0)
     {

src/conv-core/isomalloc.C

@@ -693,28 +693,28 @@ struct CmiAddressSpaceRegionMsg
   CmiAddressSpaceRegion region;
 };
 
-static std::atomic<char> CmiIsomallocSyncHandlerDone;
+static std::atomic<bool> CmiIsomallocSyncHandlerDone{};
 #if CMK_SMP && !CMK_SMP_NO_COMMTHD
 extern void CommunicationServerThread(int sleepTime);
-static std::atomic<char> CmiIsomallocSyncCommThreadDone;
+static std::atomic<bool> CmiIsomallocSyncCommThreadDone{};
 #endif
 
 #if CMK_SMP && !CMK_SMP_NO_COMMTHD
-static void CmiIsomallocSyncWaitCommThread()
+static void CmiIsomallocSyncWaitCommThread(std::atomic<bool> & done)
 {
   do
     CommunicationServerThread(5);
-  while (!CmiIsomallocSyncCommThreadDone.load());
+  while (!done.load());
 
   CommunicationServerThread(5);
 }
 #endif
 
-static void CmiIsomallocSyncWait()
+static void CmiIsomallocSyncWait(std::atomic<bool> & done)
 {
   do
     CsdSchedulePoll();
-  while (!CmiIsomallocSyncHandlerDone.load());
+  while (!done.load());
 
   CsdSchedulePoll();
 }
@@ -726,7 +726,7 @@ static void CmiIsomallocSyncReductionHandler(void * data)
   auto region = (CmiAddressSpaceRegion *)data;
   CmiAssert(region == &IsoRegion);
 
-  CmiIsomallocSyncHandlerDone = 1;
+  CmiIsomallocSyncHandlerDone = true;
 }
 static void CmiIsomallocSyncBroadcastHandler(void * msg)
 {
@@ -738,7 +738,7 @@ static void CmiIsomallocSyncBroadcastHandler(void * msg)
 
   CmiFree(msg);
 
-  CmiIsomallocSyncHandlerDone = 1;
+  CmiIsomallocSyncHandlerDone = true;
 }
 
 static void CmiIsomallocInitExtent(char ** argv)
@@ -850,6 +850,13 @@ static void CmiIsomallocInitExtent(char ** argv)
   }
   else if (CmiNumNodes() > 1)
   {
+#if CMK_SMP && !CMK_SMP_NO_COMMTHD
+    if (CmiInCommThread())
+    {
+      CmiIsomallocSyncWaitCommThread(CmiIsomallocSyncCommThreadDone);
+    }
+    else
+#endif
     if (CmiMyRank() == 0)
     {
       if (CmiMyNode() == 0)
@@ -863,7 +870,7 @@ static void CmiIsomallocInitExtent(char ** argv)
       CmiNodeReduceStruct(&IsoRegion, CmiAddressSpaceRegionPup, CmiAddressSpaceRegionMerge,
                           CmiIsomallocSyncReductionHandler, nullptr);
 
-      CmiIsomallocSyncWait();
+      CmiIsomallocSyncWait(CmiIsomallocSyncHandlerDone);
 
       if (CmiMyNode() == 0)
       {
@@ -894,15 +901,9 @@ static void CmiIsomallocInitExtent(char ** argv)
       CmiIsomallocSyncCommThreadDone = 1;
 #endif
     }
-#if CMK_SMP && !CMK_SMP_NO_COMMTHD
-    else if (CmiInCommThread())
-    {
-      CmiIsomallocSyncWaitCommThread();
-    }
-#endif
     else
     {
-      CmiIsomallocSyncWait();
+      CmiIsomallocSyncWait(CmiIsomallocSyncHandlerDone);
     }
 
     CmiBarrier();