cmiArgDebugFlag and CMK_CCS_AVAILABLE fixes (#2711) (#2730)

Make cmiArgDebugFlag a process/node level global variable

Previously, cmiArgDebugFlag was a Cpv variable, even though there
was no need to have it as a PE-private variable. This change makes
cmiArgDebugFlag a node-level global variable, which is initialized
in ConverseInit.

Additionally, this change also removes the macro that set
CMK_CCS_AVAILABLE in conv-common.h for the UCX machine layer.
Unlike other layers, CMK_CCS_AVAILABLE was separately set for
the UCX layer in conv-common.h. By default, for all layers,
CMK_CCS_AVAILABLE is set to 1 by the configure script, unless
specifically disabled.

Co-authored-by: Evan Ramos <[email protected]>

Author: evan-charmworks

src/arch/netlrts/machine.C

@@ -361,7 +361,7 @@ static void KillOnAllSigs(int sigNo)
   already_in_signal_handler=1;
 
 #if CMK_CCS_AVAILABLE
-  if (CpvAccess(cmiArgDebugFlag)) {
+  if (cmiArgDebugFlag && CmiMyRank() == 0) {
     int reply = 0;
     CpdNotify(CPD_SIGNAL,sigNo);
 #if ! CMK_BIGSIM_CHARM

src/arch/ucx/conv-common.h

@@ -54,9 +54,6 @@
    to a processors are implemented in convcore.c (1) or in machine.C (1). */
 #define CMK_VECTOR_SEND_USES_COMMON_CODE                   1
 
-/* Enable the CCS protocol if set to 1. */
-#define CMK_CCS_AVAILABLE                                  1
-
 /* Defines if there is a "charmrun" program running on the system, which
    interacts with possible connecting clients (0), or if there is no such
    program, and processor 0 does the job (1). Currently only netlrts- and

src/arch/util/machine-common-core.C

@@ -1459,6 +1459,12 @@ if (  MSG_STATISTIC)
     }
 #endif
 
+    if(CmiGetArgFlagDesc(argv, "+cpd", "Used *only* in conjunction with parallel debugger")) {
+#if CMK_CCS_AVAILABLE
+        cmiArgDebugFlag = 1;
+#endif
+    }
+
     /* CmiTimerInit(); */
 #if CMK_BROADCAST_HYPERCUBE
     /* CmiNodesDim = ceil(log2(CmiNumNodes)) except when #nodes is 1*/
@@ -1497,22 +1503,6 @@ static void ConverseRunPE(int everReturn) {
     CmiState cs;
     char** CmiMyArgv;
 
-#if CMK_CCS_AVAILABLE
-/**
- * The reason to initialize this variable here:
- * cmiArgDebugFlag is possibly accessed in CmiPrintf/CmiError etc.,
- * therefore, we have to initialize this variable before any calls
- * to those functions (such as CmiPrintf). Otherwise, we may encounter
- * a memory segmentation fault (bad memory access). Note, even
- * testing CpvInitialized(cmiArgDebugFlag) doesn't help to solve
- * this problem because the variable indicating whether cmiArgDebugFlag is
- * initialized or not is not initialized, thus possibly causing another
- * bad memory access. --Chao Mei
- */
-  CpvInitialize(int, cmiArgDebugFlag);
-  CpvAccess(cmiArgDebugFlag) = 0;
-#endif
-
     LrtsPreCommonInit(everReturn);
 
 #if CMK_OFFLOAD_BCAST_PROCESS

src/arch/verbs/machine.C

@@ -350,7 +350,7 @@ static void KillOnAllSigs(int sigNo)
   already_in_signal_handler=1;
 
 #if CMK_CCS_AVAILABLE
-  if (CpvAccess(cmiArgDebugFlag)) {
+  if (cmiArgDebugFlag && CmiMyRank() == 0) {
     int reply = 0;
     CpdNotify(CPD_SIGNAL,sigNo);
 #if ! CMK_BIGSIM_CHARM

src/ck-core/debug-charm.C

@@ -69,7 +69,7 @@ int CpdInUserCode() {return cpdInSystem==0 && _debugData.length()>0 && _debugDat
 // Function called right before an entry method
 void CpdBeforeEp(int ep, void *obj, void *msg) {
 #if CMK_CHARMDEBUG
-  if (CpvAccess(cmiArgDebugFlag)) {
+  if (cmiArgDebugFlag && CmiMyRank() == 0) {
     DebugRecursiveEntry entry;
     entry.previousChareID = setMemoryChareIDFromPtr(obj);
     entry.alreadyUserCode = _entryTable[ep]->inCharm ? 0 : 1;
@@ -99,7 +99,7 @@ void CpdBeforeEp(int ep, void *obj, void *msg) {
 // Function called right after an entry method
 void CpdAfterEp(int ep) {
 #if CMK_CHARMDEBUG
-  if (CpvAccess(cmiArgDebugFlag)) {
+  if (cmiArgDebugFlag && CmiMyRank() == 0) {
     DebugRecursiveEntry entry = _debugData.peek();
     std::vector<DebugPersistentCheck> &postExecutes = CkpvAccess(_debugEntryTable)[ep].postProcess;
     for (int i=0; i<postExecutes.size(); ++i) {

src/conv-ccs/conv-ccs.C

@@ -538,7 +538,7 @@ int _isCcsHandlerIdx(int hIdx) {
 
 void CcsBuiltinsInit(char **argv);
 
-CpvDeclare(int, cmiArgDebugFlag);
+int cmiArgDebugFlag; // Value is 0, unless reset in ConverseCommonInit
 CpvDeclare(char *, displayArgument);
 CpvCExtern(int, cpdSuspendStartup);
 CpvDeclare(int, cpdSuspendStartup);
@@ -583,9 +583,8 @@ void CcsInit(char **argv)
   }
 #endif
   /* if in parallel debug mode i.e ++cpd, freeze */
-  if (CmiGetArgFlagDesc(argv, "+cpd", "Used *only* in conjunction with parallel debugger"))
+  if (cmiArgDebugFlag && CmiMyRank() == 0)
   {
-    if(CmiMyRank() == 0) CpvAccess(cmiArgDebugFlag) = 1;
      if (CmiGetArgStringDesc(argv, "+DebugDisplay",&(CpvAccess(displayArgument)), "X display for gdb used only in cpd mode"))
      {
         if (CpvAccess(displayArgument) == NULL)

src/conv-core/convcore.C

@@ -3964,7 +3964,7 @@ void CmiPrintf(const char *format, ...)
   }
   va_end(args);
 #if CMK_CCS_AVAILABLE && CMK_CMIPRINTF_IS_A_BUILTIN
-  if (CpvAccess(cmiArgDebugFlag)) {
+  if (cmiArgDebugFlag && CmiMyRank() == 0) {
     va_start(args,format);
     print_node0(format, args);
     va_end(args);
@@ -3984,7 +3984,7 @@ void CmiError(const char *format, ...)
   CmiFlush(stderr);  /* stderr is always flushed */
   va_end(args);
 #if CMK_CCS_AVAILABLE && CMK_CMIPRINTF_IS_A_BUILTIN
-  if (CpvAccess(cmiArgDebugFlag)) {
+  if (cmiArgDebugFlag && CmiMyRank() == 0) {
     va_start(args,format);
     print_node0(format, args);
     va_end(args);

src/conv-core/debug-conv.C

@@ -371,7 +371,7 @@ void CpdInit(void)
 /* If CharmDebug is attached, try to send it a message and wait */
 void CpdAborting(const char *message) {
 #if CMK_CCS_AVAILABLE
-  if (CpvAccess(cmiArgDebugFlag)) {
+  if (cmiArgDebugFlag && CmiMyRank() == 0) {
     CpdNotify(CPD_ABORT, message);
     CpdFreeze();
   }

src/conv-core/debug-conv.h

@@ -26,7 +26,7 @@ extern void (*CpdDebug_pupMemStat)(pup_er p, void *data);
 extern void (*CpdDebug_deleteMemStat)(void *ptr);
 extern void * (*CpdDebug_mergeMemStat)(int *size, void *data, void **remoteData, int numRemote);
 
-CpvExtern(int, cmiArgDebugFlag);
+extern int cmiArgDebugFlag; // Value is 0, unless reset in ConverseCommonInit
 extern char ** memoryBackup;
 extern void CpdCheckMemory(void);
 extern void CpdResetMemory(void);