Modifies converse schedeuler's getNextMessage so nodeGroup messages can run with higher priority over local

As it is, nodeGroup messages are not checked until all local and regular Charm queue (prio Q) messages are
checked, which cause issues when the applicaiton is using nodeGroup messages in the hope that *some* PE will attend to it quickly. The change makes getNextMessage check nodeGroup queue every 2^nodeGrpFreq iterations with
high priority in addition to its usual check after exhasuting local queues (except task Q).
This commit has not been tested at all. But pusing it to allow others to help me test/fix it.

Author: Laxmikant Kale

src/conv-core/convcore.C

@@ -1656,6 +1656,7 @@ void CsdSchedulerState_new(CsdSchedulerState_t *s)
 #if CMK_NODE_QUEUE_AVAILABLE
 	s->nodeQ=CsvAccess(CsdNodeQueue);
 	s->nodeLock=CsvAccess(CsdNodeQueueLock);
+	s->nodeGrpFreq =2; // check node queue once in 4 (2^2) iterations. 
 #endif
 #if CMK_GRID_QUEUE_AVAILABLE
 	s->gridQ=CpvAccess(CsdGridQueue);
@@ -1664,6 +1665,7 @@ void CsdSchedulerState_new(CsdSchedulerState_t *s)
 	s->taskQ = CpvAccess(CsdTaskQueue);
 	s->suspendedTaskQ = CpvAccess(CmiSuspendedTaskQueue);
 #endif
+
 }
 
 
@@ -1719,7 +1721,20 @@ void CsdSchedulerState_new(CsdSchedulerState_t *s)
  */
 void *CsdNextMessage(CsdSchedulerState_t *s) {
 	void *msg;
-	if((*(s->localCounter))-- >0)
+
+s->iter++;
+
+#if CMK_NODE_QUEUE_AVAILABLE
+	if (1 == (s->iter & (1 << s->nodeGrpFreq)) // since we use nodeGrpFreq == 0 to mean
+                                           // don't check NodeQ with high priority, i
+					   // value of 1 serves well as when to check it.i but we sshould avoid "%" operator
+					// note: s->nodeGrpFreq is raised to a power of  2
+	{
+	  msg = CmiGetNonLocalNodeQ();
+	  if (NULL != msg) return msg;
+	}
+#endif
+        if((*(s->localCounter))-- >0)
 	  {
               /* This avoids a race condition with migration detected by megatest*/
               msg=CdsFifo_Dequeue(s->localQ);

src/conv-core/converse.h

@@ -1145,6 +1145,12 @@ typedef struct {
   void *localQ;
   Queue nodeQ;
   Queue schedQ;
+  unsigned short iter; // counting number of sched iterations (hopefully of for it to roll over	 
+  unsigned short nodeGrpFreq; // call nodegroup queue once every 2^nodeGrpFreq iterations with high prio 
+	// should add a function to change this from the program for advanced users. One obstacle: 
+        // it is inside a struct that is on stack, and so not accessible for standalone functions. Need to
+        // resolve this by making a schedule a c++ object, but even then we need a ptr to the currently-running scheduler
+	// 0 means do not check nodegroup queue with high prio.. will be checked with low prio after other Qs
   int *localCounter;
 #if CMK_OBJECT_QUEUE_AVAILABLE
   Queue objQ;