Use CmiSendSelf in CmiPushPE to correct hangs (#3463)

* Use CmiSendSelf in CmiPushPE to correct hangs

* Tweak Conditional Placement  RE: CMK_IMMEDIATE_MSG

Author: jszaday

src/arch/util/machine-common-core.C

@@ -451,23 +451,28 @@ extern "C" void CmiPushImmediateMsg(void *);
 void CmiPushPE(int rank,void *msg) {
     CmiState cs = CmiGetStateN(rank);
     MACHSTATE2(3,"Pushing message into rank %d's queue %p{",rank, cs->recv);
+
+    if (rank == CmiMyRank()) {
+        CmiSendSelf((char*)msg);
+    } else {
 #if CMK_IMMEDIATE_MSG
-    if (CmiIsImmediate(msg)) {
-        MACHSTATE1(3, "[%p] Push Immediate Message begin{",CmiGetState());
-        CMI_DEST_RANK(msg) = rank;
-        CmiPushImmediateMsg(msg);
-        MACHSTATE1(3, "[%p] Push Immediate Message end}",CmiGetState());
-        return;
-    }
+        if (CmiIsImmediate(msg)) {
+            MACHSTATE1(3, "[%p] Push Immediate Message begin{",CmiGetState());
+            CMI_DEST_RANK(msg) = rank;
+            CmiPushImmediateMsg(msg);
+            MACHSTATE1(3, "[%p] Push Immediate Message end}",CmiGetState());
+            return;
+        }
 #endif
 
 #if CMK_MACH_SPECIALIZED_QUEUE
-    LrtsSpecializedQueuePush(rank, msg);
+        LrtsSpecializedQueuePush(rank, msg);
 #elif CMK_SMP_MULTIQ
-    CMIQueuePush(cs->recv[CmiGetState()->myGrpIdx], (char *)msg);
+        CMIQueuePush(cs->recv[CmiGetState()->myGrpIdx], (char *)msg);
 #else
-    CMIQueuePush(cs->recv,(char*)msg);
+        CMIQueuePush(cs->recv,(char*)msg);
 #endif
+    }
 
 #if CMK_SHARED_VARS_POSIX_THREADS_SMP
   if (_Cmi_sleepOnIdle)