Properly mark a node as down and decrease the number of daemons so any

subsequent grpcomm collectives can correctly operate. Note that only the
direct grpcomm component knows how to deal with down nodes.

Author: Ralph Castain

orte/mca/errmgr/dvm/errmgr_dvm.c

@@ -331,6 +331,10 @@ static void proc_errors(int fd, short args, void *cbdata)
         }
         /* mark the daemon as gone */
         ORTE_FLAG_UNSET(pptr, ORTE_PROC_FLAG_ALIVE);
+        /* update the state */
+        pptr->state = state;
+        /* adjust our num_procs */
+        --orte_process_info.num_procs;
         /* if we have ordered orteds to terminate or abort
          * is in progress, record it */
         if (orte_orteds_term_ordered || orte_abnormal_term_ordered) {

orte/mca/grpcomm/direct/grpcomm_direct.c

@@ -432,7 +432,8 @@ static void xcast_recv(int status, orte_process_name_t* sender,
             OBJ_RELEASE(item);
             continue;
         }
-        if (ORTE_PROC_STATE_RUNNING < rec->state) {
+        if (ORTE_PROC_STATE_RUNNING < rec->state ||
+            !ORTE_FLAG_TEST(rec, ORTE_PROC_FLAG_ALIVE)) {
             opal_output(0, "%s grpcomm:direct:send_relay proc %s not running - cannot relay",
                         ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), ORTE_NAME_PRINT(&nm->name));
             OBJ_RELEASE(rly);