Merge pull request #2692 from jsquyres/pr/master/usnic-queue-fixes

master: usnic queue fixes

Author: jsquyres

opal/mca/btl/usnic/btl_usnic_ack.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2013-2016 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2013-2017 Cisco Systems, Inc.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -26,18 +26,24 @@
 #include "btl_usnic_connectivity.h"
 
 /*
- * Force a retrans of a segment
+ * Special case: we know exactly which segment is missing at the
+ * receive; explicitly force retrans of that segment.
  */
 static void
-opal_btl_usnic_force_retrans(
+opal_btl_usnic_fast_retrans(
     opal_btl_usnic_endpoint_t *endpoint,
     opal_btl_usnic_seq_t ack_seq)
 {
     opal_btl_usnic_send_segment_t *sseg;
     int is;
 
-    is = WINDOW_SIZE_MOD(ack_seq+1);
+    is = WINDOW_SIZE_MOD(ack_seq + 1);
     sseg = endpoint->endpoint_sent_segs[is];
+
+    // If the sseg is NULL, then there's nothing to retransmit.  If
+    // the hotel room is -1, the segment has already been queued up
+    // for retransmit and there's nothing additional we need to do
+    // here.
     if (sseg == NULL || sseg->ss_hotel_room == -1) {
         return;
     }
@@ -79,12 +85,14 @@ opal_btl_usnic_handle_ack(
 #endif
         ++module->stats.num_old_dup_acks;
         return;
+    }
 
-    /* A duplicate ACK means next seg was lost */
-    } else if (ack_seq == endpoint->endpoint_ack_seq_rcvd) {
+    /* A duplicate ACK means the sender did not receive the next
+       seg that we sent */
+    else if (ack_seq == endpoint->endpoint_ack_seq_rcvd) {
         ++module->stats.num_dup_acks;
 
-        opal_btl_usnic_force_retrans(endpoint, ack_seq);
+        opal_btl_usnic_fast_retrans(endpoint, ack_seq);
         return;
     }
 
@@ -114,12 +122,11 @@ opal_btl_usnic_handle_ack(
            already been evicted and queued for resend.
            If it's not in the hotel, don't check it out! */
         if (OPAL_LIKELY(sseg->ss_hotel_room != -1)) {
-
             opal_hotel_checkout(&endpoint->endpoint_hotel, sseg->ss_hotel_room);
             sseg->ss_hotel_room = -1;
-
+        }
         /* hotel_room == -1 means queued for resend, remove it */
-        } else {
+        else {
             opal_list_remove_item((&module->pending_resend_segs),
                     &sseg->ss_base.us_list.super);
         }
@@ -191,19 +198,27 @@ opal_btl_usnic_handle_ack(
 /*
  * Send an ACK
  */
-void
+int
 opal_btl_usnic_ack_send(
     opal_btl_usnic_module_t *module,
     opal_btl_usnic_endpoint_t *endpoint)
 {
     opal_btl_usnic_ack_segment_t *ack;
 
+    /* If we don't have any send credits in the priority channel,
+       don't send it */
+    if (module->mod_channels[USNIC_PRIORITY_CHANNEL].credits < 1) {
+        return OPAL_ERR_OUT_OF_RESOURCE;
+    }
+
     /* Get an ACK frag.  If we don't get one, just discard this ACK. */
     ack = opal_btl_usnic_ack_segment_alloc(module);
     if (OPAL_UNLIKELY(NULL == ack)) {
-        return;
+        return OPAL_ERR_OUT_OF_RESOURCE;
     }
 
+    --module->mod_channels[USNIC_PRIORITY_CHANNEL].credits;
+
     /* send the seq of the lowest item in the window that
        we've received */
     ack->ss_base.us_btl_header->ack_seq =
@@ -239,7 +254,7 @@ opal_btl_usnic_ack_send(
     /* Stats */
     ++module->stats.num_ack_sends;
 
-    return;
+    return OPAL_SUCCESS;
 }
 
 /*
@@ -249,6 +264,7 @@ void
 opal_btl_usnic_ack_complete(opal_btl_usnic_module_t *module,
                                    opal_btl_usnic_ack_segment_t *ack)
 {
+    ++module->mod_channels[USNIC_PRIORITY_CHANNEL].credits;
     opal_btl_usnic_ack_segment_return(module, ack);
     ++module->mod_channels[ack->ss_channel].credits;
 }
@@ -291,4 +307,3 @@ opal_btl_usnic_ack_timeout(
     /* Stats */
     ++module->stats.num_timeout_retrans;
 }
-

opal/mca/btl/usnic/btl_usnic_ack.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2013-2016 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2013-2017 Cisco Systems, Inc.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -67,7 +67,7 @@ void opal_btl_usnic_ack_complete(opal_btl_usnic_module_t *module,
 /*
  * Send an ACK
  */
-void opal_btl_usnic_ack_send(opal_btl_usnic_module_t *module,
+int opal_btl_usnic_ack_send(opal_btl_usnic_module_t *module,
                                opal_btl_usnic_endpoint_t *endpoint);
 
 /*

opal/mca/btl/usnic/btl_usnic_component.c

@@ -12,7 +12,7 @@
  *                         All rights reserved.
  * Copyright (c) 2006      Sandia National Laboratories. All rights
  *                         reserved.
- * Copyright (c) 2008-2016 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2008-2017 Cisco Systems, Inc.  All rights reserved.
  * Copyright (c) 2012-2014 Los Alamos National Security, LLC.  All rights
  *                         reserved.
  * Copyright (c) 2014      Intel, Inc. All rights reserved.
@@ -1159,6 +1159,8 @@ static int usnic_component_progress(void)
                 if (OPAL_LIKELY(OPAL_BTL_USNIC_SEG_RECV ==
                             rseg->rs_base.us_type)) {
                     opal_btl_usnic_recv_fast(module, rseg, channel);
+                    ++module->stats.num_seg_total_completions;
+                    ++module->stats.num_seg_recv_completions;
                     fastpath_ok = false;    /* prevent starvation */
                     return 1;
                 } else {
@@ -1188,6 +1190,8 @@ static int usnic_handle_completion(
     seg = (opal_btl_usnic_segment_t*)completion->op_context;
     rseg = (opal_btl_usnic_recv_segment_t*)seg;
 
+    ++module->stats.num_seg_total_completions;
+
     /* Make the completion be Valgrind-defined */
     opal_memchecker_base_mem_defined(seg, sizeof(*seg));
 
@@ -1198,24 +1202,30 @@ static int usnic_handle_completion(
 
     /**** Send ACK completions ****/
     case OPAL_BTL_USNIC_SEG_ACK:
+        ++module->stats.num_seg_ack_completions;
         opal_btl_usnic_ack_complete(module,
                 (opal_btl_usnic_ack_segment_t *)seg);
         break;
 
-    /**** Send of frag segment completion ****/
+    /**** Send of frag segment completion (i.e., the MPI message's
+          one-and-only segment has completed sending) ****/
     case OPAL_BTL_USNIC_SEG_FRAG:
+        ++module->stats.num_seg_frag_completions;
         opal_btl_usnic_frag_send_complete(module,
                 (opal_btl_usnic_frag_segment_t*)seg);
         break;
 
-    /**** Send of chunk segment completion ****/
+    /**** Send of chunk segment completion (i.e., part of a large MPI
+          message is done sending) ****/
     case OPAL_BTL_USNIC_SEG_CHUNK:
+        ++module->stats.num_seg_chunk_completions;
         opal_btl_usnic_chunk_send_complete(module,
                 (opal_btl_usnic_chunk_segment_t*)seg);
         break;
 
     /**** Receive completions ****/
     case OPAL_BTL_USNIC_SEG_RECV:
+        ++module->stats.num_seg_recv_completions;
         opal_btl_usnic_recv(module, rseg, channel);
         break;
 

opal/mca/btl/usnic/btl_usnic_endpoint.h

@@ -11,7 +11,7 @@
  *                         All rights reserved.
  * Copyright (c) 2006      Sandia National Laboratories. All rights
  *                         reserved.
- * Copyright (c) 2013-2016 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2013-2017 Cisco Systems, Inc.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -160,6 +160,8 @@ typedef struct mca_btl_base_endpoint_t {
     opal_btl_usnic_seq_t endpoint_next_seq_to_send; /* n_t */
     opal_btl_usnic_seq_t endpoint_ack_seq_rcvd; /* n_a */
 
+    /* Table where sent segments sit while waiting for their ACKs.
+       When a segment is ACKed, it is removed from this table. */
     struct opal_btl_usnic_send_segment_t *endpoint_sent_segs[WINDOW_SIZE];
 
     /* Values for the current proc to receive from this endpoint on

opal/mca/btl/usnic/btl_usnic_frag.c

@@ -11,7 +11,7 @@
  *                         All rights reserved.
  * Copyright (c) 2006      Sandia National Laboratories. All rights
  *                         reserved.
- * Copyright (c) 2013-2015 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2013-2017 Cisco Systems, Inc.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -50,15 +50,15 @@ common_send_seg_helper(opal_btl_usnic_send_segment_t *seg)
 
 static void
 chunk_seg_constructor(
-    opal_btl_usnic_send_segment_t *seg)
+    opal_btl_usnic_chunk_segment_t *cseg)
 {
     opal_btl_usnic_segment_t *bseg;
 
-    bseg = &seg->ss_base;
+    bseg = &cseg->ss_base;
     bseg->us_type = OPAL_BTL_USNIC_SEG_CHUNK;
 
     /* some more common initializaiton */
-    common_send_seg_helper(seg);
+    common_send_seg_helper(cseg);
 
     /* payload starts next byte beyond BTL chunk header */
     bseg->us_payload.raw = (uint8_t *)(bseg->us_btl_chunk_header + 1);
@@ -68,15 +68,15 @@ chunk_seg_constructor(
 
 static void
 frag_seg_constructor(
-    opal_btl_usnic_send_segment_t *seg)
+    opal_btl_usnic_frag_segment_t *fseg)
 {
     opal_btl_usnic_segment_t *bseg;
 
-    bseg = &seg->ss_base;
+    bseg = &fseg->ss_base;
     bseg->us_type = OPAL_BTL_USNIC_SEG_FRAG;
 
     /* some more common initializaiton */
-    common_send_seg_helper(seg);
+    common_send_seg_helper(fseg);
 
     /* payload starts next byte beyond BTL header */
     bseg->us_payload.raw = (uint8_t *)(bseg->us_btl_header + 1);
@@ -86,7 +86,7 @@ frag_seg_constructor(
 
 static void
 ack_seg_constructor(
-    opal_btl_usnic_send_segment_t *ack)
+    opal_btl_usnic_ack_segment_t *ack)
 {
     opal_btl_usnic_segment_t *bseg;
 

opal/mca/btl/usnic/btl_usnic_frag.h

@@ -11,7 +11,7 @@
  *                         All rights reserved.
  * Copyright (c) 2006      Sandia National Laboratories. All rights
  *                         reserved.
- * Copyright (c) 2013-2015 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2013-2017 Cisco Systems, Inc.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -370,6 +370,7 @@ opal_btl_usnic_small_send_frag_alloc(opal_btl_usnic_module_t *module)
 
     /* this belongs in constructor... */
     frag->ssf_base.sf_base.uf_freelist = &(module->small_send_frags);
+    frag->ssf_segment.ss_send_posted = 0;
 
     assert(frag);
     assert(OPAL_BTL_USNIC_FRAG_SMALL_SEND == frag->ssf_base.sf_base.uf_type);
@@ -480,6 +481,14 @@ opal_btl_usnic_frag_return(
         }
     }
 
+    /* Reset the "send_posted" flag on the embedded segment for small
+       fragments */
+    else if (frag->uf_type == OPAL_BTL_USNIC_FRAG_SMALL_SEND) {
+        opal_btl_usnic_small_send_frag_t *sfrag;
+        sfrag = (opal_btl_usnic_small_send_frag_t *) frag;
+        sfrag->ssf_segment.ss_send_posted = 0;
+    }
+
     USNIC_COMPAT_FREE_LIST_RETURN(frag->uf_freelist, &(frag->uf_base.super));
 }