Merge pull request #450 from jsquyres/pr/v2.x-usnic-updates2

Pr/v2.x usnic updates2

Author: jsquyres

opal/mca/btl/usnic/btl_usnic_module.c

@@ -1609,6 +1609,7 @@ static int init_one_channel(opal_btl_usnic_module_t *module,
     channel->fastsend_wqe_thresh = sd_num - 10;
 
     channel->credits = sd_num;
+    channel->rx_post_cnt = 0;
 
     /* We did math up in component_init() to know that there should be
        enough CQs available.  So if create_cq() fails, then either the

opal/mca/btl/usnic/btl_usnic_module.h

@@ -69,6 +69,7 @@ typedef struct opal_btl_usnic_channel_t {
     int chan_sd_num;
 
     int credits;  /* RFXXX until libfab credits fixed */
+    uint32_t rx_post_cnt;
 
     /* fastsend enabled if num_credits_available >= fastsend_wqe_thresh */
     unsigned fastsend_wqe_thresh;

opal/mca/btl/usnic/btl_usnic_recv.h

@@ -23,12 +23,27 @@ void opal_btl_usnic_recv_call(opal_btl_usnic_module_t *module,
 static inline int
 opal_btl_usnic_post_recv_list(opal_btl_usnic_channel_t *channel)
 {
+    struct iovec iov;
+    struct fi_msg msg;
+    uint64_t flag;
     opal_btl_usnic_recv_segment_t *rseg;
     int rc;
 
+    msg.msg_iov = &iov;
+    msg.iov_count = 1;
     for (rseg = channel->repost_recv_head; NULL != rseg; rseg = rseg->rs_next) {
-        rc = fi_recv(channel->ep, rseg->rs_protocol_header,
-                     rseg->rs_len, NULL, FI_ADDR_UNSPEC, rseg);
+        msg.context = rseg;
+        iov.iov_base = rseg->rs_protocol_header;
+        iov.iov_len = rseg->rs_len;
+
+        ++channel->rx_post_cnt;
+        if (OPAL_UNLIKELY((channel->rx_post_cnt & 15) == 0)) {
+            flag = 0;
+        } else {
+            flag = FI_MORE;
+        }
+
+        rc = fi_recvmsg(channel->ep, &msg, flag);
         if (0 != rc) {
             return rc;
         }

opal/mca/btl/usnic/configure.m4

@@ -47,7 +47,7 @@ AC_DEFUN([MCA_opal_btl_usnic_CONFIG],[
 ])
 
 AC_DEFUN([_OPAL_BTL_USNIC_DO_CONFIG],[
-    OPAL_VAR_SCOPE_PUSH([unit_tests opal_btl_usnic_CPPFLAGS_save])
+    OPAL_VAR_SCOPE_PUSH([unit_tests])
 
     # see README.test for information about this scheme
     AC_ARG_ENABLE([opal-btl-usnic-unit-tests],
@@ -99,6 +99,26 @@ AC_DEFUN([_OPAL_BTL_USNIC_DO_CONFIG],[
           [opal_btl_usnic_happy=yes],
           [opal_btl_usnic_happy=no])
 
+    # The usnic BTL requires at least libfabric v1.1 (there was a
+    # critical bug in libfabric v1.0).
+    AS_IF([test "$opal_btl_usnic_happy" = "yes"],
+          [AC_MSG_CHECKING([whether libfabric is >= v1.1])
+           opal_btl_usnic_CPPFLAGS_save=$CPPFLAGS
+           CPPFLAGS="$opal_common_libfabric_CPPFLAGS $CPPFLAGS"
+           AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <rdma/fabric.h>]],
+[[
+#if !defined(FI_MAJOR_VERSION)
+#error your version of libfabric is too old
+#elif FI_VERSION(FI_MAJOR_VERSION, FI_MINOR_VERSION) < FI_VERSION(1, 1)
+#error your version of libfabric is too old
+#endif
+]])],
+                 [opal_btl_usnic_happy=yes],
+                 [opal_btl_usnic_happy=no])
+           AC_MSG_RESULT([$opal_btl_usnic_happy])
+           CPPFLAGS=$opal_btl_usnic_CPPFLAGS_save
+          ])
+
     # Make sure we can find the libfabric usnic extensions header
     AS_IF([test "$opal_btl_usnic_happy" = "yes" ],
           [opal_btl_usnic_CPPFLAGS_save=$CPPFLAGS