btl/openib: update experimental verbs support

This update adds an additional check (if supported) to see if 8-byte
atomics are supported by the hardware. If 8-byte atomics are not
supported the atomics support is disabled.

This commit also includes some cleanup.

Signed-off-by: Nathan Hjelm <[email protected]>

(cherry picked from open-mpi/ompi@bb8e347)

Author: hjelmn

config/opal_check_openfabrics.m4

@@ -397,9 +397,9 @@ AC_DEFUN([OPAL_CHECK_EXP_VERBS],[
                    [have_struct_ibv_exp_send_wr=0
                     AC_MSG_RESULT([no])])
 
-    AC_DEFINE_UNQUOTED([HAVE_EXP_VERBS], [$have_struct_ibv_exp_send_wr], [Expanded verbs])
-    AC_CHECK_DECLS([IBV_EXP_ATOMIC_HCA_REPLY_BE, IBV_EXP_QP_CREATE_ATOMIC_BE_REPLY, ibv_exp_create_qp], [], [], [#include <infiniband/verbs_exp.h>])
-    AC_CHECK_HEADERS([infiniband/verbs_exp.h])
+    AC_DEFINE_UNQUOTED([HAVE_EXP_VERBS], [$have_struct_ibv_exp_send_wr], [Experimental verbs])
+    AC_CHECK_DECLS([IBV_EXP_ATOMIC_HCA_REPLY_BE, IBV_EXP_QP_CREATE_ATOMIC_BE_REPLY, ibv_exp_create_qp, ibv_exp_query_device],
+                   [], [], [#include <infiniband/verbs_exp.h>])
     AS_IF([test '$have_struct_ibv_exp_send_wr' = 1], [$1], [$2])
     OPAL_VAR_SCOPE_POP
 ])dnl

opal/mca/btl/openib/btl_openib.h

@@ -371,7 +371,11 @@ typedef struct mca_btl_openib_device_t {
 #endif
     opal_mutex_t device_lock;          /* device level lock */
     struct ibv_context *ib_dev_context;
+#if HAVE_DECL_IBV_EXP_QUERY_DEVICE
+    struct ibv_exp_device_attr ib_dev_attr;
+#else
     struct ibv_device_attr ib_dev_attr;
+#endif
     struct ibv_pd *ib_pd;
     struct ibv_cq *ib_cq[2];
     uint32_t cq_size[2];

opal/mca/btl/openib/btl_openib_component.c

@@ -824,6 +824,16 @@ static int init_one_port(opal_list_t *btl_list, mca_btl_openib_device_t *device,
 #if HAVE_DECL_IBV_ATOMIC_HCA
             openib_btl->atomic_ops_be = false;
 
+#if HAVE_DECL_IBV_EXP_QUERY_DEVICE
+            /* check that 8-byte atomics are supported */
+            if (!(device->dev_attr.ext_atom.log_atomic_arg_sizes & (1<<3ull))) {
+                openib_btl->super.btl_flags &= ~MCA_BTL_FLAGS_ATOMIC_FOPS;
+                openib_btl->super.btl_atomic_flags = 0;
+                openib_btl->super.btl_atomic_fop = NULL;
+                openib_btl->super.btl_atomic_cswap = NULL;
+            }
+#endif
+
             switch (openib_btl->device->ib_dev_attr.atomic_cap) {
             case IBV_ATOMIC_GLOB:
                 openib_btl->super.btl_flags |= MCA_BTL_ATOMIC_SUPPORTS_GLOB;
@@ -1643,12 +1653,19 @@ static int init_one_device(opal_list_t *btl_list, struct ibv_device* ib_dev)
                     ibv_get_device_name(device->ib_dev), strerror(errno)));
         goto error;
     }
-
+#if HAVE_DECL_IBV_EXP_QUERY_DEVICE
+    if(ibv_exp_query_device(device->ib_dev_context, &device->ib_dev_attr)){
+        BTL_ERROR(("error obtaining device attributes for %s errno says %s",
+                    ibv_get_device_name(device->ib_dev), strerror(errno)));
+        goto error;
+    }
+#else
     if(ibv_query_device(device->ib_dev_context, &device->ib_dev_attr)){
         BTL_ERROR(("error obtaining device attributes for %s errno says %s",
                     ibv_get_device_name(device->ib_dev), strerror(errno)));
         goto error;
     }
+#endif
     /* If mca_btl_if_include/exclude were specified, get usable ports */
     allowed_ports = (int*)malloc(device->ib_dev_attr.phys_port_cnt * sizeof(int));
     if (NULL == allowed_ports) {

opal/mca/btl/openib/connect/btl_openib_connect_udcm.c

@@ -56,9 +56,6 @@
 #include <sys/types.h>
 #include <fcntl.h>
 #include <infiniband/verbs.h>
-#ifdef HAVE_INFINIBAND_VERBS_EXP_H
-#include <infiniband/verbs_exp.h>
-#endif
 #include <signal.h>
 
 #include <pthread.h>
@@ -1341,7 +1338,7 @@ static int udcm_rc_qp_create_one(udcm_module_t *m, mca_btl_base_endpoint_t* lcl_
     init_attr.pd = m->btl->device->ib_pd;
 
     init_attr.comp_mask |= IBV_EXP_QP_INIT_ATTR_ATOMICS_ARG;
-    init_attr.max_atomic_arg = 8;
+    init_attr.max_atomic_arg = sizeof (int64_t);
 
 #if HAVE_DECL_IBV_EXP_ATOMIC_HCA_REPLY_BE
     if (IBV_EXP_ATOMIC_HCA_REPLY_BE == m->btl->device->ib_dev_attr.atomic_cap) {