Merge pull request #385 from ggouaillardet/topic/v2.x/configury_xrc_2

configury: fix XRC detection on OFED < 3.12

Author: jsquyres

config/opal_check_openfabrics.m4

@@ -148,21 +148,42 @@ AC_DEFUN([OPAL_CHECK_OPENFABRICS],[
            AC_CHECK_MEMBERS([struct ibv_device.transport_type], [], [],
                             [#include <infiniband/verbs.h>])
 
+           # We have to check functions both exits *and* are declared
+           # since some distro ship broken ibverbs devel headers
+           # IBV_DEVICE_XRC is common to all OFED versions
            # ibv_create_xrc_rcv_qp was added in OFED 1.3
            # ibv_cmd_open_xrcd (aka XRC Domains) was added in  OFED 3.12
            if test "$enable_connectx_xrc" = "yes"; then
-               $1_have_xrc=1
-               AC_CHECK_FUNCS([ibv_create_xrc_rcv_qp ibv_cmd_open_xrcd],
-                              [], [$1_have_xrc=0])
-               AC_CHECK_DECLS([IBV_SRQT_XRC],
-                              [], [$1_have_xrc=0],
+               AC_CHECK_DECLS([IBV_DEVICE_XRC],
+                              [$1_have_xrc=1
+                               $1_have_xrc_domains=1],
+                              [],
                               [#include <infiniband/verbs.h>])
            fi
            if test "$enable_connectx_xrc" = "yes" \
                && test $$1_have_xrc -eq 1; then
-               AC_CHECK_FUNCS([ibv_cmd_open_xrcd], [$1_have_xrc_domains=1])
+               AC_CHECK_DECLS([ibv_create_xrc_rcv_qp],
+                              [AC_CHECK_FUNCS([ibv_create_xrc_rcv_qp],
+                                              [], [$1_have_xrc=0])],
+                              [$1_have_xrc=0],
+                              [#include <infiniband/driver.h>])
+           fi
+           if test "$enable_connectx_xrc" = "yes" \
+               && test $$1_have_xrc_domains -eq 1; then
+               AC_CHECK_DECLS([ibv_cmd_open_xrcd],
+                              [AC_CHECK_DECLS([IBV_SRQT_XRC],
+                                              [AC_CHECK_FUNCS([ibv_cmd_open_xrcd],
+                                                              [], [$1_have_xrc_domains=0])],
+                                              [$1_have_xrc_domains=0],
+                                              [#include <infiniband/verbs.h>])],
+                              [$1_have_xrc_domains=0],
+                              [#include <infiniband/driver.h>])
+               # XRC and XRC Domains should be considered as exclusive
+               if test "$$1_have_xrc" -eq 1 && \
+                  test "$$1_have_xrc_domains" -eq 1; then
+                      $1_have_xrc=0
+               fi
            fi
-
 
            if test "no" != "$enable_openib_dynamic_sl"; then
                # We need ib_types.h file, which is installed with opensm-devel

opal/mca/btl/openib/btl_openib.h

@@ -53,8 +53,8 @@
 
 BEGIN_C_DECLS
 
-#define HAVE_XRC (1 == OPAL_HAVE_CONNECTX_XRC)
-#define ENABLE_DYNAMIC_SL (1 == OPAL_ENABLE_DYNAMIC_SL)
+#define HAVE_XRC (OPAL_HAVE_CONNECTX_XRC || OPAL_HAVE_CONNECTX_XRC_DOMAINS)
+#define ENABLE_DYNAMIC_SL OPAL_ENABLE_DYNAMIC_SL
 
 #define MCA_BTL_IB_LEAVE_PINNED 1
 #define IB_DEFAULT_GID_PREFIX 0xfe80000000000000ll

opal/mca/btl/openib/btl_openib_async.c

@@ -122,7 +122,7 @@ static mca_btl_openib_endpoint_t * qp2endpoint(struct ibv_qp *qp, mca_btl_openib
     return NULL;
 }
 
-#if HAVE_XRC && !OPAL_HAVE_CONNECTX_XRC_DOMAINS
+#if OPAL_HAVE_CONNECTX_XRC
 /* XRC recive QP to endpoint */
 static mca_btl_openib_endpoint_t * xrc_qp2endpoint(uint32_t qp_num, mca_btl_openib_device_t *device)
 {
@@ -352,24 +352,21 @@ static int btl_openib_async_deviceh(struct mca_btl_openib_async_poll *devices_po
         }
 
         event_type = event.event_type;
-#if HAVE_XRC
+#if OPAL_HAVE_CONNECTX_XRC
         /* is it XRC event ?*/
-#if OPAL_HAVE_CONNECTX_XRC_DOMAINS
-#else
         bool xrc_event = false;
         if (IBV_XRC_QP_EVENT_FLAG & event.event_type) {
             xrc_event = true;
             /* Clean the bitnd handel as usual */
             event_type ^= IBV_XRC_QP_EVENT_FLAG;
         }
-#endif
 #endif
         switch(event_type) {
             case IBV_EVENT_PATH_MIG:
                 BTL_ERROR(("Alternative path migration event reported"));
                 if (APM_ENABLED) {
                     BTL_ERROR(("Trying to find additional path..."));
-#if HAVE_XRC && !OPAL_HAVE_CONNECTX_XRC_DOMAINS
+#if OPAL_HAVE_CONNECTX_XRC
                     if (xrc_event)
                         mca_btl_openib_load_apm_xrc_rcv(event.element.xrc_qp_num,
                                 xrc_qp2endpoint(event.element.xrc_qp_num, device));
@@ -653,7 +650,7 @@ void mca_btl_openib_load_apm(struct ibv_qp *qp, mca_btl_openib_endpoint_t *ep)
                    qp->qp_num, strerror(errno), errno));
 }
 
-#if HAVE_XRC && ! OPAL_HAVE_CONNECTX_XRC_DOMAINS
+#if OPAL_HAVE_CONNECTX_XRC
 void mca_btl_openib_load_apm_xrc_rcv(uint32_t qp_num, mca_btl_openib_endpoint_t *ep)
 {
     struct ibv_qp_init_attr qp_init_attr;

opal/mca/btl/openib/btl_openib_async.h

@@ -19,7 +19,7 @@
 int        start_async_event_thread(void);
 void       mca_btl_openib_load_apm(struct ibv_qp *qp, mca_btl_openib_endpoint_t *ep);
 int        btl_openib_async_command_done(int exp);
-#if HAVE_XRC && ! OPAL_HAVE_CONNECTX_XRC_DOMAINS
+#if OPAL_HAVE_CONNECTX_XRC
 void       mca_btl_openib_load_apm_xrc_rcv(uint32_t qp_num, mca_btl_openib_endpoint_t *ep);
 #endif
 

opal/mca/btl/openib/btl_openib_atomic.c

@@ -73,7 +73,12 @@ static int mca_btl_openib_atomic_internal (struct mca_btl_base_module_t *btl, st
 
 #if HAVE_XRC
     if (MCA_BTL_XRC_ENABLED && BTL_OPENIB_QP_TYPE_XRC(qp)) {
-        frag->sr_desc.xrc_remote_srq_num=endpoint->rem_info.rem_srqs[qp].rem_srq_num;
+#if OPAL_HAVE_CONNECTX_XRC_DOMAINS
+        frag->sr_desc.qp_type.xrc.remote_srqn = endpoint->rem_info.rem_srqs[qp].rem_srq_num;
+#else
+        frag->sr_desc.xrc_remote_srq_num = endpoint->rem_info.rem_srqs[qp].rem_srq_num;
+#endif
+
     }
 #endif
 

opal/mca/btl/openib/btl_openib_put.c

@@ -103,10 +103,13 @@ int mca_btl_openib_put (mca_btl_base_module_t *btl, struct mca_btl_base_endpoint
 
 #if HAVE_XRC
     if (MCA_BTL_XRC_ENABLED && BTL_OPENIB_QP_TYPE_XRC(qp)) {
-#if OPAL_HAVE_CONNECTX_XRC_DOMAINS
+
+#if OPAL_HAVE_CONNECTX_XRC
+        to_out_frag(frag)->sr_desc.xrc_remote_srq_num = ep->rem_info.rem_srqs[qp].rem_srq_num;
+#elif OPAL_HAVE_CONNECTX_XRC_DOMAINS
         to_out_frag(frag)->sr_desc.qp_type.xrc.remote_srqn = ep->rem_info.rem_srqs[qp].rem_srq_num;
 #else
-        to_out_frag(frag)->sr_desc.xrc_remote_srq_num = ep->rem_info.rem_srqs[qp].rem_srq_num;
+#error "that should never happen"
 #endif
     }
 #endif