configury: fix XRC detection on OFED < 3.12

since ibv_create_xrc_rcv_qp is now deprecated, and in order to
be "future-proof", we have to consider the case in which only XRC Domains are supported.
also, correctly handle distro that ship broken ibverbs devel headers

Thanks Paul Hargrove for the detailled report.

(cherry picked from commit open-mpi/ompi@f7cf7d5)

Author: ggouaillardet

config/opal_check_openfabrics.m4

@@ -148,21 +148,42 @@ AC_DEFUN([OPAL_CHECK_OPENFABRICS],[
            AC_CHECK_MEMBERS([struct ibv_device.transport_type], [], [],
                             [#include <infiniband/verbs.h>])
 
+           # We have to check functions both exits *and* are declared
+           # since some distro ship broken ibverbs devel headers
+           # IBV_DEVICE_XRC is common to all OFED versions
            # ibv_create_xrc_rcv_qp was added in OFED 1.3
            # ibv_cmd_open_xrcd (aka XRC Domains) was added in  OFED 3.12
            if test "$enable_connectx_xrc" = "yes"; then
-               $1_have_xrc=1
-               AC_CHECK_FUNCS([ibv_create_xrc_rcv_qp],
-                              [], [$1_have_xrc=0])
-               AC_CHECK_DECLS([IBV_SRQT_XRC],
-                              [], [$1_have_xrc=0],
+               AC_CHECK_DECLS([IBV_DEVICE_XRC],
+                              [$1_have_xrc=1
+                               $1_have_xrc_domains=1],
+                              [],
                               [#include <infiniband/verbs.h>])
            fi
            if test "$enable_connectx_xrc" = "yes" \
                && test $$1_have_xrc -eq 1; then
-               AC_CHECK_FUNCS([ibv_cmd_open_xrcd], [$1_have_xrc_domains=1])
+               AC_CHECK_DECLS([ibv_create_xrc_rcv_qp],
+                              [AC_CHECK_FUNCS([ibv_create_xrc_rcv_qp],
+                                              [], [$1_have_xrc=0])],
+                              [$1_have_xrc=0],
+                              [#include <infiniband/driver.h>])
+           fi
+           if test "$enable_connectx_xrc" = "yes" \
+               && test $$1_have_xrc_domains -eq 1; then
+               AC_CHECK_DECLS([ibv_cmd_open_xrcd],
+                              [AC_CHECK_DECLS([IBV_SRQT_XRC],
+                                              [AC_CHECK_FUNCS([ibv_cmd_open_xrcd],
+                                                              [], [$1_have_xrc_domains=0])],
+                                              [$1_have_xrc_domains=0],
+                                              [#include <infiniband/verbs.h>])],
+                              [$1_have_xrc_domains=0],
+                              [#include <infiniband/driver.h>])
+               # XRC and XRC Domains should be considered as exclusive
+               if test "$$1_have_xrc" -eq 1 && \
+                  test "$$1_have_xrc_domains" -eq 1; then
+                      $1_have_xrc=0
+               fi
            fi
-
 
            if test "no" != "$enable_openib_dynamic_sl"; then
                # We need ib_types.h file, which is installed with opensm-devel

opal/mca/btl/openib/btl_openib.h

@@ -53,8 +53,8 @@
 
 BEGIN_C_DECLS
 
-#define HAVE_XRC (1 == OPAL_HAVE_CONNECTX_XRC)
-#define ENABLE_DYNAMIC_SL (1 == OPAL_ENABLE_DYNAMIC_SL)
+#define HAVE_XRC (OPAL_HAVE_CONNECTX_XRC || OPAL_HAVE_CONNECTX_XRC_DOMAINS)
+#define ENABLE_DYNAMIC_SL OPAL_ENABLE_DYNAMIC_SL
 
 #define MCA_BTL_IB_LEAVE_PINNED 1
 #define IB_DEFAULT_GID_PREFIX 0xfe80000000000000ll

opal/mca/btl/openib/btl_openib_async.c

@@ -122,7 +122,7 @@ static mca_btl_openib_endpoint_t * qp2endpoint(struct ibv_qp *qp, mca_btl_openib
     return NULL;
 }
 
-#if HAVE_XRC && !OPAL_HAVE_CONNECTX_XRC_DOMAINS
+#if OPAL_HAVE_CONNECTX_XRC
 /* XRC recive QP to endpoint */
 static mca_btl_openib_endpoint_t * xrc_qp2endpoint(uint32_t qp_num, mca_btl_openib_device_t *device)
 {
@@ -352,24 +352,21 @@ static int btl_openib_async_deviceh(struct mca_btl_openib_async_poll *devices_po
         }
 
         event_type = event.event_type;
-#if HAVE_XRC
+#if OPAL_HAVE_CONNECTX_XRC
         /* is it XRC event ?*/
-#if OPAL_HAVE_CONNECTX_XRC_DOMAINS
-#else
         bool xrc_event = false;
         if (IBV_XRC_QP_EVENT_FLAG & event.event_type) {
             xrc_event = true;
             /* Clean the bitnd handel as usual */
             event_type ^= IBV_XRC_QP_EVENT_FLAG;
         }
-#endif
 #endif
         switch(event_type) {
             case IBV_EVENT_PATH_MIG:
                 BTL_ERROR(("Alternative path migration event reported"));
                 if (APM_ENABLED) {
                     BTL_ERROR(("Trying to find additional path..."));
-#if HAVE_XRC && !OPAL_HAVE_CONNECTX_XRC_DOMAINS
+#if OPAL_HAVE_CONNECTX_XRC
                     if (xrc_event)
                         mca_btl_openib_load_apm_xrc_rcv(event.element.xrc_qp_num,
                                 xrc_qp2endpoint(event.element.xrc_qp_num, device));
@@ -653,7 +650,7 @@ void mca_btl_openib_load_apm(struct ibv_qp *qp, mca_btl_openib_endpoint_t *ep)
                    qp->qp_num, strerror(errno), errno));
 }
 
-#if HAVE_XRC && ! OPAL_HAVE_CONNECTX_XRC_DOMAINS
+#if OPAL_HAVE_CONNECTX_XRC
 void mca_btl_openib_load_apm_xrc_rcv(uint32_t qp_num, mca_btl_openib_endpoint_t *ep)
 {
     struct ibv_qp_init_attr qp_init_attr;

opal/mca/btl/openib/btl_openib_async.h

@@ -19,7 +19,7 @@
 int        start_async_event_thread(void);
 void       mca_btl_openib_load_apm(struct ibv_qp *qp, mca_btl_openib_endpoint_t *ep);
 int        btl_openib_async_command_done(int exp);
-#if HAVE_XRC && ! OPAL_HAVE_CONNECTX_XRC_DOMAINS
+#if OPAL_HAVE_CONNECTX_XRC
 void       mca_btl_openib_load_apm_xrc_rcv(uint32_t qp_num, mca_btl_openib_endpoint_t *ep);
 #endif
 

opal/mca/btl/openib/btl_openib_atomic.c

@@ -73,7 +73,12 @@ static int mca_btl_openib_atomic_internal (struct mca_btl_base_module_t *btl, st
 
 #if HAVE_XRC
     if (MCA_BTL_XRC_ENABLED && BTL_OPENIB_QP_TYPE_XRC(qp)) {
-        frag->sr_desc.xrc_remote_srq_num=endpoint->rem_info.rem_srqs[qp].rem_srq_num;
+#if OPAL_HAVE_CONNECTX_XRC_DOMAINS
+        frag->sr_desc.qp_type.xrc.remote_srqn = endpoint->rem_info.rem_srqs[qp].rem_srq_num;
+#else
+        frag->sr_desc.xrc_remote_srq_num = endpoint->rem_info.rem_srqs[qp].rem_srq_num;
+#endif
+
     }
 #endif
 

opal/mca/btl/openib/btl_openib_put.c

@@ -103,10 +103,13 @@ int mca_btl_openib_put (mca_btl_base_module_t *btl, struct mca_btl_base_endpoint
 
 #if HAVE_XRC
     if (MCA_BTL_XRC_ENABLED && BTL_OPENIB_QP_TYPE_XRC(qp)) {
-#if OPAL_HAVE_CONNECTX_XRC_DOMAINS
+
+#if OPAL_HAVE_CONNECTX_XRC
+        to_out_frag(frag)->sr_desc.xrc_remote_srq_num = ep->rem_info.rem_srqs[qp].rem_srq_num;
+#elif OPAL_HAVE_CONNECTX_XRC_DOMAINS
         to_out_frag(frag)->sr_desc.qp_type.xrc.remote_srqn = ep->rem_info.rem_srqs[qp].rem_srq_num;
 #else
-        to_out_frag(frag)->sr_desc.xrc_remote_srq_num = ep->rem_info.rem_srqs[qp].rem_srq_num;
+#error "that should never happen"
 #endif
     }
 #endif