configury: fix (again) XRC detection on OFED < 3.12

since ibv_create_xrc_rcv_qp is now deprecated, and in order to
be "future-proof", we have to consider the case in which only XRC Domains are supported.

Thanks Paul Hargrove for the detailled report.

Author: ggouaillardet

config/opal_check_openfabrics.m4

@@ -154,13 +154,17 @@ AC_DEFUN([OPAL_CHECK_OPENFABRICS],[
                $1_have_xrc=1
                AC_CHECK_FUNCS([ibv_create_xrc_rcv_qp],
                               [], [$1_have_xrc=0])
-               AC_CHECK_DECLS([IBV_SRQT_XRC],
-                              [], [$1_have_xrc=0],
-                              [#include <infiniband/verbs.h>])
-           fi
-           if test "$enable_connectx_xrc" = "yes" \
-               && test $$1_have_xrc -eq 1; then
-               AC_CHECK_FUNCS([ibv_cmd_open_xrcd], [$1_have_xrc_domains=1])
+               $1_have_xrc_domains=1
+               AC_CHECK_FUNCS([ibv_cmd_open_xrcd],
+                              [AC_CHECK_DECLS([IBV_SRQT_XRC],
+                                              [], [$1_have_xrc_domains=0],
+                                              [#include <infiniband/verbs.h>])],
+                              [$1_have_xrc_domains=0])
+               # XRC and XRC Domains should be considered as exclusive
+               if test "$$1_have_xrc" -eq 1 && \
+                  test "$$1_have_xrc_domains" -eq 1; then
+                      $1_have_xrc=0
+               fi
            fi
 
 

opal/mca/btl/openib/btl_openib.h

@@ -53,8 +53,8 @@
 
 BEGIN_C_DECLS
 
-#define HAVE_XRC (1 == OPAL_HAVE_CONNECTX_XRC)
-#define ENABLE_DYNAMIC_SL (1 == OPAL_ENABLE_DYNAMIC_SL)
+#define HAVE_XRC (OPAL_HAVE_CONNECTX_XRC || OPAL_HAVE_CONNECTX_XRC_DOMAINS)
+#define ENABLE_DYNAMIC_SL OPAL_ENABLE_DYNAMIC_SL
 
 #define MCA_BTL_IB_LEAVE_PINNED 1
 #define IB_DEFAULT_GID_PREFIX 0xfe80000000000000ll

opal/mca/btl/openib/btl_openib_async.c

@@ -122,7 +122,7 @@ static mca_btl_openib_endpoint_t * qp2endpoint(struct ibv_qp *qp, mca_btl_openib
     return NULL;
 }
 
-#if HAVE_XRC && !OPAL_HAVE_CONNECTX_XRC_DOMAINS
+#if OPAL_HAVE_CONNECTX_XRC
 /* XRC recive QP to endpoint */
 static mca_btl_openib_endpoint_t * xrc_qp2endpoint(uint32_t qp_num, mca_btl_openib_device_t *device)
 {
@@ -352,24 +352,21 @@ static int btl_openib_async_deviceh(struct mca_btl_openib_async_poll *devices_po
         }
 
         event_type = event.event_type;
-#if HAVE_XRC
+#if OPAL_HAVE_CONNECTX_XRC
         /* is it XRC event ?*/
-#if OPAL_HAVE_CONNECTX_XRC_DOMAINS
-#else
         bool xrc_event = false;
         if (IBV_XRC_QP_EVENT_FLAG & event.event_type) {
             xrc_event = true;
             /* Clean the bitnd handel as usual */
             event_type ^= IBV_XRC_QP_EVENT_FLAG;
         }
-#endif
 #endif
         switch(event_type) {
             case IBV_EVENT_PATH_MIG:
                 BTL_ERROR(("Alternative path migration event reported"));
                 if (APM_ENABLED) {
                     BTL_ERROR(("Trying to find additional path..."));
-#if HAVE_XRC && !OPAL_HAVE_CONNECTX_XRC_DOMAINS
+#if OPAL_HAVE_CONNECTX_XRC
                     if (xrc_event)
                         mca_btl_openib_load_apm_xrc_rcv(event.element.xrc_qp_num,
                                 xrc_qp2endpoint(event.element.xrc_qp_num, device));
@@ -653,7 +650,7 @@ void mca_btl_openib_load_apm(struct ibv_qp *qp, mca_btl_openib_endpoint_t *ep)
                    qp->qp_num, strerror(errno), errno));
 }
 
-#if HAVE_XRC && ! OPAL_HAVE_CONNECTX_XRC_DOMAINS
+#if OPAL_HAVE_CONNECTX_XRC
 void mca_btl_openib_load_apm_xrc_rcv(uint32_t qp_num, mca_btl_openib_endpoint_t *ep)
 {
     struct ibv_qp_init_attr qp_init_attr;

opal/mca/btl/openib/btl_openib_async.h

@@ -19,7 +19,7 @@
 int        start_async_event_thread(void);
 void       mca_btl_openib_load_apm(struct ibv_qp *qp, mca_btl_openib_endpoint_t *ep);
 int        btl_openib_async_command_done(int exp);
-#if HAVE_XRC && ! OPAL_HAVE_CONNECTX_XRC_DOMAINS
+#if OPAL_HAVE_CONNECTX_XRC
 void       mca_btl_openib_load_apm_xrc_rcv(uint32_t qp_num, mca_btl_openib_endpoint_t *ep);
 #endif
 

opal/mca/btl/openib/btl_openib_atomic.c

@@ -73,7 +73,12 @@ static int mca_btl_openib_atomic_internal (struct mca_btl_base_module_t *btl, st
 
 #if HAVE_XRC
     if (MCA_BTL_XRC_ENABLED && BTL_OPENIB_QP_TYPE_XRC(qp)) {
-        frag->sr_desc.xrc_remote_srq_num=endpoint->rem_info.rem_srqs[qp].rem_srq_num;
+#if OPAL_HAVE_CONNECTX_XRC_DOMAINS
+        frag->sr_desc.qp_type.xrc.remote_srqn = endpoint->rem_info.rem_srqs[qp].rem_srq_num;
+#else
+        frag->sr_desc.xrc_remote_srq_num = endpoint->rem_info.rem_srqs[qp].rem_srq_num;
+#endif
+
     }
 #endif
 

opal/mca/btl/openib/btl_openib_put.c

@@ -103,10 +103,13 @@ int mca_btl_openib_put (mca_btl_base_module_t *btl, struct mca_btl_base_endpoint
 
 #if HAVE_XRC
     if (MCA_BTL_XRC_ENABLED && BTL_OPENIB_QP_TYPE_XRC(qp)) {
-#if OPAL_HAVE_CONNECTX_XRC_DOMAINS
+
+#if OPAL_HAVE_CONNECTX_XRC
+        to_out_frag(frag)->sr_desc.xrc_remote_srq_num = ep->rem_info.rem_srqs[qp].rem_srq_num;
+#elif OPAL_HAVE_CONNECTX_XRC_DOMAINS
         to_out_frag(frag)->sr_desc.qp_type.xrc.remote_srqn = ep->rem_info.rem_srqs[qp].rem_srq_num;
 #else
-        to_out_frag(frag)->sr_desc.xrc_remote_srq_num = ep->rem_info.rem_srqs[qp].rem_srq_num;
+#error "that should never happen"
 #endif
     }
 #endif