Revert "configury: fix (again) XRC detection on OFED < 3.12"

@ggouaillardet is likely offline for the weekend, but master is broken
on RHEL 6.5 systems that do not have MOFED installed.  So I'm taking
the liberty of revering this commit; I'm guessing Gilles will fixup
and re-commit next week.

This reverts commit 77f8282.

Author: jsquyres

config/opal_check_openfabrics.m4

@@ -154,17 +154,13 @@ AC_DEFUN([OPAL_CHECK_OPENFABRICS],[
                $1_have_xrc=1
                AC_CHECK_FUNCS([ibv_create_xrc_rcv_qp],
                               [], [$1_have_xrc=0])
-               $1_have_xrc_domains=1
-               AC_CHECK_FUNCS([ibv_cmd_open_xrcd],
-                              [AC_CHECK_DECLS([IBV_SRQT_XRC],
-                                              [], [$1_have_xrc_domains=0],
-                                              [#include <infiniband/verbs.h>])],
-                              [$1_have_xrc_domains=0])
-               # XRC and XRC Domains should be considered as exclusive
-               if test "$$1_have_xrc" -eq 1 && \
-                  test "$$1_have_xrc_domains" -eq 1; then
-                      $1_have_xrc=0
-               fi
+               AC_CHECK_DECLS([IBV_SRQT_XRC],
+                              [], [$1_have_xrc=0],
+                              [#include <infiniband/verbs.h>])
+           fi
+           if test "$enable_connectx_xrc" = "yes" \
+               && test $$1_have_xrc -eq 1; then
+               AC_CHECK_FUNCS([ibv_cmd_open_xrcd], [$1_have_xrc_domains=1])
            fi
 
 

opal/mca/btl/openib/btl_openib.h

@@ -53,8 +53,8 @@
 
 BEGIN_C_DECLS
 
-#define HAVE_XRC (OPAL_HAVE_CONNECTX_XRC || OPAL_HAVE_CONNECTX_XRC_DOMAINS)
-#define ENABLE_DYNAMIC_SL OPAL_ENABLE_DYNAMIC_SL
+#define HAVE_XRC (1 == OPAL_HAVE_CONNECTX_XRC)
+#define ENABLE_DYNAMIC_SL (1 == OPAL_ENABLE_DYNAMIC_SL)
 
 #define MCA_BTL_IB_LEAVE_PINNED 1
 #define IB_DEFAULT_GID_PREFIX 0xfe80000000000000ll

opal/mca/btl/openib/btl_openib_async.c

@@ -122,7 +122,7 @@ static mca_btl_openib_endpoint_t * qp2endpoint(struct ibv_qp *qp, mca_btl_openib
     return NULL;
 }
 
-#if OPAL_HAVE_CONNECTX_XRC
+#if HAVE_XRC && !OPAL_HAVE_CONNECTX_XRC_DOMAINS
 /* XRC recive QP to endpoint */
 static mca_btl_openib_endpoint_t * xrc_qp2endpoint(uint32_t qp_num, mca_btl_openib_device_t *device)
 {
@@ -352,21 +352,24 @@ static int btl_openib_async_deviceh(struct mca_btl_openib_async_poll *devices_po
         }
 
         event_type = event.event_type;
-#if OPAL_HAVE_CONNECTX_XRC
+#if HAVE_XRC
         /* is it XRC event ?*/
+#if OPAL_HAVE_CONNECTX_XRC_DOMAINS
+#else
         bool xrc_event = false;
         if (IBV_XRC_QP_EVENT_FLAG & event.event_type) {
             xrc_event = true;
             /* Clean the bitnd handel as usual */
             event_type ^= IBV_XRC_QP_EVENT_FLAG;
         }
+#endif
 #endif
         switch(event_type) {
             case IBV_EVENT_PATH_MIG:
                 BTL_ERROR(("Alternative path migration event reported"));
                 if (APM_ENABLED) {
                     BTL_ERROR(("Trying to find additional path..."));
-#if OPAL_HAVE_CONNECTX_XRC
+#if HAVE_XRC && !OPAL_HAVE_CONNECTX_XRC_DOMAINS
                     if (xrc_event)
                         mca_btl_openib_load_apm_xrc_rcv(event.element.xrc_qp_num,
                                 xrc_qp2endpoint(event.element.xrc_qp_num, device));
@@ -650,7 +653,7 @@ void mca_btl_openib_load_apm(struct ibv_qp *qp, mca_btl_openib_endpoint_t *ep)
                    qp->qp_num, strerror(errno), errno));
 }
 
-#if OPAL_HAVE_CONNECTX_XRC
+#if HAVE_XRC && ! OPAL_HAVE_CONNECTX_XRC_DOMAINS
 void mca_btl_openib_load_apm_xrc_rcv(uint32_t qp_num, mca_btl_openib_endpoint_t *ep)
 {
     struct ibv_qp_init_attr qp_init_attr;

opal/mca/btl/openib/btl_openib_async.h

@@ -19,7 +19,7 @@
 int        start_async_event_thread(void);
 void       mca_btl_openib_load_apm(struct ibv_qp *qp, mca_btl_openib_endpoint_t *ep);
 int        btl_openib_async_command_done(int exp);
-#if OPAL_HAVE_CONNECTX_XRC
+#if HAVE_XRC && ! OPAL_HAVE_CONNECTX_XRC_DOMAINS
 void       mca_btl_openib_load_apm_xrc_rcv(uint32_t qp_num, mca_btl_openib_endpoint_t *ep);
 #endif
 

opal/mca/btl/openib/btl_openib_atomic.c

@@ -73,12 +73,7 @@ static int mca_btl_openib_atomic_internal (struct mca_btl_base_module_t *btl, st
 
 #if HAVE_XRC
     if (MCA_BTL_XRC_ENABLED && BTL_OPENIB_QP_TYPE_XRC(qp)) {
-#if OPAL_HAVE_CONNECTX_XRC_DOMAINS
-        frag->sr_desc.qp_type.xrc.remote_srqn = endpoint->rem_info.rem_srqs[qp].rem_srq_num;
-#else
-        frag->sr_desc.xrc_remote_srq_num = endpoint->rem_info.rem_srqs[qp].rem_srq_num;
-#endif
-
+        frag->sr_desc.xrc_remote_srq_num=endpoint->rem_info.rem_srqs[qp].rem_srq_num;
     }
 #endif
 

opal/mca/btl/openib/btl_openib_put.c

@@ -103,13 +103,10 @@ int mca_btl_openib_put (mca_btl_base_module_t *btl, struct mca_btl_base_endpoint
 
 #if HAVE_XRC
     if (MCA_BTL_XRC_ENABLED && BTL_OPENIB_QP_TYPE_XRC(qp)) {
-
-#if OPAL_HAVE_CONNECTX_XRC
-        to_out_frag(frag)->sr_desc.xrc_remote_srq_num = ep->rem_info.rem_srqs[qp].rem_srq_num;
-#elif OPAL_HAVE_CONNECTX_XRC_DOMAINS
+#if OPAL_HAVE_CONNECTX_XRC_DOMAINS
         to_out_frag(frag)->sr_desc.qp_type.xrc.remote_srqn = ep->rem_info.rem_srqs[qp].rem_srq_num;
 #else
-#error "that should never happen"
+        to_out_frag(frag)->sr_desc.xrc_remote_srq_num = ep->rem_info.rem_srqs[qp].rem_srq_num;
 #endif
     }
 #endif