Merge pull request #6748 from gpaulsen/topic/v4.0.x/fix_ucx_1.6_issue6640

btl/uct: add support for UCX 1.6.x

Author: gpaulsen

opal/mca/btl/uct/btl_uct_endpoint.c

@@ -4,6 +4,7 @@
  *                         reserved.
  * Copyright (c) 2018      Triad National Security, LLC. All rights
  *                         reserved.
+ * Copyright (c) 2019      Google, LLC. All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -101,6 +102,28 @@ static void mca_btl_uct_process_modex (mca_btl_uct_module_t *uct_btl, unsigned c
     }
 }
 
+static inline ucs_status_t mca_btl_uct_ep_create_connected_compat (uct_iface_h iface, uct_device_addr_t *device_addr,
+                                                                   uct_iface_addr_t *iface_addr, uct_ep_h *uct_ep)
+{
+#if UCT_API >= UCT_VERSION(1, 6)
+    uct_ep_params_t ep_params = {.field_mask = UCT_EP_PARAM_FIELD_IFACE | UCT_EP_PARAM_FIELD_DEV_ADDR | UCT_EP_PARAM_FIELD_IFACE_ADDR,
+                                 .iface = iface, .dev_addr = device_addr, .iface_addr = iface_addr};
+    return uct_ep_create (&ep_params, uct_ep);
+#else
+    return uct_ep_create_connected (iface, device_addr, iface_addr, uct_ep);
+#endif
+}
+
+static inline ucs_status_t mca_btl_uct_ep_create_compat (uct_iface_h iface, uct_ep_h *uct_ep)
+{
+#if UCT_API >= UCT_VERSION(1, 6)
+    uct_ep_params_t ep_params = {.field_mask = UCT_EP_PARAM_FIELD_IFACE, .iface = iface};
+    return uct_ep_create (&ep_params, uct_ep);
+#else
+    return uct_ep_create (iface, uct_ep);
+#endif
+}
+
 static int mca_btl_uct_endpoint_connect_iface (mca_btl_uct_module_t *uct_btl, mca_btl_uct_tl_t *tl,
                                                mca_btl_uct_device_context_t *tl_context,
                                                mca_btl_uct_tl_endpoint_t *tl_endpoint, uint8_t *tl_data)
@@ -116,7 +139,7 @@ static int mca_btl_uct_endpoint_connect_iface (mca_btl_uct_module_t *uct_btl, mc
     BTL_VERBOSE(("connecting endpoint to interface"));
 
     mca_btl_uct_context_lock (tl_context);
-    ucs_status = uct_ep_create_connected (tl_context->uct_iface, device_addr, iface_addr, &tl_endpoint->uct_ep);
+    ucs_status = mca_btl_uct_ep_create_connected_compat (tl_context->uct_iface, device_addr, iface_addr, &tl_endpoint->uct_ep);
     tl_endpoint->flags = MCA_BTL_UCT_ENDPOINT_FLAG_CONN_READY;
     mca_btl_uct_context_unlock (tl_context);
 
@@ -240,8 +263,8 @@ static int mca_btl_uct_endpoint_connect_endpoint (mca_btl_uct_module_t *uct_btl,
 
         /* create a temporary endpoint for setting up the rdma endpoint */
         MCA_BTL_UCT_CONTEXT_SERIALIZE(conn_tl_context, {
-                ucs_status = uct_ep_create_connected  (conn_tl_context->uct_iface, device_addr, iface_addr,
-                                                       &conn_ep->uct_ep);
+                ucs_status = mca_btl_uct_ep_create_connected_compat (conn_tl_context->uct_iface, device_addr, iface_addr,
+                                                                     &conn_ep->uct_ep);
             });
         if (UCS_OK != ucs_status) {
             BTL_VERBOSE(("could not create an endpoint for forming connection to remote peer. code = %d",
@@ -263,7 +286,7 @@ static int mca_btl_uct_endpoint_connect_endpoint (mca_btl_uct_module_t *uct_btl,
                      opal_process_name_print (endpoint->ep_proc->proc_name)));
 
         MCA_BTL_UCT_CONTEXT_SERIALIZE(tl_context, {
-                ucs_status = uct_ep_create (tl_context->uct_iface, &tl_endpoint->uct_ep);
+                ucs_status = mca_btl_uct_ep_create_compat (tl_context->uct_iface, &tl_endpoint->uct_ep);
             });
         if (UCS_OK != ucs_status) {
             OBJ_RELEASE(endpoint->conn_ep);
@@ -309,7 +332,8 @@ int mca_btl_uct_endpoint_connect (mca_btl_uct_module_t *uct_btl, mca_btl_uct_end
                                   void *ep_addr, int tl_index)
 {
     mca_btl_uct_tl_endpoint_t *tl_endpoint = endpoint->uct_eps[context_id] + tl_index;
-    mca_btl_uct_tl_t *tl = (tl_index == uct_btl->rdma_tl->tl_index) ? uct_btl->rdma_tl : uct_btl->am_tl;
+    mca_btl_uct_tl_t *tl = (uct_btl->rdma_tl && tl_index == uct_btl->rdma_tl->tl_index) ?
+        uct_btl->rdma_tl : uct_btl->am_tl;
     mca_btl_uct_device_context_t *tl_context = mca_btl_uct_module_get_tl_context_specific (uct_btl, tl, context_id);
     uint8_t *rdma_tl_data = NULL, *conn_tl_data = NULL, *am_tl_data = NULL, *tl_data;
     mca_btl_uct_connection_ep_t *conn_ep = NULL;

opal/mca/btl/uct/btl_uct_tl.c

@@ -6,6 +6,7 @@
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2018      Triad National Security, LLC. All rights
  *                         reserved.
+ * Copyright (c) 2019      Google, LLC. All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -270,10 +271,18 @@ static void mca_btl_uct_context_enable_progress (mca_btl_uct_device_context_t *c
 
 mca_btl_uct_device_context_t *mca_btl_uct_context_create (mca_btl_uct_module_t *module, mca_btl_uct_tl_t *tl, int context_id, bool enable_progress)
 {
+#if UCT_API >= UCT_VERSION(1, 6)
+    uct_iface_params_t iface_params = {.field_mask = UCT_IFACE_PARAM_FIELD_OPEN_MODE |
+                                                     UCT_IFACE_PARAM_FIELD_DEVICE,
+                                       .open_mode = UCT_IFACE_OPEN_MODE_DEVICE,
+                                       .mode = {.device = {.tl_name = tl->uct_tl_name,
+                                                           .dev_name = tl->uct_dev_name}}};
+#else
     uct_iface_params_t iface_params = {.rndv_cb = NULL, .eager_cb = NULL, .stats_root = NULL,
                                        .rx_headroom = 0, .open_mode = UCT_IFACE_OPEN_MODE_DEVICE,
                                        .mode = {.device = {.tl_name = tl->uct_tl_name,
                                                            .dev_name = tl->uct_dev_name}}};
+#endif
     mca_btl_uct_device_context_t *context;
     ucs_status_t ucs_status;
     int rc;
@@ -610,6 +619,7 @@ int mca_btl_uct_query_tls (mca_btl_uct_module_t *module, mca_btl_uct_md_t *md, u
 	/* no rdma tls */
 	BTL_VERBOSE(("no rdma tl matched supplied filter. disabling RDMA support"));
 
+        module->super.btl_flags &= ~MCA_BTL_FLAGS_RDMA;
 	module->super.btl_put = NULL;
 	module->super.btl_get = NULL;
 	module->super.btl_atomic_fop = NULL;