Merge pull request open-mpi#1030 from alinask/topic/release_openib_diff_subnets

btl/openib: enable connecting processes from different subnets.

Author: rhc54

config/ompi_check_openfabrics.m4

@@ -348,6 +348,10 @@ AC_DEFUN([OMPI_CHECK_OPENFABRICS_CM_ARGS],[
     AC_ARG_ENABLE([openib-rdmacm],
         [AC_HELP_STRING([--enable-openib-rdmacm],
                         [Enable Open Fabrics RDMACM support in openib BTL (default: enabled)])])
+    AC_ARG_ENABLE([openib-rdmacm-ibaddr],
+        [AC_HELP_STRING([--enable-openib-rdmacm-ibaddr],
+                        [Enable Open Fabrics RDMACM with IB addressing support in openib BTL (default: disabled)])],
+        [enable_openib_rdmacm=yes])
 ])dnl
 
 AC_DEFUN([OMPI_CHECK_OPENFABRICS_CM],[

ompi/mca/btl/openib/btl_openib.c

@@ -854,6 +854,13 @@ int mca_btl_openib_add_procs(
                 btl_rank = lcl_subnet_id_port_cnt;
             }
             lcl_subnet_id_port_cnt++;
+        } else {
+            if (mca_btl_openib_component.allow_different_subnets) {
+                if (openib_btl == mca_btl_openib_component.openib_btls[j]) {
+                    btl_rank = lcl_subnet_id_port_cnt;
+                }
+                lcl_subnet_id_port_cnt++;
+            }
         }
     }
 
@@ -924,6 +931,14 @@ int mca_btl_openib_add_procs(
                     remote_matching_port = j;
                 }
                 rem_subnet_id_port_cnt++;
+            } else {
+                if (mca_btl_openib_component.allow_different_subnets) {
+                    BTL_VERBOSE(("Using different subnets!"));
+                    if (rem_subnet_id_port_cnt == btl_rank) {
+                        remote_matching_port = j;
+                    }
+                    rem_subnet_id_port_cnt++;
+                }
             }
         }
 
@@ -993,7 +1008,14 @@ int mca_btl_openib_add_procs(
                         break;
                     else
                         rem_port_cnt ++;
-                }
+                    } else {
+                        if (mca_btl_openib_component.allow_different_subnets) {
+                            if (rem_port_cnt == btl_rank)
+                                break;
+                            else
+                                rem_port_cnt++;
+                        }
+                    }
             }
 
             assert(rem_port_cnt == btl_rank);

ompi/mca/btl/openib/btl_openib.h

@@ -301,6 +301,9 @@ struct mca_btl_openib_component_t {
     char* default_recv_qps;
     /** GID index to use */
     int gid_index;
+    /*  Whether we want to allow connecting processes from different subnets.
+    *  set to 'no' by default */
+    bool allow_different_subnets;
     /** Whether we want a dynamically resizing srq, enabled by default */
     bool enable_srq_resize;
     bool allow_max_memory_registration;

ompi/mca/btl/openib/btl_openib_mca.c

@@ -703,6 +703,11 @@ int btl_openib_register_mca_params(void)
                   0, &mca_btl_openib_component.gid_index,
                   REGINT_GE_ZERO));
 
+    CHECK(reg_bool("allow_different_subnets", NULL,
+                   "Allow connecting processes from different IB subnets."
+                   "(0 = do not allow; 1 = allow)",
+                   false, &mca_btl_openib_component.allow_different_subnets));
+
 #if MEMORY_LINUX_MALLOC_ALIGN_ENABLED
     tmp = mca_base_var_find ("opal", "memory", "linux", "memalign");
     if (0 <= tmp) {

ompi/mca/btl/openib/connect/btl_openib_connect_rdmacm.c

@@ -2111,7 +2111,8 @@ static int rdmacm_component_query(mca_btl_openib_module_t *openib_btl, ompi_btl_
     sin.sin_addr.s_addr = rdmacm_addr;
     sin.sin_port = (uint16_t) rdmacm_port;
 #else
-    rc = ibv_query_gid(openib_btl->device->ib_pd->context, openib_btl->port_num, 0, &server->gid);
+    rc = ibv_query_gid(openib_btl->device->ib_pd->context, openib_btl->port_num,
+                       mca_btl_openib_component.gid_index, &server->gid);
     if (0 != rc) {
         BTL_ERROR(("local gid query failed"));
         goto out4;