Fix for SEP when num local procs is greater than available contexts

For cases when the number of local processes is greater than the number of
available contexts, the SEP initialization phase would calculate the number of
contexts to provision for each rank to be 0 and would eventually crash.

Fix the issue here by using regular endpoints in the event the number of local
processes is more than available contexts. This fixes issue #6182.

Signed-off-by: Aravind Gopalakrishnan <[email protected]>

Author: aravindksg

ompi/mca/mtl/ofi/mtl_ofi.h

@@ -1391,7 +1391,7 @@ ompi_mtl_ofi_add_comm(struct mca_mtl_base_module_t *mtl,
                       struct ompi_communicator_t *comm)
 {
     int ret;
-    mca_mtl_ofi_ep_type ep_type = (false == ompi_mtl_ofi.sep_supported) ?
+    mca_mtl_ofi_ep_type ep_type = (0 == ompi_mtl_ofi.enable_sep) ?
                                   OFI_REGULAR_EP : OFI_SCALABLE_EP;
 
     /*
@@ -1422,7 +1422,7 @@ ompi_mtl_ofi_del_comm(struct mca_mtl_base_module_t *mtl,
                       struct ompi_communicator_t *comm)
 {
     int ret = OMPI_SUCCESS;
-    mca_mtl_ofi_ep_type ep_type = (false == ompi_mtl_ofi.sep_supported) ?
+    mca_mtl_ofi_ep_type ep_type = (0 == ompi_mtl_ofi.enable_sep) ?
                                   OFI_REGULAR_EP : OFI_SCALABLE_EP;
 
     /*

ompi/mca/mtl/ofi/mtl_ofi_component.c

@@ -473,13 +473,6 @@ static int ompi_mtl_ofi_init_sep(struct fi_info *prov)
     int ret = OMPI_SUCCESS, num_ofi_ctxts;
     struct fi_av_attr av_attr = {0};
 
-    ompi_mtl_ofi.max_ctx_cnt = (prov->domain_attr->max_ep_tx_ctx <
-                                prov->domain_attr->max_ep_rx_ctx) ?
-                                prov->domain_attr->max_ep_tx_ctx :
-                                prov->domain_attr->max_ep_rx_ctx;
-
-    /* Provision enough contexts to service all ranks in a node */
-    ompi_mtl_ofi.max_ctx_cnt /= (1 + ompi_process_info.num_local_peers);
     prov->ep_attr->tx_ctx_cnt = prov->ep_attr->rx_ctx_cnt =
                                 ompi_mtl_ofi.max_ctx_cnt;
 
@@ -601,15 +594,14 @@ static mca_mtl_base_module_t*
 ompi_mtl_ofi_component_init(bool enable_progress_threads,
                             bool enable_mpi_threads)
 {
-    int ret, fi_version;
+    int ret, fi_version, num_local_ranks;
+    int ofi_tag_leading_zeros, ofi_tag_bits_for_cid;
     struct fi_info *hints;
     struct fi_info *providers = NULL;
     struct fi_info *prov = NULL;
     struct fi_info *prov_cq_data = NULL;
     char ep_name[FI_NAME_MAX] = {0};
     size_t namelen;
-    int ofi_tag_leading_zeros;
-    int ofi_tag_bits_for_cid;
 
     /**
      * Hints to filter providers
@@ -774,10 +766,10 @@ ompi_mtl_ofi_component_init(bool enable_progress_threads,
     ompi_mtl_ofi.num_peers = 0;
 
     /* Check if Scalable Endpoints can be enabled for the provider */
-    ompi_mtl_ofi.sep_supported = false;
+    ompi_mtl_ofi.enable_sep = 0;
     if ((prov->domain_attr->max_ep_tx_ctx > 1) ||
         (prov->domain_attr->max_ep_rx_ctx > 1)) {
-        ompi_mtl_ofi.sep_supported = true;
+        ompi_mtl_ofi.enable_sep = 1;
         opal_output_verbose(1, ompi_mtl_base_framework.framework_output,
                             "%s:%d: Scalable EP supported in %s provider. Enabling in MTL.\n",
                             __FILE__, __LINE__, prov->fabric_attr->prov_name);
@@ -786,7 +778,7 @@ ompi_mtl_ofi_component_init(bool enable_progress_threads,
     /*
      * Scalable Endpoints is required for Thread Grouping feature
      */
-    if (!ompi_mtl_ofi.sep_supported && ompi_mtl_ofi.thread_grouping) {
+    if (!ompi_mtl_ofi.enable_sep && ompi_mtl_ofi.thread_grouping) {
         opal_show_help("help-mtl-ofi.txt", "SEP unavailable", true,
                        prov->fabric_attr->prov_name,
                        ompi_process_info.nodename, __FILE__, __LINE__,
@@ -848,7 +840,20 @@ ompi_mtl_ofi_component_init(bool enable_progress_threads,
      * vectors, completion counters or event queues etc, and enabled.
      * See man fi_endpoint for more details.
      */
-    if (true == ompi_mtl_ofi.sep_supported) {
+    ompi_mtl_ofi.max_ctx_cnt = (prov->domain_attr->max_ep_tx_ctx <
+                                prov->domain_attr->max_ep_rx_ctx) ?
+                                prov->domain_attr->max_ep_tx_ctx :
+                                prov->domain_attr->max_ep_rx_ctx;
+
+    num_local_ranks = 1 + ompi_process_info.num_local_peers;
+    if (ompi_mtl_ofi.max_ctx_cnt <= num_local_ranks) {
+        ompi_mtl_ofi.enable_sep = 0;
+    }
+
+    if (1 == ompi_mtl_ofi.enable_sep) {
+        /* Provision enough contexts to service all ranks in a node */
+        ompi_mtl_ofi.max_ctx_cnt /= num_local_ranks;
+
         ret = ompi_mtl_ofi_init_sep(prov);
     } else {
         ret = ompi_mtl_ofi_init_regular_ep(prov);
@@ -926,7 +931,7 @@ ompi_mtl_ofi_component_init(bool enable_progress_threads,
     if (ompi_mtl_ofi.av) {
         (void) fi_close((fid_t)ompi_mtl_ofi.av);
     }
-    if ((false == ompi_mtl_ofi.sep_supported) &&
+    if ((0 == ompi_mtl_ofi.enable_sep) &&
         ompi_mtl_ofi.ofi_ctxt != NULL &&
          ompi_mtl_ofi.ofi_ctxt[0].cq) {
         /* Check if CQ[0] was created for non-SEP case and close if needed */
@@ -964,9 +969,9 @@ ompi_mtl_ofi_finalize(struct mca_mtl_base_module_t *mtl)
         goto finalize_err;
     }
 
-    if (false == ompi_mtl_ofi.sep_supported) {
+    if (0 == ompi_mtl_ofi.enable_sep) {
         /*
-         * CQ[0] is bound to SEP object when SEP is not supported by a
+         * CQ[0] is bound to SEP object Nwhen SEP is not supported by a
          * provider. OFI spec requires that we close the Endpoint that is bound
          * to the CQ before closing the CQ itself. So, for the non-SEP case, we
          * handle the closing of CQ[0] here.

ompi/mca/mtl/ofi/mtl_ofi_types.h

@@ -71,8 +71,8 @@ typedef struct mca_mtl_ofi_module_t {
     /* MCA parameter for Thread grouping feature */
     int thread_grouping;
 
-    /* Boolen value to indicate if provider supports Scalable EP or not */
-    bool sep_supported;
+    /* If SEP is used by OFI MTL */
+    int enable_sep;
 
     /* Numbers of bits used for rx contexts */
     int rx_ctx_bits;