ompi_proc_t size reduction: part 1

We currently save the hostname of a proc when we create the ompi_proc_t for it. This was originally done because the only method we had for discovering the host of a proc was to include that info in the modex, and we had to therefore store it somewhere proc-local. Obviously, this ccarried a memory penalty for storing all those strings, and so we added a "cutoff" parameter so that we wouldn't collect hostnames above a certain number of procs.

Unfortunately, this still results in an 8-byte/proc memory cost as we have a char* pointer in the opal_proc_t that is contained in the ompi_proc_t so that we can store the hostname of the other procs if we fall below the cutoff. At scale, this can consume a fair amount of memory.

With the switch to relying on PMIx, there is no longer a need to cache the proc hostnames. Using the "optional" feature of PMIx_Get, we restrict the retrieval to be purely proc-local - i.e., we retrieve the info either via shared memory or from within the proc-internal hash storage (depending upon the active PMIx components). Thus, the retrieval of a hostname is purely a local operation involving no communication.

All RM's are required to provide a complete hostname map of all procs at startup. Thus, we have full access to all hostnames without including them in a modex or having to cache them on each proc. This allows us to remove the char* pointer from the opal_proc_t, saving us 8-bytes/proc.

Unfortunately, PMIx_Get does not currently support the return of a static pointer to memory. Thus, even though PMIx has the hostname in its memory, it can only return a malloc'd version of it. I have therefore ensured that the return from opal_get_proc_hostname is consistently malloc'd and free'd wherever used. This shouldn't be a burden as the hostname is only used in one of two circumstances:

(a) in an error message
(b) in a verbose output for debugging purposes

Thus, there should be no performance penalty associated with the malloc/free requirement. PMIx will eventually be returning static pointers, and so we can eventually simplify this method and return a "const char*" - but as noted, this really isn't an issue even today.

Signed-off-by: Ralph Castain <[email protected]>

Author: rhc54

ompi/mca/bml/r2/bml_r2.c

@@ -13,7 +13,7 @@
  * Copyright (c) 2007-2015 Los Alamos National Security, LLC.  All rights
  *                         reserved.
  * Copyright (c) 2008-2016 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2013      Intel, Inc. All rights reserved
+ * Copyright (c) 2013-2020 Intel, Inc.  All rights reserved.
  * Copyright (c) 2014      NVIDIA Corporation.  All rights reserved.
  * Copyright (c) 2014      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
@@ -233,12 +233,14 @@ static int mca_bml_r2_endpoint_add_btl (struct ompi_proc_t *proc, mca_bml_base_e
 
         if (!bml_btl || bml_btl->btl->btl_exclusivity <= btl->btl_exclusivity) {
             /* this btl has higher exclusivity than an existing btl or none exists */
-
-            opal_output_verbose(1, opal_btl_base_framework.framework_output,
-                                "mca: bml: Using %s btl for send to %s on node %s",
-                                btl->btl_component->btl_version.mca_component_name,
-                                OMPI_NAME_PRINT(&proc->super.proc_name),
-                                proc->super.proc_hostname);
+            if (0 < opal_output_get_verbosity(opal_btl_base_framework.framework_output)) {
+                char *errhost = opal_get_proc_hostname(&proc->super);
+                opal_output(0, "mca: bml: Using %s btl for send to %s on node %s",
+                            btl->btl_component->btl_version.mca_component_name,
+                            OMPI_NAME_PRINT(&proc->super.proc_name),
+                            errhost);
+                free(errhost);
+            }
 
             /* cache the endpoint on the proc */
             if (NULL == bml_btl || (bml_btl->btl->btl_exclusivity <= btl->btl_exclusivity)) {
@@ -252,15 +254,16 @@ static int mca_bml_r2_endpoint_add_btl (struct ompi_proc_t *proc, mca_bml_base_e
                  * calculate the bitwise OR of the btl flags
                  */
                 bml_endpoint->btl_flags_or |= bml_btl->btl_flags;
-            } else {
-                opal_output_verbose(20, opal_btl_base_framework.framework_output,
-                                    "mca: bml: Not using %s btl for send to %s on node %s "
-                                    "because %s btl has higher exclusivity (%d > %d)",
-                                    btl->btl_component->btl_version.mca_component_name,
-                                    OMPI_NAME_PRINT(&proc->super.proc_name), proc->super.proc_hostname,
-                                    bml_btl->btl->btl_component->btl_version.mca_component_name,
-                                    bml_btl->btl->btl_exclusivity,
-                                    btl->btl_exclusivity);
+            } else if (19 < opal_output_get_verbosity(opal_btl_base_framework.framework_output)) {
+                char *errhost = opal_get_proc_hostname(&proc->super);
+                opal_output(0, "mca: bml: Not using %s btl for send to %s on node %s "
+                            "because %s btl has higher exclusivity (%d > %d)",
+                            btl->btl_component->btl_version.mca_component_name,
+                            OMPI_NAME_PRINT(&proc->super.proc_name), errhost,
+                            bml_btl->btl->btl_component->btl_version.mca_component_name,
+                            bml_btl->btl->btl_exclusivity,
+                            btl->btl_exclusivity);
+                free(errhost);
             }
 
             btl_in_use = true;
@@ -424,14 +427,16 @@ static int mca_bml_r2_add_proc (struct ompi_proc_t *proc)
         OBJ_RELEASE(bml_endpoint);
         /* no btl is available for this proc */
         if (mca_bml_r2.show_unreach_errors) {
+            char *errhost = opal_get_proc_hostname(&proc->super);
+            char *localhost = opal_get_proc_hostname(&ompi_proc_local_proc->super);
             opal_show_help ("help-mca-bml-r2.txt", "unreachable proc", true,
                             OMPI_NAME_PRINT(&(ompi_proc_local_proc->super.proc_name)),
-                            (NULL != ompi_proc_local_proc->super.proc_hostname ?
-                             ompi_proc_local_proc->super.proc_hostname : "unknown!"),
+                            localhost,
                             OMPI_NAME_PRINT(&(proc->super.proc_name)),
-                            (NULL != proc->super.proc_hostname ?
-                             proc->super.proc_hostname : "unknown!"),
+                            errhost,
                             btl_names);
+            free(errhost);
+            free(localhost);
         }
 
         return OMPI_ERR_UNREACH;
@@ -578,14 +583,16 @@ static int mca_bml_r2_add_procs( size_t nprocs,
         if (NULL == proc->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_BML]) {
             ret = OMPI_ERR_UNREACH;
             if (mca_bml_r2.show_unreach_errors) {
+                char *errhost = opal_get_proc_hostname(&proc->super);
+                char *localhost = opal_get_proc_hostname(&ompi_proc_local_proc->super);
                 opal_show_help("help-mca-bml-r2.txt", "unreachable proc", true,
                                OMPI_NAME_PRINT(&(ompi_proc_local_proc->super.proc_name)),
-                               (NULL != ompi_proc_local_proc->super.proc_hostname ?
-                                ompi_proc_local_proc->super.proc_hostname : "unknown!"),
+                               localhost,
                                OMPI_NAME_PRINT(&(proc->super.proc_name)),
-                               (NULL != proc->super.proc_hostname ?
-                                proc->super.proc_hostname : "unknown!"),
+                               errhost,
                                btl_names);
+                free(errhost);
+                free(localhost);
             }
 
             break;

ompi/mca/mtl/ofi/mtl_ofi.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2013-2018 Intel, Inc. All rights reserved
+ * Copyright (c) 2013-2020 Intel, Inc.  All rights reserved.
  *
  * $COPYRIGHT$
  *
@@ -98,10 +98,11 @@ ompi_mtl_ofi_add_procs(struct mca_mtl_base_module_t *mtl,
                               (void**)&ep_name,
                               &size);
         if (OMPI_SUCCESS != ret) {
+            char *errhost = opal_get_proc_hostname(&procs[i]->super);
             opal_show_help("help-mtl-ofi.txt", "modex failed",
                            true, ompi_process_info.nodename,
-			   procs[i]->super.proc_hostname,
-			   opal_strerror(ret), ret);
+			                     errhost, opal_strerror(ret), ret);
+            free(errhost);
             goto bail;
         }
         memcpy(&ep_names[i*namelen], ep_name, namelen);

ompi/mca/mtl/psm2/mtl_psm2.c

@@ -324,8 +324,9 @@ ompi_mtl_psm2_add_procs(struct mca_mtl_base_module_t *mtl,
 			    errstr ? errstr : "unknown connect error");
 		for (j = 0; j < (int) nprocs; j++) {
 		  if (errs_out[j] == thiserr) {
-                      opal_output(0, " %s", (NULL == procs[j]->super.proc_hostname) ?
-                                  "unknown" : procs[j]->super.proc_hostname);
+                      char *errhost = opal_get_proc_hostname(&procs[j]->super);
+                      opal_output(0, " %s", errhost);
+                      free(errhost);
 		  }
 		}
 		opal_output(0, "\n");

ompi/mca/pml/base/pml_base_select.c

@@ -12,7 +12,7 @@
  *                         All rights reserved.
  * Copyright (c) 2012      Los Alamos National Security, LLC.  All rights
  *                         reserved.
- * Copyright (c) 2013-2019 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2013-2020 Intel, Inc.  All rights reserved.
  * Copyright (c) 2015 Cisco Systems, Inc.  All rights reserved.
  * $COPYRIGHT$
  *
@@ -370,12 +370,15 @@ mca_pml_base_pml_check_selected(const char *my_pml,
     /* if that module doesn't match my own, return an error */
     if ((size != strlen(my_pml) + 1) ||
         (0 != strcmp(my_pml, remote_pml))) {
+        char *errhost = opal_get_proc_hostname(&procs[0]->super);
         opal_output(0, "%s selected pml %s, but peer %s on %s selected pml %s",
                     OMPI_NAME_PRINT(&ompi_proc_local()->super.proc_name),
                     my_pml, OMPI_NAME_PRINT(&procs[0]->super.proc_name),
-                    (NULL == procs[0]->super.proc_hostname) ? "unknown" : procs[0]->super.proc_hostname,
+                    errhost,
                     remote_pml);
-        free(remote_pml); /* cleanup before returning */
+        free(remote_pml);
+        free(errhost);
+         /* cleanup before returning */
         return OMPI_ERR_UNREACH;
     }
 

ompi/mca/pml/ob1/pml_ob1_cuda.c

@@ -15,6 +15,7 @@
  * Copyright (c) 2012-2015 NVIDIA Corporation.  All rights reserved.
  * Copyright (c) 2015-2017 Los Alamos National Security, LLC. All rights
  *                         reserved.
+ * Copyright (c) 2020      Intel, Inc.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -202,14 +203,17 @@ void mca_pml_ob1_cuda_add_ipc_support(struct mca_btl_base_module_t* btl, int32_t
     /* Find the corresponding bml and adjust the flag to support CUDA get */
     for( i = 0; i < (int)ep->btl_send.arr_size; i++ ) {
         if( ep->btl_send.bml_btls[i].btl == btl ) {
+            if (4 < opal_output_get_verbosity(btl_verbose_stream)) {
+                char *errhost = opal_get_proc_hostname(&errproc->super);
+                opal_output(0, "BTL %s: rank=%d enabling CUDA IPC "
+                            "to rank=%d on node=%s \n",
+                            btl->btl_component->btl_version.mca_component_name,
+                            OMPI_PROC_MY_NAME->vpid,
+                            ((ompi_process_name_t*)&errproc->super.proc_name)->vpid,
+                            errhost);
+                free(errhost);
+            }
             ep->btl_send.bml_btls[i].btl_flags |= MCA_BTL_FLAGS_CUDA_GET;
-            opal_output_verbose(5, btl_verbose_stream,
-                        "BTL %s: rank=%d enabling CUDA IPC "
-                        "to rank=%d on node=%s \n",
-                        btl->btl_component->btl_version.mca_component_name,
-                        OMPI_PROC_MY_NAME->vpid,
-                        ((ompi_process_name_t*)&errproc->super.proc_name)->vpid,
-                        errproc->super.proc_hostname);
         }
     }
 }

ompi/proc/proc.c

@@ -86,9 +86,6 @@ void ompi_proc_destruct(ompi_proc_t* proc)
      * destroyed here. It will be destroyed later when the ompi_datatype_finalize is called.
      */
     OBJ_RELEASE( proc->super.proc_convertor );
-    if (NULL != proc->super.proc_hostname) {
-        free(proc->super.proc_hostname);
-    }
     opal_mutex_lock (&ompi_proc_lock);
     opal_list_remove_item(&ompi_proc_list, (opal_list_item_t*)proc);
     opal_hash_table_remove_value_ptr (&ompi_proc_hash, &proc->super.proc_name, sizeof (proc->super.proc_name));
@@ -135,22 +132,12 @@ static int ompi_proc_allocate (ompi_jobid_t jobid, ompi_vpid_t vpid, ompi_proc_t
  */
 int ompi_proc_complete_init_single (ompi_proc_t *proc)
 {
-    int ret;
-
     if ((OMPI_CAST_RTE_NAME(&proc->super.proc_name)->jobid == OMPI_PROC_MY_NAME->jobid) &&
         (OMPI_CAST_RTE_NAME(&proc->super.proc_name)->vpid  == OMPI_PROC_MY_NAME->vpid)) {
         /* nothing else to do */
         return OMPI_SUCCESS;
     }
 
-    /* we can retrieve the hostname at no cost because it
-     * was provided at startup - but make it optional so
-     * we don't chase after it if some system doesn't
-     * provide it */
-    proc->super.proc_hostname = NULL;
-    OPAL_MODEX_RECV_VALUE_OPTIONAL(ret, PMIX_HOSTNAME, &proc->super.proc_name,
-                                   (char**)&(proc->super.proc_hostname), PMIX_STRING);
-
 #if OPAL_ENABLE_HETEROGENEOUS_SUPPORT
     /* get the remote architecture - this might force a modex except
      * for those environments where the RM provides it */
@@ -264,7 +251,6 @@ int ompi_proc_init(void)
     /* set local process data */
     ompi_proc_local_proc = proc;
     proc->super.proc_flags = OPAL_PROC_ALL_LOCAL;
-    proc->super.proc_hostname = strdup(ompi_process_info.nodename);
     proc->super.proc_arch = opal_local_arch;
     /* Register the local proc with OPAL */
     opal_proc_local_set(&proc->super);
@@ -609,7 +595,6 @@ int ompi_proc_refresh(void)
         if (i == OMPI_PROC_MY_NAME->vpid) {
             ompi_proc_local_proc = proc;
             proc->super.proc_flags = OPAL_PROC_ALL_LOCAL;
-            proc->super.proc_hostname = ompi_process_info.nodename;
             proc->super.proc_arch = opal_local_arch;
             opal_proc_local_set(&proc->super);
         } else {
@@ -676,13 +661,6 @@ ompi_proc_pack(ompi_proc_t **proclist, int proclistsize,
             opal_mutex_unlock (&ompi_proc_lock);
             return rc;
         }
-        /* pass the name of the host this proc is on */
-        rc = opal_dss.pack(buf, &(proc->super.proc_hostname), 1, OPAL_STRING);
-        if(rc != OPAL_SUCCESS) {
-            OMPI_ERROR_LOG(rc);
-            opal_mutex_unlock (&ompi_proc_lock);
-            return rc;
-        }
     }
     opal_mutex_unlock (&ompi_proc_lock);
     return OMPI_SUCCESS;
@@ -747,10 +725,10 @@ ompi_proc_unpack(opal_buffer_t* buf,
         int32_t count=1;
         ompi_process_name_t new_name;
         uint32_t new_arch;
-        char *new_hostname;
         bool isnew = false;
         int rc;
         char *nspace;
+        uint16_t u16, *u16ptr;
 
         rc = opal_dss.unpack(buf, &new_name, &count, OMPI_NAME);
         if (rc != OPAL_SUCCESS) {
@@ -774,13 +752,6 @@ ompi_proc_unpack(opal_buffer_t* buf,
             free(newprocs);
             return rc;
         }
-        rc = opal_dss.unpack(buf, &new_hostname, &count, OPAL_STRING);
-        if (rc != OPAL_SUCCESS) {
-            OMPI_ERROR_LOG(rc);
-            free(plist);
-            free(newprocs);
-            return rc;
-        }
         /* see if this proc is already on our ompi_proc_list */
         plist[i] = ompi_proc_find_and_add(&new_name, &isnew);
         if (isnew) {
@@ -798,27 +769,25 @@ ompi_proc_unpack(opal_buffer_t* buf,
                 OBJ_RELEASE(plist[i]->super.proc_convertor);
                 plist[i]->super.proc_convertor = opal_convertor_create(plist[i]->super.proc_arch, 0);
 #else
+                char *errhost = opal_get_proc_hostname(&plist[i]->super);
                 opal_show_help("help-mpi-runtime.txt",
                                "heterogeneous-support-unavailable",
                                true, ompi_process_info.nodename,
-                               new_hostname == NULL ? "<hostname unavailable>" :
-                               new_hostname);
+                               errhost);
                 free(plist);
                 free(newprocs);
+                free(errhost);
                 return OMPI_ERR_NOT_SUPPORTED;
 #endif
             }
 
-            if (NULL != new_hostname) {
-                if (0 == strcmp(ompi_proc_local_proc->super.proc_hostname, new_hostname)) {
-                    plist[i]->super.proc_flags |= (OPAL_PROC_ON_NODE | OPAL_PROC_ON_CU | OPAL_PROC_ON_CLUSTER);
-                }
-
-                /* Save the hostname */
-                plist[i]->super.proc_hostname = new_hostname;
+            /* get the locality information - all RTEs are required
+             * to provide this information at startup */
+            u16ptr = &u16;
+            OPAL_MODEX_RECV_VALUE_OPTIONAL(rc, PMIX_LOCALITY, &plist[i]->super.proc_name, &u16ptr, PMIX_UINT16);
+            if (OPAL_SUCCESS == rc) {
+                plist[i]->super.proc_flags = u16;
             }
-        } else if (NULL != new_hostname) {
-            free(new_hostname);
         }
     }
 

opal/mca/btl/base/btl_base_error.h

@@ -13,7 +13,7 @@
  * Copyright (c) 2007      Sun Microsystems, Inc.  All rights reserved.
  * Copyright (c) 2012      Los Alamos National Security, LLC.
  *                         All rights reserved.
- * Copyright (c) 2013-2014 Intel, Inc. All rights reserved
+ * Copyright (c) 2013-2020 Intel, Inc.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -59,13 +59,15 @@ OPAL_DECLSPEC extern int mca_btl_base_out(const char*, ...) __opal_attribute_for
 
 #define BTL_PEER_ERROR(proc, args)                              \
     do {                                                        \
+        char *errhost;                                          \
         mca_btl_base_err("%s[%s:%d:%s] from %s ",               \
                          OPAL_NAME_PRINT(OPAL_PROC_MY_NAME),    \
                          __FILE__, __LINE__, __func__,          \
                          opal_process_info.nodename);           \
         if (proc) {                                             \
-            mca_btl_base_err("to: %s ",                         \
-                             opal_get_proc_hostname(proc));     \
+            errhost = opal_get_proc_hostname(proc);             \
+            mca_btl_base_err("to: %s ", errhost);               \
+            free(errhost);                                      \
         }                                                       \
         mca_btl_base_err args;                                  \
         mca_btl_base_err("\n");                                 \

opal/mca/btl/tcp/btl_tcp_frag.c

@@ -15,6 +15,7 @@
  * Copyright (c) 2015-2017 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2015-2016 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2020      Intel, Inc.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -44,6 +45,7 @@
 
 #include "opal/opal_socket_errno.h"
 #include "opal/mca/btl/base/btl_base_error.h"
+#include "opal/util/proc.h"
 #include "opal/util/show_help.h"
 
 #include "btl_tcp_frag.h"
@@ -168,6 +170,7 @@ bool mca_btl_tcp_frag_recv(mca_btl_tcp_frag_t* frag, int sd)
     mca_btl_base_endpoint_t* btl_endpoint = frag->endpoint;
     ssize_t cnt;
     int32_t i, num_vecs, dont_copy_data = 0;
+    char *errhost;
 
  repeat:
     num_vecs = frag->iov_cnt;
@@ -231,10 +234,11 @@ bool mca_btl_tcp_frag_recv(mca_btl_tcp_frag_t* frag, int sd)
             return false;
 
         case ECONNRESET:
+            errhost = opal_get_proc_hostname(btl_endpoint->endpoint_proc->proc_opal);
             opal_show_help("help-mpi-btl-tcp.txt", "peer hung up",
                            true, opal_process_info.nodename,
-                           getpid(),
-                           btl_endpoint->endpoint_proc->proc_opal->proc_hostname);
+                           getpid(), errhost);
+            free(errhost);
             btl_endpoint->endpoint_state = MCA_BTL_TCP_FAILED;
             mca_btl_tcp_endpoint_close(btl_endpoint);
             return false;