Several fixes related to session directories:

* enable OMPI to retrieve paths from RM through PMIx
* cleanups related to tempdirs.

Author: artpol84

orte/mca/ess/base/ess_base_std_app.c

@@ -136,10 +136,7 @@ int orte_ess_base_app_setup(bool db_restrict_local)
                              ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
                              (NULL == orte_process_info.tmpdir_base) ? "UNDEF" : orte_process_info.tmpdir_base,
                              orte_process_info.nodename));
-        if (ORTE_SUCCESS != (ret = orte_session_dir(true,
-                                                    orte_process_info.tmpdir_base,
-                                                    orte_process_info.nodename,
-                                                    ORTE_PROC_MY_NAME))) {
+        if (ORTE_SUCCESS != (ret = orte_session_dir(true, ORTE_PROC_MY_NAME))) {
             ORTE_ERROR_LOG(ret);
             error = "orte_session_dir";
             goto error;
@@ -149,29 +146,6 @@ int orte_ess_base_app_setup(bool db_restrict_local)
            proc-specific session directory. */
         opal_output_set_output_file_info(orte_process_info.proc_session_dir,
                                          "output-", NULL, NULL);
-        /* store the session directory location */
-        OBJ_CONSTRUCT(&kv, opal_value_t);
-        kv.key = strdup(OPAL_PMIX_NSDIR);
-        kv.type = OPAL_STRING;
-        kv.data.string = strdup(orte_process_info.job_session_dir);
-        if (OPAL_SUCCESS != (ret = opal_pmix.store_local(ORTE_PROC_MY_NAME, &kv))) {
-            ORTE_ERROR_LOG(ret);
-            OBJ_DESTRUCT(&kv);
-            error = "opal pmix put job sessiondir";
-            goto error;
-        }
-        OBJ_DESTRUCT(&kv);
-        OBJ_CONSTRUCT(&kv, opal_value_t);
-        kv.key = strdup(OPAL_PMIX_PROCDIR);
-        kv.type = OPAL_STRING;
-        kv.data.string = strdup(orte_process_info.proc_session_dir);
-        if (OPAL_SUCCESS != (ret = opal_pmix.store_local(ORTE_PROC_MY_NAME, &kv))) {
-            ORTE_ERROR_LOG(ret);
-            OBJ_DESTRUCT(&kv);
-            error = "opal pmix put proc sessiondir";
-            goto error;
-        }
-        OBJ_DESTRUCT(&kv);
     }
     /* Setup the communication infrastructure */
     /*

orte/mca/ess/base/ess_base_std_orted.c

@@ -237,10 +237,7 @@ int orte_ess_base_orted_setup(char **hosts)
         /* take a pass thru the session directory code to fillin the
          * tmpdir names - don't create anything yet
          */
-        if (ORTE_SUCCESS != (ret = orte_session_dir(false,
-                                                    orte_process_info.tmpdir_base,
-                                                    orte_process_info.nodename,
-                                                    ORTE_PROC_MY_NAME))) {
+        if (ORTE_SUCCESS != (ret = orte_session_dir(false, ORTE_PROC_MY_NAME))) {
             ORTE_ERROR_LOG(ret);
             error = "orte_session_dir define";
             goto error;
@@ -250,10 +247,7 @@ int orte_ess_base_orted_setup(char **hosts)
          */
         orte_session_dir_cleanup(ORTE_JOBID_WILDCARD);
         /* now actually create the directory tree */
-        if (ORTE_SUCCESS != (ret = orte_session_dir(true,
-                                                    orte_process_info.tmpdir_base,
-                                                    orte_process_info.nodename,
-                                                    ORTE_PROC_MY_NAME))) {
+        if (ORTE_SUCCESS != (ret = orte_session_dir(true, ORTE_PROC_MY_NAME))) {
             ORTE_ERROR_LOG(ret);
             error = "orte_session_dir";
             goto error;
@@ -277,11 +271,8 @@ int orte_ess_base_orted_setup(char **hosts)
             /* define a log file name in the session directory */
             snprintf(log_file, PATH_MAX, "output-orted-%s-%s.log",
                      jobidstring, orte_process_info.nodename);
-            log_path = opal_os_path(false,
-                                    orte_process_info.tmpdir_base,
-                                    orte_process_info.top_session_dir,
-                                    log_file,
-                                    NULL);
+            log_path = opal_os_path(false, orte_process_info.top_session_dir,
+                                    log_file, NULL);
 
             fd = open(log_path, O_RDWR|O_CREAT|O_TRUNC, 0640);
             if (fd < 0) {

orte/mca/ess/base/ess_base_std_tool.c

@@ -145,10 +145,9 @@ int orte_ess_base_tool_setup(void)
      * tmp base where any other session directories on
      * this node might be located
      */
-    if (ORTE_SUCCESS != (ret = orte_session_dir_get_name(NULL,
-                                                         &orte_process_info.tmpdir_base,
-                                                         &orte_process_info.top_session_dir,
-                                                         orte_process_info.nodename, NULL))) {
+
+    ret = orte_session_setup_base(NULL);
+    if (ORTE_SUCCESS != ret ) {
         ORTE_ERROR_LOG(ret);
         error = "define session dir names";
         goto error;

orte/mca/ess/hnp/ess_hnp_module.c

@@ -138,7 +138,7 @@ static int rte_init(void)
 {
     int ret;
     char *error = NULL;
-    char *contact_path, *jobfam_dir;
+    char *contact_path;
     orte_job_t *jdata;
     orte_node_t *node;
     orte_proc_t *proc;
@@ -294,10 +294,7 @@ static int rte_init(void)
         /* take a pass thru the session directory code to fillin the
          * tmpdir names - don't create anything yet
          */
-        if (ORTE_SUCCESS != (ret = orte_session_dir(false,
-                                                    orte_process_info.tmpdir_base,
-                                                    orte_process_info.nodename,
-                                                    ORTE_PROC_MY_NAME))) {
+        if (ORTE_SUCCESS != (ret = orte_session_dir(false, ORTE_PROC_MY_NAME))) {
             error = "orte_session_dir define";
             goto error;
         }
@@ -307,10 +304,7 @@ static int rte_init(void)
         orte_session_dir_cleanup(ORTE_JOBID_WILDCARD);
 
         /* now actually create the directory tree */
-        if (ORTE_SUCCESS != (ret = orte_session_dir(true,
-                                                    orte_process_info.tmpdir_base,
-                                                    orte_process_info.nodename,
-                                                    ORTE_PROC_MY_NAME))) {
+        if (ORTE_SUCCESS != (ret = orte_session_dir(true, ORTE_PROC_MY_NAME))) {
             error = "orte_session_dir";
             goto error;
         }
@@ -586,9 +580,12 @@ static int rte_init(void)
         opal_output_set_output_file_info(orte_process_info.proc_session_dir,
                                          "output-", NULL, NULL);
         /* save my contact info in a file for others to find */
-        jobfam_dir = opal_dirname(orte_process_info.job_session_dir);
-        contact_path = opal_os_path(false, jobfam_dir, "contact.txt", NULL);
-        free(jobfam_dir);
+        if( NULL == orte_process_info.jobfam_session_dir ){
+            /* has to be set here! */
+            ORTE_ERROR_LOG(ORTE_ERR_BAD_PARAM);
+            goto error;
+        }
+        contact_path = opal_os_path(false, orte_process_info.jobfam_session_dir, "contact.txt", NULL);
         OPAL_OUTPUT_VERBOSE((2, orte_debug_output,
                              "%s writing contact file %s",
                              ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
@@ -758,10 +755,9 @@ static int rte_init(void)
                        true, error, ORTE_ERROR_NAME(ret), ret);
     }
     /* remove my contact info file, if we have session directories */
-    if (NULL != orte_process_info.job_session_dir) {
-        jobfam_dir = opal_dirname(orte_process_info.job_session_dir);
-        contact_path = opal_os_path(false, jobfam_dir, "contact.txt", NULL);
-        free(jobfam_dir);
+    if (NULL != orte_process_info.jobfam_session_dir) {
+        contact_path = opal_os_path(false, orte_process_info.jobfam_session_dir,
+                                    "contact.txt", NULL);
         unlink(contact_path);
         free(contact_path);
     }
@@ -775,7 +771,6 @@ static int rte_init(void)
 static int rte_finalize(void)
 {
     char *contact_path;
-    char *jobfam_dir;
 
     if (signals_set) {
         /* Remove the epipe handler */
@@ -816,10 +811,9 @@ static int rte_finalize(void)
     (void) mca_base_framework_close(&opal_pstat_base_framework);
 
     /* remove my contact info file, if we have session directories */
-    if (NULL != orte_process_info.job_session_dir) {
-        jobfam_dir = opal_dirname(orte_process_info.job_session_dir);
-        contact_path = opal_os_path(false, jobfam_dir, "contact.txt", NULL);
-        free(jobfam_dir);
+    if (NULL != orte_process_info.jobfam_session_dir) {
+        contact_path = opal_os_path(false, orte_process_info.jobfam_session_dir,
+                                    "contact.txt", NULL);
         unlink(contact_path);
         free(contact_path);
     }

orte/mca/ess/pmi/ess_pmi_module.c

@@ -242,6 +242,43 @@ static int rte_init(void)
         free(string_key);
     }
 
+    /* retrieve temp directories info */
+    OPAL_MODEX_RECV_VALUE_OPTIONAL(ret, OPAL_PMIX_TMPDIR, &wildcard_rank, &val, OPAL_STRING);
+    if (OPAL_SUCCESS == ret && NULL != val) {
+        /* TODO: who has precedence - pmix of MCA setting??? */
+        if( NULL == orte_process_info.top_session_dir ){
+            orte_process_info.top_session_dir = val;
+        } else {
+            /* keep the MCA setting */
+            free(val);
+        }
+        val = NULL;
+    }
+
+    OPAL_MODEX_RECV_VALUE_OPTIONAL(ret, OPAL_PMIX_NSDIR, &wildcard_rank, &val, OPAL_STRING);
+    if (OPAL_SUCCESS == ret && NULL != val) {
+        /* TODO: who has precedence - pmix of MCA setting??? */
+        if( NULL == orte_process_info.job_session_dir ){
+            orte_process_info.job_session_dir = val;
+        } else {
+            /* keep the MCA setting */
+            free(val);
+        }
+        val = NULL;
+    }
+
+    OPAL_MODEX_RECV_VALUE_OPTIONAL(ret, OPAL_PMIX_PROCDIR, &wildcard_rank, &val, OPAL_STRING);
+    if (OPAL_SUCCESS == ret && NULL != val) {
+        /* TODO: who has precedence - pmix of MCA setting??? */
+        if( NULL == orte_process_info.proc_session_dir ){
+            orte_process_info.proc_session_dir = val;
+        } else {
+            /* keep the MCA setting */
+            free(val);
+        }
+        val = NULL;
+    }
+
     /* retrieve our topology */
     val = NULL;
     OPAL_MODEX_RECV_VALUE_OPTIONAL(ret, OPAL_PMIX_LOCAL_TOPO,

orte/mca/filem/raw/filem_raw_module.c

@@ -105,6 +105,17 @@ static void recv_ack(int status, orte_process_name_t* sender,
                      void* cbdata);
 static void write_handler(int fd, short event, void *cbdata);
 
+static char *filem_session_dir()
+{
+    char *session_dir = orte_process_info.jobfam_session_dir;
+    if( NULL == session_dir ){
+        /* if no job family session dir was provided -
+         * use the job session dir */
+        session_dir = orte_process_info.job_session_dir;
+    }
+    return session_dir;
+}
+
 static int raw_init(void)
 {
     OBJ_CONSTRUCT(&incoming_files, opal_list_t);
@@ -657,25 +668,26 @@ static int create_link(char *my_dir, char *path,
 static int raw_link_local_files(orte_job_t *jdata,
                                 orte_app_context_t *app)
 {
-    char *my_dir, *path=NULL;
+    char *session_dir, *path=NULL;
     orte_proc_t *proc;
-    char *prefix;
     int i, j, rc;
     orte_filem_raw_incoming_t *inbnd;
     opal_list_item_t *item;
     char **files=NULL, *bname, *filestring;
 
-    /* check my session directory for files I have received and
+    /* check my jobfam session directory for files I have received and
      * symlink them to the proc-level session directory of each
      * local process in the job
+     *
+     * TODO: @rhc - please check that I've correctly interpret your
+     *  intention here
      */
-    my_dir = opal_dirname(orte_process_info.job_session_dir);
-
-    /* setup */
-    if (NULL != orte_process_info.tmpdir_base) {
-        prefix = strdup(orte_process_info.tmpdir_base);
-    } else {
-        prefix = NULL;
+    session_dir = filem_session_dir();
+    if( NULL == session_dir){
+        /* we were unable to find any suitable directory */
+        rc = ORTE_ERR_BAD_PARAM;
+        ORTE_ERROR_LOG(rc);
+        return rc;
     }
 
     /* get the list of files this app wants */
@@ -692,10 +704,6 @@ static int raw_link_local_files(orte_job_t *jdata,
 
     /* if there are no files to link, then ignore this */
     if (NULL == files) {
-        free(my_dir);
-        if (NULL != prefix) {
-            free(prefix);
-        }
         return ORTE_SUCCESS;
     }
 
@@ -736,22 +744,15 @@ static int raw_link_local_files(orte_job_t *jdata,
                              ORTE_NAME_PRINT(&proc->name)));
 
         /* get the session dir name in absolute form */
-        path = NULL;
-        rc = orte_session_dir_get_name(&path, &prefix, NULL,
-                                       orte_process_info.nodename,
-                                       &proc->name);
+        path = orte_process_info.proc_session_dir;
+
         /* create it, if it doesn't already exist */
         if (OPAL_SUCCESS != (rc = opal_os_dirpath_create(path, S_IRWXU))) {
             ORTE_ERROR_LOG(rc);
             /* doesn't exist with correct permissions, and/or we can't
              * create it - either way, we are done
              */
             free(files);
-            if (NULL != prefix) {
-                free(prefix);
-            }
-            free(path);
-            free(my_dir);
             return rc;
         }
 
@@ -775,13 +776,8 @@ static int raw_link_local_files(orte_job_t *jdata,
                                              inbnd->file));
                         /* cycle thru the link points and create symlinks to them */
                         for (j=0; NULL != inbnd->link_pts[j]; j++) {
-                            if (ORTE_SUCCESS != (rc = create_link(my_dir, path, inbnd->link_pts[j]))) {
+                            if (ORTE_SUCCESS != (rc = create_link(session_dir, path, inbnd->link_pts[j]))) {
                                 ORTE_ERROR_LOG(rc);
-                                free(my_dir);
-                                free(path);
-                                if (NULL != prefix) {
-                                    free(prefix);
-                                }
                                 free(files);
                                 return rc;
                             }
@@ -796,13 +792,8 @@ static int raw_link_local_files(orte_job_t *jdata,
                 }
             }
         }
-        free(path);
     }
     opal_argv_free(files);
-    if (NULL != prefix) {
-        free(prefix);
-    }
-    free(my_dir);
     return ORTE_SUCCESS;
 }
 
@@ -999,7 +990,7 @@ static void recv_files(int status, orte_process_name_t* sender,
                        opal_buffer_t* buffer, orte_rml_tag_t tag,
                        void* cbdata)
 {
-    char *file, *jobfam_dir;
+    char *file, *session_dir;
     int32_t nchunk, n, nbytes;
     unsigned char data[ORTE_FILEM_RAW_CHUNK_MAX];
     int rc;
@@ -1086,9 +1077,9 @@ static void recv_files(int status, orte_process_name_t* sender,
         incoming->top = strdup(tmp);
         free(tmp);
         /* define the full path to where we will put it */
-        jobfam_dir = opal_dirname(orte_process_info.job_session_dir);
-        incoming->fullpath = opal_os_path(false, jobfam_dir, file, NULL);
-        free(jobfam_dir);
+        session_dir = filem_session_dir();
+
+        incoming->fullpath = opal_os_path(false, session_dir, file, NULL);
 
         OPAL_OUTPUT_VERBOSE((1, orte_filem_base_framework.framework_output,
                              "%s filem:raw: opening target file %s",

orte/mca/oob/usock/oob_usock_component.c

@@ -164,8 +164,7 @@ static int component_available(void)
 
     /* if session directories were forbidden, then we cannot be used */
     if (!orte_create_session_dirs ||
-        NULL == orte_process_info.tmpdir_base ||
-        NULL == orte_process_info.top_session_dir) {
+        NULL == orte_process_info.jobfam_session_dir ) {
         return ORTE_ERR_NOT_SUPPORTED;
     }
 
@@ -216,9 +215,7 @@ static int component_startup(void)
     /* setup the path to the daemon rendezvous point */
     memset(&mca_oob_usock_component.address, 0, sizeof(struct sockaddr_un));
     mca_oob_usock_component.address.sun_family = AF_UNIX;
-    session = opal_os_path(false, orte_process_info.tmpdir_base,
-                           orte_process_info.top_session_dir,
-                           orte_process_info.jobfam_session_dir,
+    session = opal_os_path(false, orte_process_info.jobfam_session_dir,
                            "usock", NULL);
     if ((strlen(session) + 1) > sizeof(mca_oob_usock_component.address.sun_path)-1) {
         opal_output(0, "SESSION DIR TOO LONG");