sharedfp/sm and lockedfile: fix naming bug

If an application opens a file from multiple processes
using MPI_COMM_SELF (or another set of communicators that have
the same CID on distinct process groups, as can happen as the result
of comm_split), the naming chosen for the managing the shared file pointer position
in a file (in sharedfp/lockedfile) or the mmapped shared memory region
(in sharedfp/sm) component would lead to a collision between the distinct process groups.
This patch ensures that the filename is different by integrating the process id
of rank 0 for each sub-communicator.

This fixes one aspect of the problem reported in github issue 5593

This commit is the equivalent of commit 9b65ec9 on master.
It can not be cherry-picked directly due to significant changes in the overal
organization of the file.

Signed-off-by: Edgar Gabriel <[email protected]>

Author: edgargabriel

ompi/mca/sharedfp/lockedfile/sharedfp_lockedfile_file_open.c

@@ -34,6 +34,7 @@
 #include <sys/stat.h>
 #endif
 #include <fcntl.h>
+#include <unistd.h>
 
 int mca_sharedfp_lockedfile_file_open (struct ompi_communicator_t *comm,
 				       const char* filename,
@@ -49,6 +50,9 @@ int mca_sharedfp_lockedfile_file_open (struct ompi_communicator_t *comm,
     mca_io_ompio_file_t * shfileHandle, *ompio_fh;
     mca_io_ompio_data_t *data;
 
+    pid_t my_pid;
+    int int_pid;
+
     /*------------------------------------------------------------*/
     /*Open the same file again without shared file pointer support*/
     /*------------------------------------------------------------*/
@@ -109,15 +113,27 @@ int mca_sharedfp_lockedfile_file_open (struct ompi_communicator_t *comm,
     comm->c_coll->coll_bcast ( &masterjobid, 1, MPI_UNSIGNED, 0, comm, 
                                comm->c_coll->coll_bcast_module );
 
-    size_t filenamelen = strlen(filename) + 16;
+    if ( 0 == fh->f_rank ) {
+        my_pid = getpid();
+        int_pid = (int) my_pid;
+    }
+    err = comm->c_coll->coll_bcast (&int_pid, 1, MPI_INT, 0, comm, comm->c_coll->coll_bcast_module );
+    if ( OMPI_SUCCESS != err ) {
+        opal_output(0, "[%d]mca_sharedfp_lockedfile_file_open: Error in bcast operation\n", fh->f_rank);
+        free (sh);
+        free(module_data);
+        return err;
+    }
+
+    size_t filenamelen = strlen(filename) + 24;
     lockedfilename = (char*)malloc(sizeof(char) * filenamelen);
     if ( NULL == lockedfilename ) {
 	free (shfileHandle);
 	free (sh);
         free (module_data);
         return OMPI_ERR_OUT_OF_RESOURCE;
     }
-    snprintf(lockedfilename, filenamelen, "%s-%u%s",filename,masterjobid,".lock");
+    snprintf(lockedfilename, filenamelen, "%s-%u-%d%s",filename,masterjobid,int_pid,".lock");
     module_data->filename = lockedfilename;
 
     /*-------------------------------------------------*/

ompi/mca/sharedfp/sm/sharedfp_sm_file_open.c

@@ -43,7 +43,7 @@
 #include <semaphore.h>
 #include <sys/mman.h>
 #include <libgen.h>
-
+#include <unistd.h>
 
 int mca_sharedfp_sm_file_open (struct ompi_communicator_t *comm,
                                const char* filename,
@@ -57,12 +57,16 @@ int mca_sharedfp_sm_file_open (struct ompi_communicator_t *comm,
     mca_io_ompio_file_t * shfileHandle, *ompio_fh;
     char * filename_basename;
     char * sm_filename;
+    int sm_filename_length;
     struct mca_sharedfp_sm_offset * sm_offset_ptr;
     struct mca_sharedfp_sm_offset sm_offset;
     mca_io_ompio_data_t *data;
     int sm_fd;
     int rank;
-
+    uint32_t comm_cid;
+    int int_pid;
+    pid_t my_pid;
+    
     /*----------------------------------------------------*/
     /*Open the same file again without shared file pointer*/
     /*----------------------------------------------------*/
@@ -132,25 +136,33 @@ int mca_sharedfp_sm_file_open (struct ompi_communicator_t *comm,
     ** TODO: properly name the file so that different jobs can run on the same system w/o
     **      overwriting each other, e.g.  orte_process_info.proc_session_dir
     */
-    /*sprintf(sm_filename,"%s%s",filename,".sm");*/
-    filename_basename = basename((void *)filename);
-    sm_filename = (char*) malloc( sizeof(char) * (strlen(filename_basename)+64) );
+    filename_basename = basename((char *)filename);
+    /* format is "%s/%s_cid-%d-%d.sm", see below */
+    sm_filename_length = strlen(ompi_process_info.job_session_dir) + 1 + strlen(filename_basename) + 5 + (3*sizeof(uint32_t)+1) + 4;
+    sm_filename = (char*) malloc( sizeof(char) * sm_filename_length);
     if (NULL == sm_filename) {
         free(sm_data);
         free(sh);
         free(shfileHandle);
         return OMPI_ERR_OUT_OF_RESOURCE;
     }
 
-    opal_jobid_t masterjobid;
-    if ( 0 == comm->c_my_rank ) {
-        ompi_proc_t *masterproc = ompi_group_peer_lookup(comm->c_local_group, 0 );
-        masterjobid = OMPI_CAST_RTE_NAME(&masterproc->super.proc_name)->jobid;
+    comm_cid = ompi_comm_get_cid(comm);
+    if ( 0 == fh->f_rank ) {
+        my_pid = getpid();
+        int_pid = (int) my_pid;
+    }
+    err = comm->c_coll->coll_bcast (&int_pid, 1, MPI_INT, 0, comm, comm->c_coll->coll_bcast_module );
+    if ( OMPI_SUCCESS != err ) {
+        opal_output(0,"mca_sharedfp_sm_file_open: Error in bcast operation \n");
+        free(sm_filename);
+        free(sm_data);
+        free(sh);
+        return err;
     }
-    comm->c_coll->coll_bcast ( &masterjobid, 1, MPI_UNSIGNED, 0, comm, 
-                               comm->c_coll->coll_bcast_module );
+    snprintf(sm_filename, sm_filename_length, "%s/%s_cid-%d-%d.sm", ompi_process_info.job_session_dir,
+             filename_basename, comm_cid, int_pid);
 
-    sprintf(sm_filename,"/tmp/OMPIO_%s_%d_%s",filename_basename, masterjobid, ".sm");
     /* open shared memory file, initialize to 0, map into memory */
     sm_fd = open(sm_filename, O_RDWR | O_CREAT,
                  S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH);