Merge remote-tracking branch 'upstream/main' into oshmem_base_exchange

Author: tvegas1

.ci/community-jenkins/Jenkinsfile

@@ -35,7 +35,11 @@ milestone(buildNumber)
 // back to the PR. The "Details" link at the bottom of the GitHub PR page brings
 // you to the Jenkins Build page, so we're adding the link back to the GitHub PR
 // page.
-currentBuild.description = "This is a build of <a href=\"${CHANGE_URL}\"}\">Open MPI PR #${CHANGE_ID}</a>"
+if (env.CHANGE_URL) {
+    currentBuild.description = "This is a build of <a href=\"${CHANGE_URL}\"}\">Open MPI PR #${CHANGE_ID}</a>"
+} else {
+    currentBuild.description = "Build of ${BRANCH_NAME}"
+}
 
 check_stages = prepare_check_stages()
 println("Initialized Pipeline")
@@ -53,8 +57,8 @@ println('Tests Completed')
 // build stage is a map of different configurations to test.
 def prepare_check_stages() {
     def configure_options = ["--disable-dlopen", "--disable-oshmem", "--enable-builtin-atomic", "--enable-ipv6"]
-    def compilers = ["clang10", "gcc5", "gcc6", "gcc7", "gcc8", "gcc9", "gcc10"]
-    def platforms = ["amazon_linux_2", "amazon_linux_2-arm64", "rhel8", "ubuntu_18.04"]
+    def compilers = ["clang10", "gcc7", "gcc8", "gcc9", "gcc10"]
+    def platforms = ["amazon_linux_2", "amazon_linux_2-arm64", "rhel8"]
     def check_stages_list = []
 
     // Build everything stage

docs/man-openmpi/man1/mpirun.1.rst

@@ -262,7 +262,7 @@ To map processes:
 
 * ``--bynode``: Launch processes one per node, cycling by node in a
   round-robin fashion.  This spreads processes evenly among nodes and
-  assigns MPI_COMM_WORLD ranks in a round-robin, "by node" manner.
+  assigns ``MPI_COMM_WORLD`` ranks in a round-robin, "by node" manner.
   (deprecated in favor of ``--map-by node``)
 
 * ``--cpu-list <cpus>``: Comma-delimited list of processor IDs to
@@ -272,7 +272,7 @@ To map processes:
   .. note:: You can run Run the hwloc ``lstopo(1)`` command to see a
             list of available cores and their logical IDs.
 
-To order processes' ranks in MPI_COMM_WORLD:
+To order processes' ranks in ``MPI_COMM_WORLD``:
 
 * ``--rank-by <mode>``: Rank in round-robin fashion according to the
   specified mode, defaults to slot. Supported options include
@@ -311,15 +311,15 @@ To manage standard I/O:
   specified filename. Any directories in the filename will
   automatically be created.  Each output file will consist of
   ``filename.id``, where the ``id`` will be the processes' rank in
-  MPI_COMM_WORLD, left-filled with zero's for correct ordering in
+  ``MPI_COMM_WORLD``, left-filled with zero's for correct ordering in
   listings. A relative path value will be converted to an absolute
   path based on the cwd where mpirun is executed. Note that this will
   not work on environments where the file system on compute nodes
   differs from that where :ref:`mpirun(1) <man1-mpirun>` is
   executed.
 
-* ``--stdin <rank>``: The MPI_COMM_WORLD rank of the process that is
-  to receive stdin.  The default is to forward stdin to MPI_COMM_WORLD
+* ``--stdin <rank>``: The ``MPI_COMM_WORLD`` rank of the process that is
+  to receive stdin.  The default is to forward stdin to ``MPI_COMM_WORLD``
   rank 0, but this option can be used to forward stdin to any
   process. It is also acceptable to specify none, indicating that no
   processes are to receive stdin.
@@ -329,7 +329,7 @@ To manage standard I/O:
 
 * ``--tag-output``: Tag each line of output to stdout, stderr, and
   stddiag with ``[jobid, MCW_rank]<stdxxx>`` indicating the process
-  jobid and MPI_COMM_WORLD rank of the process that generated the
+  jobid and ``MPI_COMM_WORLD`` rank of the process that generated the
   output, and the channel which generated it.
 
 * ``--timestamp-output``: Timestamp each line of output to stdout,
@@ -342,7 +342,7 @@ To manage standard I/O:
   specified file.
 
 * ``--xterm <ranks>``: Display the output from the processes
-  identified by their MPI_COMM_WORLD ranks in separate xterm
+  identified by their ``MPI_COMM_WORLD`` ranks in separate xterm
   windows. The ranks are specified as a comma-separated list of
   ranges, with a ``-1`` indicating all. A separate window will be created
   for each specified process.
@@ -422,7 +422,22 @@ Setting MCA parameters:
 
 * ``--mca <key> <value>``: Send arguments to various MCA modules.  See
   the :ref:`Setting MCA Parameters
-  <man1-mpirun-setting-mca-parameters>` section for mode details.
+  <man1-mpirun-setting-mca-parameters>` section for more details.
+
+  .. note:: Open MPI will attempt to discern PMIx and PRRTE MCA
+            parameters passed via ``--mca`` and handle them
+            appropriately, but it may not always guess correctly.  It
+            is best to use ``--pmixmca`` and ``--prtemca`` when
+            passing MCA parammeters to PMIx and PRRTE, respectively.
+
+* ``--pmixmca <key> <value>``: Send arguments to MCA modules in the
+  PMIx subsystem.  See the :ref:`Setting MCA Parameters
+  <man1-mpirun-setting-mca-parameters>` section for more details.
+
+* ``--prtemca <key> <value>``: Send arguments to MCA modules in the
+  PMIx Reference Runtime Environment (PRRTE) subsystem.  See the
+  :ref:`Setting MCA Parameters <man1-mpirun-setting-mca-parameters>`
+  section for more details.
 
 * ``--tune <tune_file>``: Specify a tune file to set arguments for
   various MCA modules and environment variables.  See the :ref:`
@@ -532,11 +547,11 @@ generally useful to most Open MPI users:
   just before launch.
 
 * ``--launch-agent``: Name of the executable that is to be used to
-  start processes on the remote nodes. The default is ``PRRTEd``. This
+  start processes on the remote nodes. The default is ``prted``. This
   option can be used to test new daemon concepts, or to pass options
   back to the daemons without having mpirun itself see them. For
-  example, specifying a launch agent of ``PRRTEd -mca odls_base_verbose
-  5`` allows the developer to ask the ``PRRTEd`` for debugging output
+  example, specifying a launch agent of ``prted --prtemca odls_base_verbose
+  5`` allows the developer to ask the ``prted`` for debugging output
   without clutter from ``mpirun`` itself.
 
 * ``--report-state-on-timeout``: When paired with the ``--timeout``
@@ -716,7 +731,7 @@ options that describe mapping policies.
 
 Consider the same hostfile as above, again with ``-n 6``.  The table
 below lists a few ``mpirun`` variations, and shows which
-MPI_COMM_WORLD ranks end up on which node:
+``MPI_COMM_WORLD`` ranks end up on which node:
 
 .. list-table::
    :header-rows: 1
@@ -832,7 +847,7 @@ Open MPI employs a three-phase procedure for assigning process locations
 and ranks:
 
 #. **Mapping**: Assigns a default location to each process
-#. **Ranking**: Assigns an MPI_COMM_WORLD rank value to each process
+#. **Ranking**: Assigns an ``MPI_COMM_WORLD`` rank value to each process
 #. **Binding**: Constrains each process to run on specific processors
 
 The mapping step is used to assign a default location to each process
@@ -864,7 +879,7 @@ gives you detailed control over process binding as well.  Rankfiles
 are discussed :ref:`below <man1-mpirun-rankfiles>`.
 
 The second phase focuses on the ranking of the process within the
-job's MPI_COMM_WORLD.  Open MPI separates this from the mapping
+job's ``MPI_COMM_WORLD``.  Open MPI separates this from the mapping
 procedure to allow more flexibility in the relative placement of MPI
 processes. This is best illustrated by considering the following
 cases where we used the ``--np 8 --map-by ppr:2:package --host aa:4,bb:4`` option:
@@ -1013,10 +1028,12 @@ MCA parameters can be set not only on the mpirun command line, but
 alternatively in a system or user ``mca-params.conf`` file or as
 environment variables, as described in the :ref:`Setting MCA
 Parameters <man1-mpirun-setting-mca-parameters>`. These are MCA parameters for
-the PRRTE runtime so the command line argument ``--PRRTEmca`` must be used to
-pass the MCA parameter key/value pair. Alternatively, the MCA parameter key/
-value pair may be specific on the command line by prefixing the key with
-``PRRTE_MCA_``. Some examples include:
+the PRRTE runtime so the command line argument ``--prtemca``
+(yes, ``prte`` with a single ``r``, not two ``r``'s) must be used to
+pass the MCA parameter key/value pair. Alternatively, the MCA parameter
+key/value pair may be specific on the command line by prefixing the key with
+``PRTE_MCA_`` (again, that is not a typo: ``PRTE`` not ``PRRTE``).
+Some examples include:
 
 .. list-table::
    :header-rows: 1
@@ -1166,7 +1183,7 @@ Rankfiles are text files that specify detailed information about how
 individual processes should be mapped to nodes, and to which
 processor(s) they should be bound.  Each line of a rankfile specifies
 the location of one process (for MPI jobs, the process' "rank" refers
-to its rank in MPI_COMM_WORLD).  The general form of each line in the
+to its rank in ``MPI_COMM_WORLD``).  The general form of each line in the
 rankfile is:
 
 .. code::
@@ -1299,11 +1316,11 @@ Standard I/O
    retained, or removed?
 
 Open MPI directs UNIX standard input to ``/dev/null`` on all processes
-except the MPI_COMM_WORLD rank 0 process. The MPI_COMM_WORLD rank 0
+except the ``MPI_COMM_WORLD`` rank 0 process. The ``MPI_COMM_WORLD`` rank 0
 process inherits standard input from ``mpirun``.
 
 .. note:: The node that invoked ``mpirun`` need not be the same as the
-          node where the MPI_COMM_WORLD rank 0 process resides. Open
+          node where the ``MPI_COMM_WORLD`` rank 0 process resides. Open
           MPI handles the redirection of ``mpirun``'s standard input
           to the rank 0 process.
 
@@ -1320,7 +1337,7 @@ example:
 
    shell$ mpirun -n 2 my_app < my_input > my_output
 
-Note that in this example only the MPI_COMM_WORLD rank 0 process will
+Note that in this example only the ``MPI_COMM_WORLD`` rank 0 process will
 receive the stream from ``my_input`` on stdin.  The stdin on all the other
 nodes will be tied to ``/dev/null``.  However, the stdout from all nodes
 will be collected into the ``my_output`` file.
@@ -1643,15 +1660,15 @@ that job are designated "secondary" jobs):
 
 * If one or more processes in the primary job normally terminate with
   non-zero exit status, ``mpirun`` returns the exit status of the
-  process with the lowest MPI_COMM_WORLD rank to have a non-zero
+  process with the lowest ``MPI_COMM_WORLD`` rank to have a non-zero
   status.
 
 * If all processes in the primary job normally terminate with exit
   status 0, and one or more processes in a secondary job normally
   terminate with non-zero exit status, ``mpirun``:
 
   #. Returns the exit status of the process with the lowest
-     MPI_COMM_WORLD rank in the lowest jobid to have a non-zero
+     ``MPI_COMM_WORLD`` rank in the lowest jobid to have a non-zero
      status, and
   #. Outputs a message summarizing the exit status of the primary and
      all secondary jobs.
@@ -1662,7 +1679,7 @@ that job are designated "secondary" jobs):
   summary print statement.
 
 By default, the job will abort when any process terminates with
-non-zero status. The MCA parameter ``--PRRTEmca state_base_error_non_zero_exit``
+non-zero status. The MCA parameter ``--prtemca state_base_error_non_zero_exit``
 can be set to "false" (or "0") to cause Open MPI to not abort a job if
 one or more processes return a non-zero status. In that situation the
 Open MPI records and notes that processes exited with non-zero
@@ -1705,7 +1722,7 @@ processes exited before calling :ref:`MPI_FINALIZE(3) <mpi_finalize>`.
 If an internal error occurred in mpirun, the corresponding error code
 is returned.  In the event that one or more processes exit before
 calling :ref:`MPI_FINALIZE(3) <mpi_finalize>`, the return value of
-the MPI_COMM_WORLD rank of the process that mpirun first notices died
+the ``MPI_COMM_WORLD`` rank of the process that mpirun first notices died
 before calling :ref:`MPI_FINALIZE(3) <mpi_finalize>` will be
 returned.  Note that, in general, this will be the first process that
 died but is not guaranteed to be so.

docs/release-notes/platform.rst

@@ -27,11 +27,10 @@ that a release of Open MPI supports.
 * Systems that have been tested are:
 
   * Linux (various flavors/distros), 64 bit (x86, ppc, aarch64),
-    with gcc (>=4.8.x+), clang (>=3.6.0), Absoft (fortran), Intel,
+    with gcc/gfortran (>=7.x+), clang (>=10.x), Intel,
     and Portland (be sure to also see :ref:`the Compiler Notes
     section <compiler-notes-section-label>`)
-  * macOS (10.14-10.15, 11.x, 12.x), 64 bit (x86_64) with XCode
-    compilers
+  * macOS (14.x, 15.x), 64 bit (x86_64) with XCode compilers
 
 * Other systems have been lightly (but not fully) tested:
 

docs/tuning-apps/coll-tuned.rst

@@ -82,7 +82,7 @@ Dynamic Decisions and the Rules File
 
 Given that the best choice of algorithm for a given collective depends on a
 number of factors only known at run time, and that some of these factors may
-vary with in a run, setting an algorithm on the command line often is an
+vary within a run, setting an algorithm on the command line often is an
 ineffective means of tuning.  The rules file provides a means of choosing
 an algorithm at run time based on communicator and message size.  The rules
 file can be specified on the command line, or the other usual ways to set MCA
@@ -104,13 +104,29 @@ Dynamic tuning files are organized in this format:
 .. code-block:: sh
    :linenos:
 
-   1             # Number of collectives
-   1             # Collective ID
-   1             # Number of comm sizes
-   2             # Comm size
-   2             # Number of message sizes
-   0 1 0 0       # Message size 0, algorithm 1, topo and segmentation at 0
-   1024 2 0 0    # Message size 1024, algorithm 2, topo and segmentation at 0
+   rule-file-version-2
+   1   # num of collectives
+   3   # collective ID
+   1   # number of comm sizes
+   #=====================
+   64   # comm size
+   14   # number of rules
+   # Bytes   alg topo segs reqs
+   #----------------------
+   0            0 0 0 0
+   512000       4 0 0 64
+   1536000      4 0 0 64
+   3072000      4 0 0 64
+   6144000      4 0 0 64
+   12288000     4 0 0 16
+   24576000     4 0 0 16
+   49152000     4 0 0 16
+   98304000     4 0 0 16
+   196608000    4 0 0 8
+   393216000    4 0 0 8
+   786432000    4 0 0 1
+   1572864000   4 0 0 1
+   2621440000   0 0 0 0
 
 The rules file effectively defines, for one or more collectives, a function of
 two variables, which given communicator and message size, returns an algorithm
@@ -128,10 +144,20 @@ for details.
 One may provide rules for as many collectives, communicator sizes, and message
 sizes as desired. Simply repeat the sections as needed and adjust the relevant
 count parameters.  One must always provide a rule for message size of zero.
-Message size rules are expected in ascending order.  The last two parameters in
-the rule may or may not be used and have different meaning depending on the
-collective and algorithm.  As of writing not all of the relevant control
-parameters can be set by the rules file (See issue #12589).
+Message size rules are expected in ascending order. The last parameters in the
+message size rule may or may not be used and have different meaning depending
+on the collective and algorithm. The first two parameters in the rule following
+the algorithm ID, `topo` and `segment size`, are always required. In version 2
+of the file format a third parameter, `max requests`, may also be provided. A
+release of Open MPI at least v5.0.7 is required for version 2 features.
+
+The file format version specifier, `rule-file-version-N` where N is an integer
+greater or equal to 1, should appear on the first line of the file.  If the
+version specifier is not present, the file format is assumed to be version 1.
+Version 2 or greater is required to use the `max requests` parameter. Open MPI
+releases older than v5.0.7 do not support the file format version
+identifier. When using older releases of Open MPI do not include a version
+specifier and do not use the `max requests` parameter in message size rules.
 
 .. _CollectivesAndAlgorithms:
 

docs/tuning-apps/networking/cuda.rst

@@ -53,9 +53,12 @@ be found, please also pass ``--with-cuda-libdir``. For example:
 ``--with-cuda=<path-to-cuda> --with-cuda-libdir=/usr/local/cuda/lib64/stubs``.
 
 Open MPI supports building with CUDA libraries and running on systems
-without CUDA libraries or hardware. In order to take advantage of
-this functionality, when compiling, you have to specify the CUDA
-dependent components to be built as DSOs using the
+without CUDA libraries or hardware.
+
+For releases v5.0.2 and newer no special steps are required to get this behavior.
+
+In order to realize this behavior for the v5.0.0 and v5.0.1 releases,
+when configuring Open MPI, you have to specify the CUDA dependent components to be built as DSOs using the
 ``--enable-mca-dso=<comma-delimited-list-of-cuda-components.``
 configure option.
 

ompi/communicator/comm.c

@@ -1784,22 +1784,22 @@ int ompi_intercomm_create_from_groups (ompi_group_t *local_group, int local_lead
                 leader_procs[1] = tmp;
             }
 
-            /* create a unique tag for allocating the leader communicator. we can eliminate this step
-             * if we take a CID from the newly allocated block belonging to local_comm. this is
-             * a note to make this change at a later time. */
-            opal_asprintf (&sub_tag, "%s-OMPIi-LC", tag);
-            if (OPAL_UNLIKELY(NULL == sub_tag)) {
-                ompi_comm_free (&local_comm);
+            leader_group = ompi_group_allocate_plist_w_procs (NULL, leader_procs, 2);
+            ompi_set_group_rank (leader_group, my_proc);
+            if (OPAL_UNLIKELY(NULL == leader_group)) {
                 free(leader_procs);
+                ompi_comm_free (&local_comm);
                 return OMPI_ERR_OUT_OF_RESOURCE;
             }
 
-            leader_group = ompi_group_allocate_plist_w_procs (NULL, leader_procs, 2);
-            ompi_set_group_rank (leader_group, my_proc);
-            if (OPAL_UNLIKELY(NULL == leader_group)) {
-                free (sub_tag);
+            /* create a unique tag for allocating the leader communicator. we can eliminate this step
+             * if we take a CID from the newly allocated block belonging to local_comm. this is
+             * a note to make this change at a later time. */
+            opal_asprintf (&sub_tag, "%s-OMPIi-LC-%s", tag, OPAL_NAME_PRINT(ompi_group_get_proc_name (leader_group, 0)));
+            if (OPAL_UNLIKELY(NULL == sub_tag)) {
                 free(leader_procs);
                 ompi_comm_free (&local_comm);
+                OBJ_RELEASE(leader_group);
                 return OMPI_ERR_OUT_OF_RESOURCE;
             }
 
@@ -1809,6 +1809,7 @@ int ompi_intercomm_create_from_groups (ompi_group_t *local_group, int local_lead
             rc = ompi_comm_create_from_group (leader_group, sub_tag, info, errhandler, &leader_comm);
             OBJ_RELEASE(leader_group);
             free (sub_tag);
+            sub_tag = NULL;
             if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) {
                 free(leader_procs);
                 ompi_comm_free (&local_comm);
@@ -1864,7 +1865,16 @@ int ompi_intercomm_create_from_groups (ompi_group_t *local_group, int local_lead
         return rc;
     }
 
-    rc = ompi_comm_nextcid (newcomp, NULL, NULL, (void *) tag, NULL, false, OMPI_COMM_CID_GROUP_NEW);
+    /*
+     * append the pmix CONTEXT_ID obtained when creating the leader comm as discriminator
+     */
+    opal_asprintf (&sub_tag, "%s-%ld", tag, data[1]);
+    if (OPAL_UNLIKELY(NULL == sub_tag)) {
+        return OMPI_ERR_OUT_OF_RESOURCE;
+    }
+
+    rc = ompi_comm_nextcid (newcomp, NULL, NULL, (void *) sub_tag, NULL, false, OMPI_COMM_CID_GROUP_NEW);
+    free (sub_tag);
     if ( OMPI_SUCCESS != rc ) {
         OBJ_RELEASE(newcomp);
         return rc;

ompi/communicator/comm_cid.c

@@ -317,7 +317,7 @@ static int ompi_comm_ext_cid_new_block (ompi_communicator_t *newcomm, ompi_commu
     char *tag = NULL;
     size_t proc_count = 0, rproc_count = 0, tproc_count = 0, cid_base = 0UL, ninfo;
     int rc, leader_rank;
-    pmix_proc_t *procs;
+    pmix_proc_t *procs = NULL;
     void *grpinfo = NULL, *list = NULL;
     pmix_data_array_t darray;