Create an mpirun dedicated executable

If we install the embedded versions or PRRTE and PMIx, then
OPAL_PREFIX needs to adjust the installed location for PMIx
as well as for PRRTE when launching via mpirun.

In mpirun: Detect OPAL_PREFIX and set the corresponding PRRTE and PMIx
prefixes. Take advantage of the simplified proxy detection
support in PRRTE. Deal with absolute path names

In MPI_Init: Detect OPAL_PREFIX and set the corresponding PMIx
prefix

Signed-off-by: Ralph Castain <[email protected]>

Author: rhc54

.gitignore

@@ -341,6 +341,7 @@ ompi/tools/mpisync/mpisync
 ompi/tools/mpisync/mpirun_prof
 ompi/tools/mpisync/ompi_timing_post
 ompi/tools/mpisync/mpisync.1
+ompi/tools/mpirun/mpirun
 
 ompi/tools/ompi_info/ompi_info
 ompi/tools/ompi_info/ompi_info.1

3rd-party/openpmix

@@ -98,7 +98,9 @@ AC_DEFUN([OMPI_SETUP_PRRTE],[
     # so try the internal version.
     AS_IF([test "$prrte_setup_external_happy" = "0" -a "$prrte_setup_internal_happy" = "1"],
           [opal_prrte_mode="internal"
-           _OMPI_SETUP_PRRTE_INTERNAL_POST()])
+           OMPI_USING_INTERNAL_PRRTE=1
+           _OMPI_SETUP_PRRTE_INTERNAL_POST()],
+          [OMPI_USING_INTERNAL_PRRTE=0])
 
     AS_IF([test "$opal_prrte_mode" != "disabled"],
           [AS_IF([test "$prrte_setup_external_happy" = "0" -a "$prrte_setup_internal_happy" = "0"],
@@ -115,6 +117,10 @@ AC_DEFUN([OMPI_SETUP_PRRTE],[
                        [$OMPI_HAVE_PRRTE],
                        [Whether or not PRRTE is available])
 
+    AC_DEFINE_UNQUOTED([OMPI_USING_INTERNAL_PRRTE],
+                       [$OMPI_USING_INTERNAL_PRRTE],
+                       [Whether or not we are using the internal PRRTE])
+
     OPAL_SUMMARY_ADD([[Miscellaneous]], [[prrte]], [prrte], [$opal_prrte_mode])
 
     OPAL_VAR_SCOPE_POP

3rd-party/prrte

@@ -122,11 +122,17 @@ AC_DEFUN([OPAL_CONFIG_PMIX], [
     # so try the internal version.
     AS_IF([test "$external_pmix_happy" = "0" -a "$internal_pmix_happy" = "1"],
           [opal_pmix_mode="internal"
-           _OPAL_CONFIG_PMIX_INTERNAL_POST()])
+           OPAL_USING_INTERNAL_PMIX=1
+           _OPAL_CONFIG_PMIX_INTERNAL_POST()],
+          [OPAL_USING_INTERNAL_PMIX=0])
 
     AS_IF([test "$external_pmix_happy" = "0" -a "$internal_pmix_happy" = "0"],
           [AC_MSG_ERROR([Could not find viable pmix build.])])
 
+    AC_DEFINE_UNQUOTED([OPAL_USING_INTERNAL_PMIX],
+                       [$OPAL_USING_INTERNAL_PMIX],
+                       [Whether or not we are using the internal PMIx])
+
     AC_SUBST(opal_pmix_CPPFLAGS)
     AC_SUBST(opal_pmix_LDFLAGS)
     AC_SUBST(opal_pmix_LIBS)

config/ompi_setup_prrte.m4

@@ -397,6 +397,7 @@ int ompi_mpi_init(int argc, char **argv, int requested, int *provided,
     ompi_proc_t** procs;
     size_t nprocs;
     char *error = NULL;
+    char *evar;
     volatile bool active;
     bool background_fence = false;
     pmix_info_t info[2];
@@ -439,6 +440,14 @@ int ompi_mpi_init(int argc, char **argv, int requested, int *provided,
         }
     }
 
+    /* deal with OPAL_PREFIX to ensure that an internal PMIx installation
+     * is also relocated if necessary */
+#if OPAL_USING_INTERNAL_PMIX
+    if (NULL != (evar = getenv("OPAL_PREFIX"))) {
+        opal_setenv("PMIX_PREFIX", evar, true, &environ);
+    }
+#endif
+
     /* Figure out the final MPI thread levels.  If we were not
        compiled for support for MPI threads, then don't allow
        MPI_THREAD_MULTIPLE.  Set this stuff up here early in the

config/opal_config_pmix.m4

@@ -3,6 +3,7 @@
 # Copyright (c) 2020      IBM Corporation.  All rights reserved.
 # Copyright (c) 2021      Amazon.com, Inc. or its affiliates.
 #                         All Rights reserved.
+# Copyright (c) 2021      Nanook Consulting.  All rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -11,10 +12,18 @@
 #
 
 if OMPI_WANT_PRRTE
+
+bin_PROGRAMS = mpirun
+
+mpirun_SOURCES = \
+    main.c
+
+mpirun_LDADD = \
+	$(top_builddir)/opal/libopen-pal.la
+
 install-exec-hook:
-	(cd $(DESTDIR)$(bindir); rm -f mpirun$(EXEEXT); $(LN_S) $(PRTE_PATH)$(EXEEXT) mpirun$(EXEEXT))
-	(cd $(DESTDIR)$(bindir); rm -f mpiexec$(EXEEXT); $(LN_S) $(PRTE_PATH)$(EXEEXT) mpiexec$(EXEEXT))
-	(cd $(DESTDIR)$(bindir); rm -f oshrun$(EXEEXT); $(LN_S) $(PRTE_PATH)$(EXEEXT) oshrun$(EXEEXT))
+	(cd $(DESTDIR)$(bindir); rm -f mpiexec$(EXEEXT); $(LN_S) mpirun$(EXEEXT) mpiexec$(EXEEXT))
+	(cd $(DESTDIR)$(bindir); rm -f oshrun$(EXEEXT); $(LN_S) mpirun$(EXEEXT) oshrun$(EXEEXT))
 
 uninstall-local:
 	rm -f $(DESTDIR)$(bindir)/mpirun$(EXEEXT) \

ompi/runtime/ompi_mpi_init.c

@@ -0,0 +1,117 @@
+/***************************************************************************
+ *                                                                         *
+ *          Open MPI: Open Source High Performance Computing               *
+ *                                                                         *
+ *                   http://www.open-mpi.org/                              *
+ *                                                                         *
+ ***************************************************************************/
+#include "ompi_config.h"
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#if HAVE_SYS_STAT_H
+#    include <sys/stat.h>
+#endif /* HAVE_SYS_STAT_H */
+
+#include "opal/mca/base/base.h"
+#include "opal/mca/installdirs/base/base.h"
+#include "opal/util/argv.h"
+#include "opal/util/basename.h"
+#include "opal/util/opal_environ.h"
+#include "opal/util/os_dirpath.h"
+#include "opal/util/os_path.h"
+#include "opal/util/path.h"
+
+int main(int argc, char *argv[])
+{
+    char *evar, *pvar;
+    char **pargs = NULL;
+    char *pfx = NULL;
+    int m, param_len;
+    char *truepath;
+
+    if (NULL != (evar = getenv("OPAL_PREFIX"))) {
+
+#if OMPI_USING_INTERNAL_PRRTE
+        (void)asprintf(&pvar, "PRTE_PREFIX=%s", evar);
+        putenv(pvar);
+#endif
+
+#if OPAL_USING_INTERNAL_PMIX
+        (void)asprintf(&pvar, "PMIX_PREFIX=%s", evar);
+        putenv(pvar);
+#endif
+    }
+    putenv("PRTE_MCA_schizo_proxy=ompi");
+
+    opal_argv_append_nosize(&pargs, "prterun");
+    for (m=1; NULL != argv[m]; m++) {
+        opal_argv_append_nosize(&pargs, argv[m]);
+        /* Did the user specify a prefix, or want prefix by default? */
+        if (0 == strcmp(argv[m], "--prefix")) {
+            pfx = strdup(argv[m+1]);
+        }
+    }
+
+    if (NULL != pfx) {
+        /* "Parse" the param, aka remove superfluous path_sep. */
+        param_len = strlen(pfx);
+        while (0 == strcmp(OPAL_PATH_SEP, &(pfx[param_len - 1]))) {
+            pfx[param_len - 1] = '\0';
+            param_len--;
+            if (0 == param_len) {
+                fprintf(stderr, "A prefix was supplied to mpirun that only contained slashes.\n"
+                        "This is a fatal error; mpirun will now abort.\nNo processes were launched.\n");
+                exit(1);
+            }
+        }
+    } else if (opal_path_is_absolute(argv[0])) {
+        /* Check if called with fully-qualified path to mpirun.
+         * (Note: Put this second so can override with --prefix (above). */
+        pfx = opal_dirname(argv[0]);
+#if OMPI_USING_INTERNAL_PRRTE
+    } else {
+        /* in case --enable-prefix-by-default was given */
+        mca_base_framework_open(&opal_installdirs_base_framework, 0);  // fill in the installdirs
+        if (NULL != opal_install_dirs.prefix) {
+            pfx = strdup(opal_install_dirs.prefix);
+        }
+#endif
+    }
+
+    if (NULL == pfx) {
+        truepath = opal_path_findv("prterun", X_OK, environ, NULL);
+    } else {
+        truepath = opal_os_path(0, pfx, "prterun", NULL);
+        free(pfx);
+    }
+
+    if (NULL == truepath) {
+        fprintf(stderr, "prterun executable could not be found - unable to run\n");
+        exit(1);
+    }
+
+    execve(truepath, pargs, environ);
+}
+
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2017-2020 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2020      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2021      Nanook Consulting.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */