Extend the schizo framework to allow definition of CLI options by environment. Refactor orterun to mesh with the orted_submit code, thus improving code reuse. Eliminate the orte-submit tool as orterun can now meet that need.

Cleanups per @jjhursey review

Author: Ralph Castain

config/orte_config_files.m4

@@ -6,7 +6,7 @@
 #                         Corporation.  All rights reserved.
 # Copyright (c) 2011-2012 Los Alamos National Security, LLC.  All rights
 #                         reserved.
-# Copyright (c) 2015      Intel, Inc. All rights reserved
+# Copyright (c) 2015-2016 Intel, Inc. All rights reserved
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -33,7 +33,6 @@ AC_DEFUN([ORTE_CONFIG_FILES],[
         orte/tools/orte-migrate/Makefile
         orte/tools/orte-info/Makefile
         orte/tools/orte-server/Makefile
-        orte/tools/orte-submit/Makefile
         orte/tools/orte-dvm/Makefile
     ])
 ])

ompi/mca/rte/orte/Makefile.am

@@ -1,7 +1,8 @@
 #
 # Copyright (c) 2012      Los Alamos National Security, LLC.
 #                         All rights reserved.
-# Copyright (c) 2014 Cisco Systems, Inc.  All rights reserved.
+# Copyright (c) 2014      Cisco Systems, Inc.  All rights reserved.
+# Copyright (c) 2016      Intel, Inc. All rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -27,7 +28,7 @@ libmca_rte_orte_la_SOURCES =$(sources) $(headers)
 libmca_rte_orte_la_LDFLAGS = -module -avoid-version
 libmca_rte_orte_la_LIBADD = $(top_builddir)/orte/lib@[email protected]
 
-man_pages = mpirun.1 mpiexec.1 ompi-ps.1 ompi-clean.1 ompi-top.1 ompi-server.1 ompi-dvm.1 ompi-submit.1
+man_pages = mpirun.1 mpiexec.1 ompi-ps.1 ompi-clean.1 ompi-top.1 ompi-server.1 ompi-dvm.1
 
 if WANT_FT
 man_pages += ompi-checkpoint.1 ompi-restart.1
@@ -44,7 +45,6 @@ install-exec-hook:
 	(cd $(DESTDIR)$(bindir); rm -f ompi-top$(EXEEXT); $(LN_S) orte-top$(EXEEXT) ompi-top$(EXEEXT))
 	(cd $(DESTDIR)$(bindir); rm -f ompi-server$(EXEEXT); $(LN_S) orte-server$(EXEEXT) ompi-server$(EXEEXT))
 	(cd $(DESTDIR)$(bindir); rm -f ompi-dvm$(EXEEXT); $(LN_S) orte-dvm$(EXEEXT) ompi-dvm$(EXEEXT))
-	(cd $(DESTDIR)$(bindir); rm -f ompi-submit$(EXEEXT); $(LN_S) orte-submit$(EXEEXT) ompi-submit$(EXEEXT))
 if WANT_FT
 	(cd $(DESTDIR)$(bindir); rm -f ompi-checkpoint$(EXEEXT); $(LN_S) orte-checkpoint$(EXEEXT) ompi-checkpoint$(EXEEXT))
 	(cd $(DESTDIR)$(bindir); rm -f ompi-restart$(EXEEXT); $(LN_S) orte-restart$(EXEEXT) ompi-restart$(EXEEXT))
@@ -58,8 +58,7 @@ uninstall-local:
 		$(DESTDIR)$(bindir)/ompi-clean$(EXEEXT) \
 		$(DESTDIR)$(bindir)/ompi-top$(EXEEXT) \
 		$(DESTDIR)$(bindir)/ompi-server$(EXEEXT) \
-		$(DESTDIR)$(bindir)/ompi-dvm$(EXEEXT) \
-		$(DESTDIR)$(bindir)/ompi-submit$(EXEEXT)
+		$(DESTDIR)$(bindir)/ompi-dvm$(EXEEXT)
 if WANT_FT
 	rm -f $(DESTDIR)$(bindir)/ompi-checkpoint$(EXEEXT) \
 		$(DESTDIR)$(bindir)/ompi-restart$(EXEEXT) \
@@ -122,8 +121,5 @@ ompi-server.1: $(top_builddir)/orte/tools/orte-server/orte-server.1
 ompi-dvm.1: $(top_builddir)/orte/tools/orte-dvm/orte-dvm.1
 	cp -f $(top_builddir)/orte/tools/orte-dvm/orte-dvm.1 ompi-dvm.1
 
-ompi-submit.1: $(top_builddir)/orte/tools/orte-submit/orte-submit.1
-	cp -f $(top_builddir)/orte/tools/orte-submit/orte-submit.1 ompi-submit.1
-
 clean-local:
 	rm -f $(man_pages)

opal/mca/base/base.h

@@ -156,7 +156,7 @@ OPAL_DECLSPEC int mca_base_is_component_required(opal_list_t *components_availab
 /* mca_base_cmd_line.c */
 
 OPAL_DECLSPEC int mca_base_cmd_line_setup(opal_cmd_line_t *cmd);
-OPAL_DECLSPEC int mca_base_cmd_line_process_args(opal_cmd_line_t *cmd,
+OPAL_DECLSPEC int mca_base_cmd_line_process_args(char **argv,
                                                  char ***app_env,
                                                  char ***global_env);
 OPAL_DECLSPEC void mca_base_cmd_line_wrap_args(char **args);

opal/mca/base/mca_base_cmd_line.c

@@ -94,29 +94,25 @@ int mca_base_cmd_line_setup(opal_cmd_line_t *cmd)
 /*
  * Look for and handle any -mca options on the command line
  */
-int mca_base_cmd_line_process_args(opal_cmd_line_t *cmd,
+int mca_base_cmd_line_process_args(char **argv,
                                    char ***context_env, char ***global_env)
 {
-    int i, num_insts, rc;
+    int i, rc;
     char **params;
     char **values;
 
-    /* If no relevant parameters were given, just return */
-
-    if (!opal_cmd_line_is_taken(cmd, OPAL_MCA_CMD_LINE_ID) &&
-        !opal_cmd_line_is_taken(cmd, "g"OPAL_MCA_CMD_LINE_ID)) {
-        return OPAL_SUCCESS;
-    }
-
-    /* Handle app context-specific parameters */
-
-    num_insts = opal_cmd_line_get_ninsts(cmd, OPAL_MCA_CMD_LINE_ID);
     params = values = NULL;
-    for (i = 0; i < num_insts; ++i) {
-        if (OPAL_SUCCESS != (rc = process_arg(opal_cmd_line_get_param(cmd, OPAL_MCA_CMD_LINE_ID, i, 0),
-                                              opal_cmd_line_get_param(cmd, OPAL_MCA_CMD_LINE_ID, i, 1),
-                                              &params, &values))) {
-            return rc;
+    for (i = 0; NULL != argv[i]; ++i) {
+        if (0 == strcmp("-"OPAL_MCA_CMD_LINE_ID,  argv[i]) ||
+            0 == strcmp("--"OPAL_MCA_CMD_LINE_ID, argv[i])) {
+            if (NULL == argv[i+1] || NULL == argv[i+2]) {
+                return OPAL_ERR_BAD_PARAM;
+            }
+            if (OPAL_SUCCESS != (rc = process_arg(argv[i+1], argv[i+2],
+                                                  &params, &values))) {
+                return rc;
+            }
+            i += 2;
         }
     }
     if (NULL != params) {
@@ -125,15 +121,19 @@ int mca_base_cmd_line_process_args(opal_cmd_line_t *cmd,
         opal_argv_free(values);
     }
 
-    /* Handle global parameters */
 
-    num_insts = opal_cmd_line_get_ninsts(cmd, "g"OPAL_MCA_CMD_LINE_ID);
     params = values = NULL;
-    for (i = 0; i < num_insts; ++i) {
-        if (OPAL_SUCCESS != (rc = process_arg(opal_cmd_line_get_param(cmd, "g"OPAL_MCA_CMD_LINE_ID, i, 0),
-                                              opal_cmd_line_get_param(cmd, "g"OPAL_MCA_CMD_LINE_ID, i, 1),
-                                              &params, &values))) {
-            return rc;
+    for (i = 0; NULL != argv[i]; ++i) {
+        if (0 == strcmp("-g"OPAL_MCA_CMD_LINE_ID,  argv[i]) ||
+            0 == strcmp("--g"OPAL_MCA_CMD_LINE_ID, argv[i])) {
+            if (NULL == argv[i+1] || NULL == argv[i+2]) {
+                return OPAL_ERR_BAD_PARAM;
+            }
+            if (OPAL_SUCCESS != (rc = process_arg(argv[i+1], argv[i+2],
+                                                  &params, &values))) {
+                return rc;
+            }
+            i += 2;
         }
     }
     if (NULL != params) {
@@ -190,7 +190,6 @@ static int process_arg(const char *param, const char *value,
 
     /* If we didn't already have an value for the same param, save
        this one away */
-
     opal_argv_append_nosize(params, param);
     opal_argv_append_nosize(values, p1);
     free(p1);

opal/runtime/opal_info_support.c

@@ -207,7 +207,7 @@ int opal_info_init(int argc, char **argv,
         exit(cmd_error ? 1 : 0);
     }
 
-    mca_base_cmd_line_process_args(opal_info_cmd_line, &app_env, &global_env);
+    mca_base_cmd_line_process_args(argv, &app_env, &global_env);
 
 
     /* set the flags */

opal/util/cmd_line.c

@@ -156,7 +156,9 @@ int opal_cmd_line_create(opal_cmd_line_t *cmd,
     }
     OBJ_CONSTRUCT(cmd, opal_cmd_line_t);
 
-    ret = opal_cmd_line_add(cmd, table);
+    if (NULL != table) {
+        ret = opal_cmd_line_add(cmd, table);
+    }
     return ret;
 }
 

orte/mca/ess/hnp/ess_hnp_module.c

@@ -76,6 +76,7 @@
 #include "orte/mca/state/base/base.h"
 #include "orte/mca/state/state.h"
 
+#include "orte/orted/orted_submit.h"
 #include "orte/orted/pmix/pmix_server.h"
 
 #include "orte/util/show_help.h"
@@ -713,6 +714,14 @@ static int rte_init(void)
         goto error;
     }
 
+    /* setup to support debugging */
+    orte_state.add_job_state(ORTE_JOB_STATE_READY_FOR_DEBUGGERS,
+                             orte_debugger_init_after_spawn,
+                             ORTE_SYS_PRI);
+    orte_state.add_job_state(ORTE_JOB_STATE_DEBUGGER_DETACH,
+                             orte_debugger_detached,
+                             ORTE_SYS_PRI);
+
     /* if a tool has launched us and is requesting event reports,
      * then set its contact info into the comm system
      */

orte/mca/grpcomm/base/grpcomm_base_stubs.c

@@ -324,7 +324,7 @@ static int create_dmns(orte_grpcomm_signature_t *sig,
             *dmns = NULL;
             return ORTE_ERR_NOT_FOUND;
         }
-        if (NULL == jdata->map) {
+        if (0 == jdata->map->num_nodes) {
             /* we haven't generated a job map yet - if we are the HNP,
              * then we should only involve ourselves. Otherwise, we have
              * no choice but to abort to avoid hangs */
@@ -340,12 +340,6 @@ static int create_dmns(orte_grpcomm_signature_t *sig,
             *dmns = NULL;
             return ORTE_ERR_NOT_FOUND;
         }
-        /* get the array */
-        if (0 == jdata->map->num_nodes) {
-            ORTE_UPDATE_EXIT_STATUS(ORTE_ERROR_DEFAULT_EXIT_CODE);
-            ORTE_ERROR_LOG(ORTE_ERR_BAD_PARAM);
-            return ORTE_ERR_SILENT;
-        }
         dns = (orte_vpid_t*)malloc(jdata->map->num_nodes * sizeof(vpid));
         nds = 0;
         for (i=0; i < jdata->map->nodes->size && (int)nds < jdata->map->num_nodes; i++) {

orte/mca/plm/base/plm_base_launch_support.c

@@ -1537,6 +1537,9 @@ int orte_plm_base_setup_virtual_machine(orte_job_t *jdata)
         ORTE_ERROR_LOG(ORTE_ERR_NOT_FOUND);
         return ORTE_ERR_NOT_FOUND;
     }
+    if (NULL == daemons->map) {
+        daemons->map = OBJ_NEW(orte_job_map_t);
+    }
     map = daemons->map;
 
     /* if this job is being launched against a fixed DVM, then there is
@@ -1552,8 +1555,7 @@ int orte_plm_base_setup_virtual_machine(orte_job_t *jdata)
      * the virtual machine unless specifically requested to do so
      */
     if (ORTE_JOBID_INVALID != jdata->originator.jobid) {
-        OBJ_CONSTRUCT(&nodes, opal_list_t);
-        if (NULL == daemons->map) {
+        if (0 == map->num_nodes) {
             OPAL_OUTPUT_VERBOSE((5, orte_plm_base_framework.framework_output,
                                  "%s plm:base:setup_vm creating map",
                                  ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
@@ -1562,16 +1564,15 @@ int orte_plm_base_setup_virtual_machine(orte_job_t *jdata)
              * are obviously already here! The ess will already
              * have assigned our node to us.
              */
-            daemons->map = OBJ_NEW(orte_job_map_t);
             node = (orte_node_t*)opal_pointer_array_get_item(orte_node_pool, 0);
-            opal_pointer_array_add(daemons->map->nodes, (void*)node);
-            ++(daemons->map->num_nodes);
+            opal_pointer_array_add(map->nodes, (void*)node);
+            ++(map->num_nodes);
             /* maintain accounting */
             OBJ_RETAIN(node);
             /* mark that this is from a singleton */
             singleton = true;
         }
-        map = daemons->map;
+        OBJ_CONSTRUCT(&nodes, opal_list_t);
         for (i=1; i < orte_node_pool->size; i++) {
             if (NULL == (node = (orte_node_t*)opal_pointer_array_get_item(orte_node_pool, i))) {
                 continue;
@@ -1618,16 +1619,6 @@ int orte_plm_base_setup_virtual_machine(orte_job_t *jdata)
      */
     if (orte_get_attribute(&daemons->attributes, ORTE_JOB_NO_VM, NULL, OPAL_BOOL)) {
         OBJ_CONSTRUCT(&nodes, opal_list_t);
-        if (NULL == daemons->map) {
-            OPAL_OUTPUT_VERBOSE((5, orte_plm_base_framework.framework_output,
-                                 "%s plm:base:setup_vm creating map",
-                                 ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
-            /* this is the first time thru, so the vm is just getting
-             * defined - create a map for it
-             */
-            daemons->map = OBJ_NEW(orte_job_map_t);
-        }
-        map = daemons->map;
         /* loop across all nodes and include those that have
          * num_procs > 0 && no daemon already on them
          */
@@ -1685,23 +1676,21 @@ int orte_plm_base_setup_virtual_machine(orte_job_t *jdata)
         goto process;
     }
 
-    if (NULL == daemons->map) {
+    if (0 == map->num_nodes) {
         OPAL_OUTPUT_VERBOSE((5, orte_plm_base_framework.framework_output,
                              "%s plm:base:setup_vm creating map",
                              ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
         /* this is the first time thru, so the vm is just getting
-         * defined - create a map for it and put us in as we
+         * defined - put us in as we
          * are obviously already here! The ess will already
          * have assigned our node to us.
          */
-        daemons->map = OBJ_NEW(orte_job_map_t);
         node = (orte_node_t*)opal_pointer_array_get_item(orte_node_pool, 0);
-        opal_pointer_array_add(daemons->map->nodes, (void*)node);
-        ++(daemons->map->num_nodes);
+        opal_pointer_array_add(map->nodes, (void*)node);
+        ++(map->num_nodes);
         /* maintain accounting */
         OBJ_RETAIN(node);
     }
-    map = daemons->map;
 
     /* zero-out the number of new daemons as we will compute this
      * each time we are called

orte/mca/rmaps/ppr/rmaps_ppr.c

@@ -110,8 +110,10 @@ static int ppr_mapper(orte_job_t *jdata)
         ORTE_MAPPING_PPR != ORTE_GET_MAPPING_POLICY(jdata->map->mapping)) {
         /* not for us */
         opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
-                            "mca:rmaps:ppr: job %s not using ppr mapper",
-                            ORTE_JOBID_PRINT(jdata->jobid));
+                            "mca:rmaps:ppr: job %s not using ppr mapper PPR %s policy %s",
+                            ORTE_JOBID_PRINT(jdata->jobid),
+                            (NULL == jdata->map->ppr) ? "NULL" : jdata->map->ppr,
+                            (ORTE_MAPPING_PPR == ORTE_GET_MAPPING_POLICY(jdata->map->mapping)) ? "PPRSET" : "PPR NOTSET");
         return ORTE_ERR_TAKE_NEXT_OPTION;
     }