Merge branch 'master' of https://github.com/ECP-Candle/Supervisor

Author: brettinanl

workflows/common/sh/langs-theta.sh

@@ -0,0 +1,16 @@
+
+# LANGS Theta
+# Language settings for Theta (Swift, Python, R, Tcl, etc.)
+
+TCL=/home/wozniak/Public/sfw/theta/tcl-8.6.1
+export R=/home/wozniak/Public/sfw/theta/R-3.4.0/lib64/R
+export PY=/home/wozniak/Public/sfw/theta/Python-2.7.12
+export LD_LIBRARY_PATH=$PY/lib:$R/lib:$LD_LIBRARY_PATH
+COMMON_DIR=$EMEWS_PROJECT_ROOT/../common/python
+PYTHONPATH=$EMEWS_PROJECT_ROOT/python:$BENCHMARK_DIR:$COMMON_DIR
+PYTHONHOME=/home/wozniak/Public/sfw/theta/Python-2.7.12
+
+# STC=/home/wozniak/Public/sfw/theta/swift-t-pyr/stc
+STC=/projects/Candle_ECP/swift/pyr/stc
+
+export PATH=$STC/bin:$TCL/bin:$PATH

workflows/common/sh/utils.sh

@@ -35,10 +35,13 @@ EOF
   fi
 
   export EXPID=$1
-  if [ $EXPID = "-a" ]; then
+  if [ $EXPID = "-a" ]
+  then
     export TURBINE_OUTPUT_ROOT=$EMEWS_PROJECT_ROOT/experiments
-    export TURBINE_OUTPUT_FORMAT=X%Q
-    EXPID=SWIFT
+    # Creates a X + a unique integer padded to 3 digits: e.g., X023
+    export TURBINE_OUTPUT_FORMAT="X%Q"
+    EXPID="AUTO"
+    shift
   else
     export TURBINE_OUTPUT=$EMEWS_PROJECT_ROOT/experiments/$EXPID
     check_directory_exists

workflows/p3b1_mlrMBO/R/mlrMBO3.R

@@ -25,7 +25,8 @@ parallelMap2 <- function(fun, ...,
   else{
     dots <- list(...)
     string_params <- elements_of_lists_to_json(dots[[1L]])
-    print(paste0("parallelMap2 called with list_param: ",string_params))
+    # print(paste0("parallelMap2 called with list_param: ",string_params))
+    # print(paste0("parallelMap2 called with list of length: ", ???)
     OUT_put(string_params)
     string_results = IN_get()
 

workflows/p3b1_mlrMBO/etc/emews_utils.sh

@@ -1,7 +1,8 @@
 log_script() {
+  # usage: Provide shell script for inclusion in log
   SCRIPT_NAME=$(basename $0)
   mkdir -p $TURBINE_OUTPUT
-  LOG_NAME="${TURBINE_OUTPUT}/${SCRIPT_NAME}.log"
+  LOG_NAME="${TURBINE_OUTPUT}/emews.log"
   echo "### VARIABLES ###" > $LOG_NAME
   set +u
   VARS=( "EMEWS_PROJECT_ROOT" "EXPID" "TURBINE_OUTPUT" \
@@ -24,7 +25,7 @@ log_script() {
 
   echo "" >> $LOG_NAME
   echo "## SCRIPT ###" >> $LOG_NAME
-  cat $EMEWS_PROJECT_ROOT/swift/$SCRIPT_NAME >> $LOG_NAME
+  cat $1 >> $LOG_NAME
 }
 
 check_directory_exists() {

workflows/p3b1_mlrMBO/python/p3b1_runner.py

@@ -8,14 +8,39 @@
 import os
 import p3b1
 import runner_utils
+import socket
+
+node_pid = "%s,%i" % (socket.gethostname(), os.getpid())
+print("node,pid: " + node_pid)
+
+logger = None
+
+def get_logger():
+    """ Set up logging """
+    global logger
+    if logger is not None:
+        return logger
+    import logging, sys
+    logger = logging.getLogger(__name__)
+    logger.setLevel(logging.DEBUG)
+    h = logging.StreamHandler(stream=sys.stdout)
+    fmtr = logging.Formatter('%(asctime)s %(name)s %(levelname)-9s %(message)s',
+                             datefmt='%Y/%m/%d %H:%M:%S')
+    h.setFormatter(fmtr)
+    logger.addHandler(h)
+    return logger
 
 def run(hyper_parameter_map):
+
+    logger = get_logger()
     framework = hyper_parameter_map['framework']
+    logger.debug("IMPORT START")
     if framework == 'keras':
         import p3b1_baseline_keras2
         pkg = p3b1_baseline_keras2
     else:
         raise ValueError("Unsupported framework: {}".format(framework))
+    logger.debug("IMPORT STOP")
 
     # params is python dictionary
     params = pkg.initialize_parameters()
@@ -26,8 +51,12 @@ def run(hyper_parameter_map):
         #    raise Exception("Parameter '{}' not found in set of valid arguments".format(k))
         params[k] = v
 
+    logger.debug("WRITE_PARAMS START")
     runner_utils.write_params(params, hyper_parameter_map)
+    logger.debug("WRITE_PARAMS STOP")
+    logger.debug("DO_N_FOLD START")
     avg_loss = pkg.do_n_fold(params)
+    logger.debug("DO_N_FOLD STOP")
 
     if framework == 'keras':
         # works around this error:
@@ -41,16 +70,24 @@ def run(hyper_parameter_map):
     return avg_loss
 
 if __name__ == '__main__':
+    logger = get_logger()
+    logger.debug("RUN START")
+
     param_string = sys.argv[1]
     instance_directory = sys.argv[2]
     framework = sys.argv[3]
     exp_id = sys.argv[4]
     run_id = sys.argv[5]
+    logger.debug("RUN INIT START")
     hyper_parameter_map = runner_utils.init(param_string, instance_directory,
                                             framework, 'save_path')
+    logger.debug("RUN INIT STOP")
     hyper_parameter_map['experiment_id'] = exp_id
     hyper_parameter_map['run_id'] = run_id
     # clear sys.argv so that argparse doesn't object
     sys.argv = ['p3b1_runner']
     result = run(hyper_parameter_map)
+    logger.debug("WRITE OUTPUT START")
     runner_utils.write_output(result, instance_directory)
+    logger.debug("WRITE OUTPUT STOP")
+    logger.debug("RUN STOP")

workflows/p3b1_mlrMBO/scripts/theta_run_model.sh

@@ -61,18 +61,29 @@ export PYTHONPATH
 arg_array=("$emews_root/python/p3b1_runner.py" "$parameter_string" "$instance_directory" "$framework"  "$exp_id" "$run_id")
 MODEL_CMD="python ${arg_array[@]}"
 
+msg()
+{
+  echo "theta_run_model.sh: $*"
+}
+
 # Turn bash error checking off. This is
 # required to properly handle the model execution return value
 # the optional timeout.
 set +e
-echo $MODEL_CMD
-$TIMEOUT_CMD python "${arg_array[@]}"
 
+# Format and report model parameters
+# as represented on Python command line:
+msg MODEL_CMD: $MODEL_CMD | \
+  tr -d "{}\""            | \
+  tr "," " "              | \
+  fmt -t -w 1
+echo
+$TIMEOUT_CMD $MODEL_CMD
 RES=$?
 if [ "$RES" -ne 0 ]; then
-	if [ "$RES" == 124 ]; then
-    echo "---> Timeout error in $MODEL_CMD"
+  if [ "$RES" == 124 ]; then
+    msg "---> Timeout error in MODEL_CMD"
   else
-	   echo "---> Error in $MODEL_CMD"
+    msg "---> Error in MODEL_CMD"
   fi
 fi

workflows/p3b1_mlrMBO/swift/ai_workflow3.swift

@@ -21,6 +21,8 @@ file model_script = input(argv("script_file"));
 file log_script = input(argv("log_script"));
 string exp_id = argv("exp_id");
 
+printf("TURBINE_OUTPUT: %s", turbine_output);
+
 string FRAMEWORK = "keras";
 
 

workflows/p3b1_mlrMBO/swift/init-theta.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+set -eu
+
+source $EMEWS_PROJECT_ROOT/etc/emews_utils.sh
+
+log_script $EMEWS_PROJECT_ROOT/swift/theta_workflow.sh
+

workflows/p3b1_mlrMBO/swift/theta_workflow.sh

@@ -1,7 +1,7 @@
 #! /usr/bin/env bash
 set -eu
 
-# CORI WORKFLOW
+# THETA WORKFLOW
 # Main entry point for P1B3 mlrMBO workflow
 
 # Autodetect this workflow directory
@@ -19,17 +19,15 @@ BENCHMARK_DIR=$EMEWS_PROJECT_ROOT/../../../Benchmarks/Pilot3/P3B1
 export PROCS=${PROCS:-10}
 
 # MPI processes per node
-# Cori has 32 cores per node, 128GB per node
+# (Theta has 64 cores per node, 192GB per node)
 export PPN=${PPN:-1}
 
-
 export QUEUE=${QUEUE:-default}
 export WALLTIME=${WALLTIME:-02:00:00}
 
 # mlrMBO settings
 # How many to runs evaluate per iteration
 
-
 MAX_BUDGET=${MAX_BUDGET:-110}
 # Total iterations
 MAX_ITERATIONS=${MAX_ITERATIONS:-4}
@@ -42,83 +40,62 @@ PARAM_SET_FILE=${PARAM_SET_FILE:-$EMEWS_PROJECT_ROOT/data/parameter_set3.R}
 
 # USER SETTINGS END
 
-
 # Source some utility functions used by EMEWS in this script
 source "${EMEWS_PROJECT_ROOT}/etc/emews_utils.sh"
 
-if [ "$#" -ne 1 ]; then
-  script_name=$(basename $0)
-  echo "Usage: ${script_name} EXPERIMENT_ID (e.g. ${script_name} experiment_1)"
-  exit 1
-fi
+WORKFLOWS_ROOT=$( cd $EMEWS_PROJECT_ROOT ; cd .. ; /bin/pwd )
+source $WORKFLOWS_ROOT/common/sh/langs-theta.sh
+source $WORKFLOWS_ROOT/common/sh/utils.sh
 
 # uncomment to turn on swift/t logging. Can also set TURBINE_LOG,
 # TURBINE_DEBUG, and ADLB_DEBUG to 0 to turn off logging
-#export TURBINE_LOG=1 TURBINE_DEBUG=1 ADLB_DEBUG=1
+# export TURBINE_LOG=1 # TURBINE_DEBUG=1 ADLB_DEBUG=1
 
-export EXPID=$1
-export TURBINE_OUTPUT_ROOT=${TURBINE_OUTPUT_ROOT:-$EMEWS_PROJECT_ROOT/experiments}
-export TURBINE_OUTPUT=$TURBINE_OUTPUT_ROOT/$EXPID
-check_directory_exists
+get_expid $*
 
 export TURBINE_JOBNAME="${EXPID}_job"
 
-# if R cannot be found, then these will need to be
-# uncommented and set correctly.
-# export R_HOME=/path/to/R
-# export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$R_HOME/lib
-# export PYTHONHOME=
-
-
-
-TCL=/home/wozniak/Public/sfw/theta/tcl-8.6.1
-export R=/home/wozniak/Public/sfw/theta/R-3.4.0/lib64/R
-export PY=/home/wozniak/Public/sfw/theta/Python-2.7.12
-export LD_LIBRARY_PATH=$PY/lib:$R/lib:$LD_LIBRARY_PATH
-COMMON_DIR=$EMEWS_PROJECT_ROOT/../common/python
-PYTHONPATH=$EMEWS_PROJECT_ROOT/python:$BENCHMARK_DIR:$COMMON_DIR
-PYTHONHOME=/home/wozniak/Public/sfw/theta/Python-2.7.12
-
-export PATH=/home/wozniak/Public/sfw/theta/swift-t-pyr/stc/bin:$TCL/bin:$PATH
-#$PYTHONHOME/bin:$TCL/bin:$PATH
-
 # Resident task workers and ranks
 export TURBINE_RESIDENT_WORK_WORKERS=1
 export RESIDENT_WORK_RANKS=$(( PROCS - 2 ))
 
 # EQ/R location
 EQR=$EMEWS_PROJECT_ROOT/ext/EQ-R
 
-CMD_LINE_ARGS="$* -pp=$PROPOSE_POINTS -mi=$MAX_ITERATIONS -mb=$MAX_BUDGET -ds=$DESIGN_SIZE "
-CMD_LINE_ARGS+="-param_set_file=$PARAM_SET_FILE -script_file=$EMEWS_PROJECT_ROOT/scripts/theta_run_model.sh "
-CMD_LINE_ARGS+="-exp_id=$EXPID -log_script=$EMEWS_PROJECT_ROOT/../common/sh/theta_run_logger.sh"
+SCRIPT_FILE=$EMEWS_PROJECT_ROOT/scripts/theta_run_model.sh
+LOG_SCRIPT=$EMEWS_PROJECT_ROOT/../common/sh/theta_run_logger.sh
 
-TURBINE_DIR=/home/wozniak/Public/sfw/theta/swift-t-pyr/turbine/lib
+CMD_LINE_ARGS=( $*
+                -exp_id=$EXPID
+                -pp=$PROPOSE_POINTS
+                -mi=$MAX_ITERATIONS
+                -mb=$MAX_BUDGET
+                -ds=$DESIGN_SIZE
+                -param_set_file=$PARAM_SET_FILE
+                -script_file=$SCRIPT_FILE
+                -log_script=$LOG_SCRIPT
+              )
 
-# set machine to your scheduler type (e.g. pbs, slurm, cobalt etc.),
-# or empty for an immediate non-queued unscheduled run
-MACHINE="theta"
-
-if [ -n "$MACHINE" ]; then
-  MACHINE="-m $MACHINE"
-fi
+TURBINE_DIR=/home/wozniak/Public/sfw/theta/swift-t-pyr/turbine/lib
 
 # Add any script variables that you want to log as
 # part of the experiment meta data to the USER_VARS array,
 # for example, USER_VARS=("VAR_1" "VAR_2")
 USER_VARS=($CMD_LINE_ARGS)
 # log variables and script to to TURBINE_OUTPUT directory
-log_script
+# log_script
 
 # echo's anything following this to standard out
 set -x
-WORKFLOW_SWIFT=ai_workflow3.swift
-swift-t -n $PROCS $MACHINE -p -I $EQR -r $EQR -r $TURBINE_DIR \
+WORKFLOW_SWIFT=$EMEWS_PROJECT_ROOT/swift/ai_workflow3.swift
+swift-t -m theta \
+        -n $PROCS \
+        -p -I $EQR -r $EQR -r $TURBINE_DIR \
+        -t i:$EMEWS_PROJECT_ROOT/swift/init-theta.sh \
         -e LD_LIBRARY_PATH=$LD_LIBRARY_PATH \
-        -e TURBINE_RESIDENT_WORK_WORKERS=$TURBINE_RESIDENT_WORK_WORKERS \
-    -e RESIDENT_WORK_RANKS=$RESIDENT_WORK_RANKS \
-    -e EMEWS_PROJECT_ROOT=$EMEWS_PROJECT_ROOT \
-    -e PYTHONPATH=$PYTHONPATH \
-    -e PYTHONHOME=$PYTHONHOME \
-    -e TURBINE_OUTPUT=$TURBINE_OUTPUT \
-        $EMEWS_PROJECT_ROOT/swift/$WORKFLOW_SWIFT $CMD_LINE_ARGS
+        -e TURBINE_RESIDENT_WORK_WORKERS \
+        -e RESIDENT_WORK_RANKS \
+        -e EMEWS_PROJECT_ROOT \
+        -e PYTHONPATH=$PYTHONPATH \
+        -e PYTHONHOME=$PYTHONHOME \
+        $WORKFLOW_SWIFT ${CMD_LINE_ARGS[@]}